From 13734a74316373b6e05dee8576eaaa360119c295 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 05:32:15 +0000 Subject: [PATCH 1/4] fix(controller): separate queue wait from execution deadline The execution deadline previously started at enqueue time while a single worker drains the queue, so a task could be killed (or expire before it ever ran) purely because of work ahead of it. - Execution deadline (FORGE_TASK_TIMEOUT) now starts at dispatch. - New FORGE_QUEUE_TIMEOUT (default 600s) bounds queue wait; expiry is the distinct terminal error 'queue_expired', decided before the task is marked running and without consuming inference. - Docs updated; this intentionally changes documented v1 semantics to meet docs/v2 SRV-04 / MCP-07 (queue wait separate from execution time). --- docs/controller-architecture.md | 2 +- docs/controller.md | 5 ++- docs/mcp.md | 4 +- scripts/controller.py | 2 +- src/forge_controller/app.py | 16 ++++++-- src/forge_controller/config.py | 4 ++ tests/test_queue_deadline.py | 70 +++++++++++++++++++++++++++++++++ 7 files changed, 93 insertions(+), 10 deletions(-) create mode 100644 tests/test_queue_deadline.py diff --git a/docs/controller-architecture.md b/docs/controller-architecture.md index 69c53d2..698738e 100644 --- a/docs/controller-architecture.md +++ b/docs/controller-architecture.md @@ -6,7 +6,7 @@ Use Python 3.12+, FastAPI/Pydantic for strict API inputs, HTTPX for asynchronous The controller owns task lifecycle, identity, tool authorization, deadlines, and audit metadata. The model proposes actions; code checks the tool name, declared task capabilities, argument shape, and authenticated principal's resource grant before executing. The current tool returns a constant synthetic result. Future real tools must provide independent domain-side authorization and explicit person/resource/action scopes. -A single bounded worker consumes an in-memory queue. SQLite persists task IDs, owner, agent, status, times, results, errors, and redacted tool audit metadata. Prompts and intermediate messages stay in memory. Accepted unfinished tasks fail as `interrupted` after shutdown/restart; they are never replayed silently. SQLite task records are authoritative for lifecycle, not domain memory. Admission has a global capacity; queue wait counts toward the task deadline. +A single bounded worker consumes an in-memory queue. SQLite persists task IDs, owner, agent, status, times, results, errors, and redacted tool audit metadata. Prompts and intermediate messages stay in memory. Accepted unfinished tasks fail as `interrupted` after shutdown/restart; they are never replayed silently. SQLite task records are authoritative for lifecycle, not domain memory. Admission has a global capacity. Queue wait and execution are separate budgets: the execution deadline starts at dispatch, and a task that waits longer than the queue limit fails as `queue_expired` without any inference. One controller process owns one database, enforced with a Unix file lock. Do not use multiple Uvicorn workers or replicas. There is no broker, workflow framework, container requirement, vector database, or distributed scheduler. Fair scheduling, durable resumable execution, and multiple replicas are future requirements, not promises of this prototype. diff --git a/docs/controller.md b/docs/controller.md index 8fb95bb..5012e24 100644 --- a/docs/controller.md +++ b/docs/controller.md @@ -281,7 +281,8 @@ with context 4096 in the prototype comparison. | `FORGE_EXAMPLES_FILE` | Optional reviewed example collection; see [feedback loop](feedback-loop.md) | | `FORGE_CONTEXT_LENGTH` | Positive integer; default 4096 context tokens | | `FORGE_MAX_OUTPUT_TOKENS` | Positive integer; default 512 output tokens | -| `FORGE_TASK_TIMEOUT` | 60 seconds including queue wait | +| `FORGE_TASK_TIMEOUT` | 60 seconds of execution, measured from dispatch (queue wait is not counted) | +| `FORGE_QUEUE_TIMEOUT` | 600 seconds a task may wait before dispatch; positive number | | `FORGE_CAPACITY` | 8 admitted tasks, including running work | | `FORGE_TOOL_ROUNDS` | Maximum 3 total tool calls | | `FORGE_CONTROLLER_URL` | Client URL, default `http://127.0.0.1:8787` | @@ -303,7 +304,7 @@ All routes except liveness require `Authorization: Bearer `. There is no | `POST /v1/tasks/{id}/cancel` | Cancel queued/running task; completed records remain terminal | | `DELETE /v1/tasks/{id}` | Delete settled terminal record; 409 while active | -Statuses: `queued`, `running`, `succeeded`, `failed`, `cancelled`. Errors include `model_unavailable`, `invalid_model_response`, `tool_denied`, `tool_limit_exceeded`, `deadline_exceeded`, and `interrupted`. Unknown or another owner's ID returns 404. Invalid credentials return 401; forbidden capability requests 403; invalid input 422; large bodies 413; full admission capacity 503. Cancellation prevents result publication but cannot undo a remote action; only synthetic read-only actions exist here. +Statuses: `queued`, `running`, `succeeded`, `failed`, `cancelled`. Errors include `model_unavailable`, `invalid_model_response`, `tool_denied`, `tool_limit_exceeded`, `deadline_exceeded` (execution exceeded its limit), `queue_expired` (waited too long before dispatch; no inference ran), and `interrupted`. Unknown or another owner's ID returns 404. Invalid credentials return 401; forbidden capability requests 403; invalid input 422; large bodies 413; full admission capacity 503. Cancellation prevents result publication but cannot undo a remote action; only synthetic read-only actions exist here. ## Operation and portability diff --git a/docs/mcp.md b/docs/mcp.md index 3320319..c893921 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -128,8 +128,8 @@ There is no automatic fallback to the old host. Submit analyst/implementer tasks with `read_repository_file` in their declared tools and specific source paths/ranges. Poll using the returned ID until -`succeeded`, `failed` or `cancelled`. Avoid busy polling; queue wait counts toward -the controller deadline. Every inference round uses the configured role model. +`succeeded`, `failed` or `cancelled`. Avoid busy polling; queue wait is limited +separately (`queue_expired`) and does not count toward the execution deadline. Every inference round uses the configured role model. The bridge performs no implicit retries: a lost submission response may correspond to an accepted task. Do not blindly resubmit. Retain known task IDs across client reconnections; disconnecting does not cancel work. Use explicit cancellation. diff --git a/scripts/controller.py b/scripts/controller.py index 6d37dc9..700747b 100644 --- a/scripts/controller.py +++ b/scripts/controller.py @@ -81,7 +81,7 @@ def start(config_path): allowed = {'FORGE_PRINCIPALS_FILE', 'FORGE_REPOSITORIES_FILE', 'FORGE_EXAMPLES_FILE', 'FORGE_DATABASE', 'FORGE_MODEL', 'FORGE_ROLE_MODELS', 'FORGE_TOOL_ROUNDS', 'FORGE_CONTEXT_LENGTH', 'FORGE_MAX_OUTPUT_TOKENS', 'FORGE_TASK_TIMEOUT', - 'FORGE_CAPACITY', 'FORGE_MODEL_URL'} | LISTENER_KEYS + 'FORGE_CAPACITY', 'FORGE_QUEUE_TIMEOUT', 'FORGE_MODEL_URL'} | LISTENER_KEYS if not isinstance(config, dict) or set(config)-allowed or any(not isinstance(v,str) for v in config.values()): raise SystemExit('Invalid local configuration') command = server_command(config) diff --git a/src/forge_controller/app.py b/src/forge_controller/app.py index 81121b7..9b24b2f 100644 --- a/src/forge_controller/app.py +++ b/src/forge_controller/app.py @@ -67,6 +67,10 @@ class TaskInput(BaseModel): tools: list[str] = Field(default_factory=list, max_length=3) +class QueueExpired(Exception): + """Task waited longer than queue_timeout before dispatch; no inference ran.""" + + class Engine: def __init__(self, settings, store, model): self.settings, self.store, self.model = settings, store, model @@ -163,14 +167,16 @@ async def work(self): try: if self.store.get(task_id)['status'] in TERMINAL: continue + # Queue wait and execution are separate budgets. The execution + # deadline starts at dispatch; waiting too long is its own outcome + # and is decided before the task is ever marked running. + if time.monotonic() - queued_at >= self.settings.queue_timeout: + raise QueueExpired() self.store.update(task_id, 'running') - remaining = self.settings.timeout - (time.monotonic() - queued_at) - if remaining <= 0: - raise TimeoutError() running = asyncio.create_task(self.run(task_id, owner, task, binding)) self.active[task_id] = running try: - async with asyncio.timeout(remaining): + async with asyncio.timeout(self.settings.timeout): result, audit = await running self.store.update(task_id, 'succeeded', result=result, audit=audit) except asyncio.CancelledError: @@ -178,6 +184,8 @@ async def work(self): if self.closing: self.store.update(task_id, 'failed', error='interrupted', audit=self.store.get(task_id)['audit']) raise + except QueueExpired: + self.store.update(task_id, 'failed', error='queue_expired', audit=self.store.get(task_id)['audit']) except TimeoutError: self.store.update(task_id, 'failed', error='deadline_exceeded', audit=self.store.get(task_id)['audit']) except ModelError as exc: diff --git a/src/forge_controller/config.py b/src/forge_controller/config.py index 08e4527..083a43e 100644 --- a/src/forge_controller/config.py +++ b/src/forge_controller/config.py @@ -20,6 +20,7 @@ class Settings: repositories: dict[str, dict] = field(default_factory=dict) examples_file: Path | None = None role_models: dict[str, str] = field(default_factory=dict) + queue_timeout: float = 600 def __post_init__(self): if not isinstance(self.role_models, dict) or any( @@ -37,6 +38,8 @@ def __post_init__(self): raise ValueError("Invalid model URL") if self.timeout <= 0 or self.capacity < 1 or self.tool_rounds < 1: raise ValueError("Limits must be positive") + if type(self.queue_timeout) not in {int, float} or self.queue_timeout <= 0: + raise ValueError("queue_timeout must be a positive number") if not self.model or not self.principals: raise ValueError("Model and principals are required") tokens = [] @@ -93,6 +96,7 @@ def from_environment(cls): context_length=int(os.environ.get("FORGE_CONTEXT_LENGTH", "4096")), max_output_tokens=int(os.environ.get("FORGE_MAX_OUTPUT_TOKENS", "512")), timeout=float(os.environ.get("FORGE_TASK_TIMEOUT", "60")), + queue_timeout=float(os.environ.get("FORGE_QUEUE_TIMEOUT", "600")), capacity=int(os.environ.get("FORGE_CAPACITY", "8")), tool_rounds=int(os.environ.get("FORGE_TOOL_ROUNDS", "3")), examples_file=Path(os.environ["FORGE_EXAMPLES_FILE"]).expanduser() if os.environ.get("FORGE_EXAMPLES_FILE") else None, diff --git a/tests/test_queue_deadline.py b/tests/test_queue_deadline.py new file mode 100644 index 0000000..25de68f --- /dev/null +++ b/tests/test_queue_deadline.py @@ -0,0 +1,70 @@ +"""Queue wait and execution time are separate budgets (v2 SRV-04, MCP-07).""" +import asyncio +import json + +import pytest + +from forge_controller.config import Settings +from tests.test_controller import FakeModel, client, submit, terminal + + +class SlowModel(FakeModel): + def __init__(self, delay): + super().__init__() + self.delay = delay + + async def chat(self, messages, tools, *, agent='forge'): + await asyncio.sleep(self.delay) + return await super().chat(messages, tools, agent=agent) + + +async def test_queue_wait_is_not_charged_to_execution_deadline(tmp_path): + # Three 0.15s tasks run serially under a 0.3s execution limit. The third task + # waits ~0.3s in the queue; with a queue-inclusive deadline it would have failed. + async with client(tmp_path, SlowModel(.15), timeout=.3, queue_timeout=30) as (c, app): + ids = [await submit(c) for _ in range(3)] + rows = [await terminal(c, task_id) for task_id in ids] + assert [row['status'] for row in rows] == ['succeeded'] * 3, rows + + +async def test_queue_expiry_is_distinct_and_never_runs_inference(tmp_path): + fake = FakeModel('wait') + async with client(tmp_path, fake, timeout=5, queue_timeout=.05, capacity=4) as (c, app): + first = await submit(c) + await fake.started.wait() + second = await submit(c) + await asyncio.sleep(.1) # second outlives its queue budget + fake.release.set() + assert (await terminal(c, first))['status'] == 'succeeded' + row = await terminal(c, second) + assert row['status'] == 'failed' and row['error'] == 'queue_expired' + assert row['audit'] == [] # never reached the running state + assert fake.calls == 1 # only the first task consumed inference + + +async def test_execution_deadline_still_enforced_after_dispatch(tmp_path): + fake = FakeModel('wait') + async with client(tmp_path, fake, timeout=.05, queue_timeout=30) as (c, app): + row = await terminal(c, await submit(c)) + assert row['status'] == 'failed' and row['error'] == 'deadline_exceeded' + + +@pytest.mark.parametrize('value', [0, -1, True, '5', None]) +def test_queue_timeout_must_be_a_positive_number(tmp_path, value): + principals = {'mike': {'token': 'a' * 40, 'agents': ['forge'], 'services': []}} + with pytest.raises(ValueError): + Settings(database=tmp_path / 't.sqlite3', principals=principals, queue_timeout=value) + + +@pytest.mark.parametrize('env,expected', [(None, 600), ('12.5', 12.5)]) +def test_queue_timeout_environment(tmp_path, monkeypatch, env, expected): + path = tmp_path / 'principals.json' + path.write_text(json.dumps({'mike': {'token': 'a' * 40, 'agents': ['forge'], 'services': []}})) + path.chmod(0o600) + monkeypatch.setenv('FORGE_PRINCIPALS_FILE', str(path)) + monkeypatch.setenv('FORGE_DATABASE', str(tmp_path / 't.sqlite3')) + if env is None: + monkeypatch.delenv('FORGE_QUEUE_TIMEOUT', raising=False) + else: + monkeypatch.setenv('FORGE_QUEUE_TIMEOUT', env) + assert Settings.from_environment().queue_timeout == expected From ea890bd84c00a34574d19b8f0a3be8744e566716 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 05:33:20 +0000 Subject: [PATCH 2/4] fix(controller): one definition of a source line; refuse CR sources explicitly Line numbers were produced with str.splitlines(), which also splits on \x0b, \x0c, \x1c-\x1e, \x85 and U+2028/9. Any such character shifted every later line number away from git/editor numbering (8 of 8 separators reproduced), while evidence validation used the same wrong numbering, so a citation could pass validation yet point a reviewer at the wrong line. - Add tools.split_lines (LF only) and use it for reads, snapshot reads, result-evidence bounds and the evaluator. - CRLF sources can never satisfy exact multi-line replacement (the model is shown text the raw source does not contain). Preflight now rejects any source containing CR with the explicit reason 'unsupported_line_endings' instead of failing every edit later as 'replacement_not_exact'. - Tests compare snapshot numbering with 'git grep -n' as the oracle. --- docs/controller.md | 2 +- src/forge_controller/evaluate.py | 4 +- src/forge_controller/tools.py | 18 +++++- src/forge_controller/work_orders.py | 14 +++-- tests/test_line_model.py | 94 +++++++++++++++++++++++++++++ 5 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 tests/test_line_model.py diff --git a/docs/controller.md b/docs/controller.md index 5012e24..1ca3a5e 100644 --- a/docs/controller.md +++ b/docs/controller.md @@ -235,7 +235,7 @@ Set `FORGE_REPOSITORIES_FILE` to an operator-owned JSON manifest outside Git: Grant the intended principal `agents: ["forge", "repository_analyst"]` and `repositories: ["forge"]` in its private credential record. Submit an analyst task with `agent: "repository_analyst"` and `tools: ["list_repository_files", "read_repository_file"]`. The convenience CLI currently submits only the original `forge` tasks; use the HTTP API for analyst tasks. Configure an adequate tool budget (`FORGE_TOOL_ROUNDS=8` was used for the first experiment). -The analyst can list only manifest entries and read up to 80 numbered lines per call (at most 12 KiB returned). Approved files must be non-hidden `.py`, `.md`, or `.toml` sources. File reads are limited to 64 KiB, reject symlinks at every component, and reject non-regular/multiply-linked files and invalid UTF-8. Neither listing nor reading uses a command-execution tool. Selecting a source for the manifest is an operator responsibility: even source files may contain secrets and must be reviewed before granting access. Keep the manifest restricted to selected source files, not an entire home directory. +The analyst can list only manifest entries and read up to 80 numbered lines per call (at most 12 KiB returned). A line is LF-terminated, exactly as git and editors number it; other Unicode line separators (form feed, U+2028 and similar) do not start a new line. Work orders reject sources containing carriage returns (`unsupported_line_endings`) because exact-replacement admission cannot rely on them. Approved files must be non-hidden `.py`, `.md`, or `.toml` sources. File reads are limited to 64 KiB, reject symlinks at every component, and reject non-regular/multiply-linked files and invalid UTF-8. Neither listing nor reading uses a command-execution tool. Selecting a source for the manifest is an operator responsibility: even source files may contain secrets and must be reviewed before granting access. Keep the manifest restricted to selected source files, not an entire home directory. Every read rechecks principal repository permission, requested task capability, path membership, argument shape, and limits. Approved file/range metadata is audited without source content. A task cannot finish as an analyst without at least one successful source read (`sources_not_read`); this does not prove its conclusions are correct. Source content remains untrusted. Reports require senior review; see the [first experiment](experiments/repository-analyst-2026-10-03.md). diff --git a/src/forge_controller/evaluate.py b/src/forge_controller/evaluate.py index b664e0a..dbbcfc6 100644 --- a/src/forge_controller/evaluate.py +++ b/src/forge_controller/evaluate.py @@ -32,7 +32,7 @@ def assess(case, task, settings, owner): try: source = execute(settings, owner, case['agent'], 'read_repository_file', { 'repository': 'forge', 'path': 'src/forge_controller/config.py', 'start_line': 1, 'end_line': 80}) - lines = {int(line.split(': ', 1)[0]): line.split(': ', 1)[1] for line in source['content'].splitlines()} + lines = {int(line.split(': ', 1)[0]): line.split(': ', 1)[1] for line in source['content'].split('\n')} cited = [int(n) for n in re.findall(r'src/forge_controller/config\.py:(\d+)', result)] checks['cited_defaults_match_source'] = bool(cited) and all(n in lines for n in cited) and all( any(field in lines[n] and value in lines[n] for n in cited) for field, value in [('context_length','4096'),('max_output_tokens','512')]) @@ -57,7 +57,7 @@ def assess(case, task, settings, owner): 'repository': 'forge', 'path': patch['path'], 'start_line': 1, 'end_line': 80}) if source['total_lines'] > 80: raise ValueError('Fixture exceeds checker scope') - text = '\n'.join(line.split(': ', 1)[1] for line in source['content'].splitlines()) + '\n' + text = '\n'.join(line.split(': ', 1)[1] for line in source['content'].split('\n')) + '\n' if text.count(edit['before']) != 1 or text.count('temperature=0') != 1: raise ValueError('Patch does not match exact fixture') candidate = text.replace(edit['before'], edit['after'], 1) diff --git a/src/forge_controller/tools.py b/src/forge_controller/tools.py index 638fa5f..6180791 100644 --- a/src/forge_controller/tools.py +++ b/src/forge_controller/tools.py @@ -9,6 +9,20 @@ class ToolDenied(Exception): pass +def split_lines(text): + """Split on LF only, as git, editors and reviewers number lines. + + str.splitlines() also breaks on \\x0b, \\x0c, \\x1c-\\x1e, \\x85 and U+2028/9, + which shifts every later line number away from the file a reviewer opens. + A final newline terminates the last line rather than starting another one; + any CR is preserved so the model sees what is actually in the source. + """ + lines = text.split('\n') + if lines and lines[-1] == '': + lines.pop() + return lines + + def schema(name, description, properties): return {'type': 'function', 'function': {'name': name, 'description': description, 'parameters': {'type': 'object', 'properties': properties, @@ -74,7 +88,7 @@ def execute(settings, owner, agent, name, args): data = stream.read(65537) if len(data) > 65536: raise ToolDenied() - lines = data.decode('utf-8').splitlines() + lines = split_lines(data.decode('utf-8')) if start > len(lines): raise ToolDenied() text = '\n'.join(f'{n}: {line}' for n, line in enumerate(lines[start-1:end], start)) @@ -106,7 +120,7 @@ def execute_snapshot(settings, owner, agent, name, args, order, sources): if (not isinstance(path, str) or path not in sources or type(start) is not int or type(end) is not int or start < 1 or end < start or end-start >= 80): raise ToolDenied() - lines = sources[path].splitlines() + lines = split_lines(sources[path]) if start > len(lines): raise ToolDenied() text = '\n'.join(f'{n}: {line}' for n,line in enumerate(lines[start-1:end],start)) diff --git a/src/forge_controller/work_orders.py b/src/forge_controller/work_orders.py index 37c8d19..151f04a 100644 --- a/src/forge_controller/work_orders.py +++ b/src/forge_controller/work_orders.py @@ -9,7 +9,7 @@ from jsonschema import Draft202012Validator from forge_controller.config import Settings -from forge_controller.tools import execute, permitted, ToolDenied +from forge_controller.tools import execute, permitted, split_lines, ToolDenied ROOT = Path(__file__).resolve().parent @@ -91,9 +91,15 @@ def preflight(order, settings, owner): or current['source_sha256']!=file['sha256']): raise HandoffError('source_hash_mismatch') try: - sources[path] = pinned.decode('utf-8') + text = pinned.decode('utf-8') except UnicodeError: raise HandoffError('invalid_source_encoding') from None + # Exact-replacement admission compares the model's text with the raw source. + # CR cannot be shown or reproduced reliably, so refuse explicitly rather than + # let every multi-line edit fail later as an unexplained mismatch. + if '\r' in text: + raise HandoffError('unsupported_line_endings') + sources[path] = text return sources @@ -155,7 +161,7 @@ def validate_snapshot_result(order, result, execution, sources, owner): raise HandoffError('undeclared_source_read') start, end = entry.get('start_line'), entry.get('end_line') if (type(start) is not int or type(end) is not int or start<1 or endlen(sources[entry['path']].splitlines())): + or end>len(split_lines(sources[entry['path']]))): raise HandoffError('invalid_read_audit') ranges.setdefault(entry['path'], []).append((start, end)) reads += 1 @@ -169,7 +175,7 @@ def validate_snapshot_result(order, result, execution, sources, owner): raise HandoffError('invalid_blocked_result') for item in result['evidence']: if (item['path'] not in sources or item['start_line']>item['end_line'] - or item['end_line']>len(sources[item['path']].splitlines()) + or item['end_line']>len(split_lines(sources[item['path']])) or not any(start<=item['start_line'] and item['end_line']<=end for start,end in ranges.get(item['path'], []))): raise HandoffError('invalid_evidence_scope') diff --git a/tests/test_line_model.py b/tests/test_line_model.py new file mode 100644 index 0000000..6c0442b --- /dev/null +++ b/tests/test_line_model.py @@ -0,0 +1,94 @@ +"""A 'line' is LF-terminated, exactly as git, editors and reviewers count it.""" +import hashlib +import json +import subprocess +from types import SimpleNamespace + +import pytest + +from forge_controller.tools import execute_snapshot, split_lines +from forge_controller.work_orders import HandoffError, preflight, validate_snapshot_result +from tests.test_work_orders import assignment # noqa: F401 (fixture) + +# Characters str.splitlines() treats as line breaks but git and editors do not. +SPLITLINES_ONLY = ['\x0b', '\x0c', '\x1c', '\x1d', '\x1e', '\x85', '\u2028', '\u2029'] + + +@pytest.mark.parametrize('text,expected', [ + ('', []), ('a', ['a']), ('a\n', ['a']), ('a\n\n', ['a', '']), ('\n', ['']), + ('a\nb', ['a', 'b']), ('a\r\nb\r\n', ['a\r', 'b\r']), +]) +def test_split_lines_basic_shapes(text, expected): + assert split_lines(text) == expected + + +@pytest.mark.parametrize('char', SPLITLINES_ONLY) +def test_split_lines_ignores_non_lf_separators(char): + text = f'one{char}still one\ntwo\n' + assert split_lines(text) == [f'one{char}still one', 'two'] + assert len(text.splitlines()) != len(split_lines(text)) # the bug this guards against + + +def settings(): + return SimpleNamespace(principals={'mike': {'agents': ['implementer'], 'repositories': ['r'], 'services': []}}) + + +def read(source, start, end): + order = {'tools': ['read_repository_file'], 'source': {'repository_id': 'r'}} + return execute_snapshot(settings(), 'mike', 'implementer', 'read_repository_file', + {'repository': 'r', 'path': 'a.py', 'start_line': start, 'end_line': end}, + order, {'a.py': source}) + + +@pytest.mark.parametrize('char', SPLITLINES_ONLY) +def test_snapshot_line_numbers_match_git(tmp_path, char): + source = f'first\n# page{char}break\nMARKER\nlast\n' + subprocess.run(['git', 'init', '-q', str(tmp_path)], check=True) + (tmp_path / 'a.py').write_bytes(source.encode()) + subprocess.run(['git', '-C', str(tmp_path), 'add', 'a.py'], check=True) + git_line = int(subprocess.check_output( + ['git', '-C', str(tmp_path), 'grep', '-n', '--cached', 'MARKER'], text=True).split(':')[1]) + result = read(source, 1, 80) + shown = {int(row.split(': ', 1)[0]) for row in result['content'].split('\n') if row.endswith('MARKER')} + assert shown == {git_line} == {3} + assert result['total_lines'] == 4 + + +def test_result_evidence_cannot_cite_a_line_past_the_real_end(assignment): # noqa: F811 + _, _, order, result, execution = assignment + source = 'a\n\x0c\nc\nd\n' # 4 real lines; splitlines() would say 5 + sources = {'source.py': source} + execution['audit'] = [{'tool': 'read_repository_file', 'status': 'allowed', 'repository': 'forge', + 'path': 'source.py', 'start_line': 1, 'end_line': 4}] + result['evidence'][0].update(start_line=4, end_line=4) + execution['result'] = json.dumps(result) + assert validate_snapshot_result(order, result, execution, sources, 'operator')['source_valid'] + result['evidence'][0].update(start_line=4, end_line=5) + execution['result'] = json.dumps(result) + with pytest.raises(HandoffError, match='invalid_evidence_scope'): + validate_snapshot_result(order, result, execution, sources, 'operator') + + +def commit(root, data): + (root / 'source.py').write_bytes(data) + subprocess.run(['git', '-C', str(root), 'add', '.'], check=True) + subprocess.run(['git', '-C', str(root), '-c', 'user.name=T', '-c', 'user.email=t@example.invalid', + 'commit', '-qm', 'x'], check=True) + return subprocess.check_output(['git', '-C', str(root), 'rev-parse', 'HEAD'], text=True).strip() + + +def test_crlf_source_is_refused_with_an_explicit_reason(assignment): # noqa: F811 + root, config, order, _, _ = assignment + data = b'def f():\r\n return 1\r\n' + order['source']['revision'] = commit(root, data) + order['source']['files'][0]['sha256'] = hashlib.sha256(data).hexdigest() + with pytest.raises(HandoffError, match='unsupported_line_endings'): + preflight(order, config, 'operator') + + +def test_lf_source_still_passes_preflight(assignment): # noqa: F811 + root, config, order, _, _ = assignment + data = b'def f():\n return 1\n' + order['source']['revision'] = commit(root, data) + order['source']['files'][0]['sha256'] = hashlib.sha256(data).hexdigest() + assert preflight(order, config, 'operator') == {'source.py': data.decode()} From 29d56ff7be029353908846f3068feeb6c1070181 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 05:34:27 +0000 Subject: [PATCH 3/4] feat(controller): report why a work order was rejected Every preflight failure (about a dozen distinct HandoffError codes such as stale_revision, runtime_limits_mismatch, source_hash_mismatch, scope_denied) collapsed to a bare 422 'invalid_work_order', and the MCP bridge collapsed it again to 'invalid_request', so callers could not tell what to fix (#76, docs/v2 MCP-07). - Controller: 422 body now carries {'detail','reason'} when the failure is a HandoffError whose code matches ^[a-z0-9_-]{1,64}$. KeyError/ValueError and malformed codes still return the bare 'invalid_work_order'. - Bridge: relays 'invalid_request: ' only for a bounded, well-formed reason on a 422 invalid_work_order; never relays other upstream text. The existing 403 source_reads_required behaviour is unchanged. - Docs list the closed reason vocabulary. --- docs/work-orders.md | 13 ++++ src/forge_controller/app.py | 11 ++- src/forge_controller/mcp_bridge.py | 21 ++++-- tests/test_admission_reasons.py | 105 +++++++++++++++++++++++++++++ 4 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 tests/test_admission_reasons.py diff --git a/docs/work-orders.md b/docs/work-orders.md index 6bd0678..5ffec73 100644 --- a/docs/work-orders.md +++ b/docs/work-orders.md @@ -193,6 +193,19 @@ file is bounded to 64 KiB, at most 20 files per order; normal admission capacity still applies. The existing 32 KiB request and 8,000-character compiled prompt limits remain in force. Oversized or invalid orders fail rather than truncate. +A rejected order returns `422` with `{"detail": "invalid_work_order", "reason": CODE}` +and never queues inference. `CODE` comes from a closed vocabulary, so a caller can +act on it: `invalid_work-order` (does not match the published schema), +`principal_denied`, `scope_denied` (repository or tool not granted), +`runtime_limits_mismatch`, `source_revision_unavailable`, `stale_revision` +(order revision is not the checkout's current HEAD), `duplicate_source_path`, +`source_denied`, `source_hash_mismatch`, `invalid_source_encoding`, +`unsupported_line_endings` (source contains a carriage return) and +`compiled_prompt_too_large`. Only codes of that exact shape are echoed; any other +internal failure returns the bare `invalid_work_order`. The MCP bridge relays the +code as `invalid_request: CODE` and never relays other upstream text. Duplicate IDs +still return `409`. + Native workers list only selected files and read only captured UTF-8 source from the pinned commit. Subsequent checkout changes cannot alter their reads. Reviewed examples are omitted from native tasks to avoid unrelated source context. Existing diff --git a/src/forge_controller/app.py b/src/forge_controller/app.py index 9b24b2f..a7b8bd8 100644 --- a/src/forge_controller/app.py +++ b/src/forge_controller/app.py @@ -2,6 +2,7 @@ import hmac import json import logging +import re import time from datetime import datetime, timezone from contextlib import asynccontextmanager @@ -29,6 +30,9 @@ def format(self, record): log = logging.getLogger('forge_controller') +# HandoffError messages are a closed vocabulary of snake_case codes written by this +# project. Only a value of that exact shape is echoed; anything else stays generic. +REASON = re.compile(r'[a-z0-9_-]{1,64}') @@ -289,7 +293,12 @@ async def submit_order(order: dict, owner=Depends(principal)): # Capture before queueing; fixed Git reads are never worker tools. sources = await asyncio.to_thread(preflight, order, app.state.settings, owner) arguments = task_arguments(order) - except (HandoffError, ValueError, KeyError): + except HandoffError as exc: + body = {'detail': 'invalid_work_order'} + if REASON.fullmatch(str(exc)): + body['reason'] = str(exc) + return JSONResponse(body, status_code=422) + except (ValueError, KeyError): raise HTTPException(422, 'invalid_work_order') from None if app.state.store.has_order(owner, order['work_order_id']): raise HTTPException(409, 'work_order_already_submitted') diff --git a/src/forge_controller/mcp_bridge.py b/src/forge_controller/mcp_bridge.py index 872d98f..2077ce4 100644 --- a/src/forge_controller/mcp_bridge.py +++ b/src/forge_controller/mcp_bridge.py @@ -112,20 +112,27 @@ async def request(method, path, payload=None): try: async with client.stream(method, path, json=payload) as response: if response.status_code >= 300: - if response.status_code == 403: - # Expose only this known validation reason, never arbitrary upstream text. + if response.status_code in {403, 422}: + # Expose only known, bounded validation reasons, never arbitrary upstream text. raw = bytearray() async for chunk in response.aiter_bytes(): raw.extend(chunk) if len(raw) > 1024: break + body = None if len(raw) <= 1024: try: - detail = json.loads(raw).get('detail') - except (ValueError, AttributeError): - detail = None - if detail == 'source_reads_required': - raise ToolError('source_reads_required: analyst/implementer tasks must explicitly include read_repository_file in tools') + body = json.loads(raw) + except ValueError: + body = None + if not isinstance(body, dict): + body = {} + if response.status_code == 403 and body.get('detail') == 'source_reads_required': + raise ToolError('source_reads_required: analyst/implementer tasks must explicitly include read_repository_file in tools') + reason = body.get('reason') + if (response.status_code == 422 and body.get('detail') == 'invalid_work_order' + and isinstance(reason, str) and re.fullmatch(r'[a-z0-9_-]{1,64}', reason)): + raise ToolError('invalid_request: ' + reason) error = {401:'unauthorized', 403:'permission_denied', 404:'task_not_found', 409:'task_conflict', 422:'invalid_request', 503:'controller_unavailable'}.get( response.status_code, 'controller_request_failed') diff --git a/tests/test_admission_reasons.py b/tests/test_admission_reasons.py new file mode 100644 index 0000000..ad3e924 --- /dev/null +++ b/tests/test_admission_reasons.py @@ -0,0 +1,105 @@ +"""Work-order admission failures say why (MCP-07) without echoing arbitrary text.""" +import hashlib + +import httpx +import pytest + +pytest.importorskip('mcp') +from mcp import Client + +import forge_controller.app as controller_app +from forge_controller.mcp_bridge import BridgeSettings, create_bridge +from forge_controller.work_orders import HandoffError +from tests.test_mcp_bridge import credential +from tests.test_native_work_orders import Worker, assignment, instance # noqa: F401 +from tests.test_line_model import commit +from tests.test_work_orders import assignment as source_assignment # noqa: F401 (fixture dependency) + + +def mutate(case, root, settings, order): + if case == 'unknown_field': + order['extra'] = True + elif case == 'hash': + order['source']['files'][0]['sha256'] = '0' * 64 + elif case == 'revision': + order['source']['revision'] = '0' * 40 + elif case == 'limits': + order['runtime_limits']['output_tokens'] += 1 + elif case == 'scope': + settings.principals['operator']['repositories'] = [] + elif case == 'tools': + order['tools'] = ['execute_commands'] + elif case == 'crlf': + data = b'a\r\nb\r\n' + order['source']['revision'] = commit(root, data) + order['source']['files'][0]['sha256'] = hashlib.sha256(data).hexdigest() + + +@pytest.mark.parametrize('case,reason', [ + ('unknown_field', 'invalid_work-order'), ('tools', 'invalid_work-order'), + ('hash', 'source_hash_mismatch'), ('revision', 'stale_revision'), + ('limits', 'runtime_limits_mismatch'), ('scope', 'scope_denied'), + ('crlf', 'unsupported_line_endings')]) +async def test_admission_failure_reports_its_reason(assignment, case, reason): # noqa: F811 + root, settings, order, result, _ = assignment + mutate(case, root, settings, order) + model = Worker(result) + async with instance(settings, model) as (client, app): + response = await client.post('/v1/work-orders', json=order) + assert response.status_code == 422 + assert response.json() == {'detail': 'invalid_work_order', 'reason': reason} + assert not model.started.is_set() + assert app.state.store.db.execute('SELECT count(*) FROM tasks').fetchone()[0] == 0 + + +@pytest.mark.parametrize('error', [KeyError('private-key'), ValueError('private detail'), + HandoffError('free text with spaces & private detail'), + HandoffError('x' * 65), HandoffError('UPPER_CASE')]) +async def test_only_well_formed_codes_are_echoed(assignment, monkeypatch, error): # noqa: F811 + _, settings, order, result, _ = assignment + + def boom(*args, **kwargs): + raise error + monkeypatch.setattr(controller_app, 'preflight', boom) + async with instance(settings, Worker(result)) as (client, app): + response = await client.post('/v1/work-orders', json=order) + assert response.status_code == 422 + assert response.json() == {'detail': 'invalid_work_order'} + assert 'private' not in response.text + + +def bridge_for(tmp_path, status, content): + transport = httpx.MockTransport(lambda request: httpx.Response(status, content=content)) + return Client(create_bridge(BridgeSettings(credential(tmp_path)), transport)) + + +async def submit(client): + return await client.call_tool('forge_submit_work_order', {'order': {'any': 'thing'}}) + + +async def test_bridge_passes_a_well_formed_reason_through(tmp_path): + body = b'{"detail":"invalid_work_order","reason":"stale_revision"}' + async with bridge_for(tmp_path, 422, body) as client: + result = await submit(client) + assert result.is_error and 'invalid_request: stale_revision' in str(result.content) + + +@pytest.mark.parametrize('body', [ + b'{"detail":"invalid_work_order","reason":"has spaces and private detail"}', + b'{"detail":"invalid_work_order","reason":["stale_revision"]}', + b'{"detail":"something_else","reason":"stale_revision"}', + b'{"detail":"invalid_work_order","reason":"' + b'x' * 2000 + b'"}', + b'not json', b'[1,2]', b'private detail']) +async def test_bridge_never_leaks_unbounded_upstream_text(tmp_path, body): + async with bridge_for(tmp_path, 422, body) as client: + result = await submit(client) + text = str(result.content) + assert result.is_error and 'invalid_request' in text + assert 'private' not in text and 'spaces' not in text and 'stale_revision' not in text + assert 'xxxx' not in text + + +async def test_bridge_403_source_reads_required_is_unchanged(tmp_path): + async with bridge_for(tmp_path, 403, b'{"detail":"source_reads_required"}') as client: + result = await client.call_tool('forge_submit_task', {'prompt': 'x', 'agent': 'implementer'}) + assert result.is_error and 'source_reads_required' in str(result.content) From 2711efa615747dda4b60b4f27f1dc34f594a40ae Mon Sep 17 00:00:00 2001 From: kudama <19508681+kudama@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:56:48 -0800 Subject: [PATCH 4/4] Expire queued tasks independently and require finite queue budgets --- docs/controller.md | 2 +- src/forge_controller/app.py | 35 ++++++++++++++ src/forge_controller/config.py | 6 ++- tests/test_queue_deadline.py | 88 +++++++++++++++++++++++++++++++++- 4 files changed, 127 insertions(+), 4 deletions(-) diff --git a/docs/controller.md b/docs/controller.md index 1ca3a5e..ab406b3 100644 --- a/docs/controller.md +++ b/docs/controller.md @@ -282,7 +282,7 @@ with context 4096 in the prototype comparison. | `FORGE_CONTEXT_LENGTH` | Positive integer; default 4096 context tokens | | `FORGE_MAX_OUTPUT_TOKENS` | Positive integer; default 512 output tokens | | `FORGE_TASK_TIMEOUT` | 60 seconds of execution, measured from dispatch (queue wait is not counted) | -| `FORGE_QUEUE_TIMEOUT` | 600 seconds a task may wait before dispatch; positive number | +| `FORGE_QUEUE_TIMEOUT` | 600 seconds a task may wait before dispatch; finite positive number. Expiry settles the task and frees capacity even while another task is running. | | `FORGE_CAPACITY` | 8 admitted tasks, including running work | | `FORGE_TOOL_ROUNDS` | Maximum 3 total tool calls | | `FORGE_CONTROLLER_URL` | Client URL, default `http://127.0.0.1:8787` | diff --git a/src/forge_controller/app.py b/src/forge_controller/app.py index a7b8bd8..f46937a 100644 --- a/src/forge_controller/app.py +++ b/src/forge_controller/app.py @@ -79,6 +79,7 @@ class Engine: def __init__(self, settings, store, model): self.settings, self.store, self.model = settings, store, model self.queue = asyncio.Queue(maxsize=settings.capacity) + self.queue_timers = {} self.active = {} self.capture_lock = asyncio.Lock() self.worker = None @@ -91,9 +92,37 @@ def submit(self, owner, task, binding=None): task_id = self.store.create(owner, task.agent, binding) self.active[task_id] = None self.queue.put_nowait((task_id, owner, task, time.monotonic(), binding)) + self.queue_timers[task_id] = asyncio.get_running_loop().call_later( + self.settings.queue_timeout, self.expire_queued, task_id) log.info('task_queued', extra={'task_id': task_id, 'agent': task.agent}) return task_id + def expire_queued(self, task_id): + self.queue_timers.pop(task_id, None) + if task_id not in self.active or self.active[task_id] is not None: + return + row = self.store.get(task_id) + if row['status'] == 'queued': + self.store.update(task_id, 'failed', error='queue_expired', audit=row['audit']) + # Remove the source packet as well as the admission slot. Rebuild through + # Queue's public API, preserving FIFO order and unfinished-task accounting. + # This synchronous callback cannot interleave with dispatch or submission. + pending = [] + removed = 0 + while not self.queue.empty(): + entry = self.queue.get_nowait() + if entry[0] != task_id: + pending.append(entry) + else: + removed += 1 + for entry in pending: + self.queue.put_nowait(entry) + self.queue.task_done() + for _ in range(removed): + self.queue.task_done() + self.active.pop(task_id) + log.info('task_finished', extra={'task_id': task_id, 'status': self.store.get(task_id)['status']}) + def cancel(self, task_id): if self.store.get(task_id)['status'] not in TERMINAL: self.store.update(task_id, 'cancelled', error='cancelled_by_user', audit=self.store.get(task_id)['audit']) @@ -168,6 +197,9 @@ async def run(self, task_id, owner, task, binding=None): async def work(self): while True: task_id, owner, task, queued_at, binding = await self.queue.get() + timer = self.queue_timers.pop(task_id, None) + if timer: + timer.cancel() try: if self.store.get(task_id)['status'] in TERMINAL: continue @@ -204,6 +236,9 @@ async def work(self): async def stop(self): self.closing = True + for timer in self.queue_timers.values(): + timer.cancel() + self.queue_timers.clear() if self.worker: self.worker.cancel() await asyncio.gather(self.worker, return_exceptions=True) diff --git a/src/forge_controller/config.py b/src/forge_controller/config.py index 083a43e..3fbc5c3 100644 --- a/src/forge_controller/config.py +++ b/src/forge_controller/config.py @@ -1,4 +1,5 @@ import json +import math import os import stat from dataclasses import dataclass, field @@ -38,8 +39,9 @@ def __post_init__(self): raise ValueError("Invalid model URL") if self.timeout <= 0 or self.capacity < 1 or self.tool_rounds < 1: raise ValueError("Limits must be positive") - if type(self.queue_timeout) not in {int, float} or self.queue_timeout <= 0: - raise ValueError("queue_timeout must be a positive number") + if (type(self.queue_timeout) not in {int, float} + or not math.isfinite(self.queue_timeout) or self.queue_timeout <= 0): + raise ValueError("queue_timeout must be a finite positive number") if not self.model or not self.principals: raise ValueError("Model and principals are required") tokens = [] diff --git a/tests/test_queue_deadline.py b/tests/test_queue_deadline.py index 25de68f..a9e8436 100644 --- a/tests/test_queue_deadline.py +++ b/tests/test_queue_deadline.py @@ -49,7 +49,82 @@ async def test_execution_deadline_still_enforced_after_dispatch(tmp_path): assert row['status'] == 'failed' and row['error'] == 'deadline_exceeded' -@pytest.mark.parametrize('value', [0, -1, True, '5', None]) +async def test_expiry_releases_capacity_while_inference_is_blocked(tmp_path): + fake = FakeModel('wait') + async with client(tmp_path, fake, timeout=5, queue_timeout=.05, capacity=2) as (c, app): + first = await submit(c) + await fake.started.wait() + # Repeat without releasing inference: expired queue entries must not + # accumulate and fill the physical queue after admission slots are freed. + for _ in range(4): + queued = await submit(c) + row = await terminal(c, queued) + assert row['error'] == 'queue_expired' and row['audit'] == [] + assert (await c.get('/v1/tasks/' + first)).json()['status'] == 'running' + assert fake.calls == 1 + assert (await c.delete('/v1/tasks/' + queued)).status_code == 204 + last = await submit(c) + fake.release.set() + assert (await terminal(c, first))['status'] == 'succeeded' + assert (await terminal(c, last))['status'] == 'succeeded' + assert fake.calls == 2 + await asyncio.wait_for(app.state.engine.queue.join(), timeout=.5) + + +async def test_expiry_preserves_other_tasks_fifo_order(tmp_path): + class RecordingModel(FakeModel): + def __init__(self): + super().__init__('wait') + self.prompts = [] + + async def chat(self, messages, tools, *, agent='forge'): + self.prompts.append(messages[-1]['content']) + return await super().chat(messages, tools, agent=agent) + + fake = RecordingModel() + async with client(tmp_path, fake, timeout=5, queue_timeout=.3) as (c, app): + first = await submit(c, prompt='first') + await fake.started.wait() + expired = await submit(c, prompt='expired') + await asyncio.sleep(.2) + third = await submit(c, prompt='third') + fourth = await submit(c, prompt='fourth') + assert (await terminal(c, expired))['error'] == 'queue_expired' + fake.release.set() + for task_id in [first, third, fourth]: + assert (await terminal(c, task_id))['status'] == 'succeeded' + assert fake.prompts == ['first', 'third', 'fourth'] + await asyncio.wait_for(app.state.engine.queue.join(), timeout=.5) + + +async def test_dispatched_task_does_not_expire_at_its_old_queue_deadline(tmp_path): + fake = FakeModel('wait') + async with client(tmp_path, fake, timeout=5, queue_timeout=.05) as (c, app): + task_id = await submit(c) + await fake.started.wait() + await asyncio.sleep(.1) + assert (await c.get('/v1/tasks/' + task_id)).json()['status'] == 'running' + fake.release.set() + assert (await terminal(c, task_id))['status'] == 'succeeded' + + +async def test_expiry_timer_preserves_cancellation_and_stops_on_shutdown(tmp_path): + fake = FakeModel('wait') + async with client(tmp_path, fake, timeout=5, queue_timeout=.05) as (c, app): + await submit(c) + await fake.started.wait() + queued = await submit(c) + await c.post('/v1/tasks/' + queued + '/cancel') + await asyncio.sleep(.1) + row = (await c.get('/v1/tasks/' + queued)).json() + assert row['status'] == 'cancelled' and row['error'] == 'cancelled_by_user' + pending = await submit(c) + # Timer callbacks must not access the closed store or overwrite interrupted. + await asyncio.sleep(.1) + assert app.state.engine.queue_timers == {} + + +@pytest.mark.parametrize('value', [0, -1, True, '5', None, float('nan'), float('inf'), -float('inf')]) def test_queue_timeout_must_be_a_positive_number(tmp_path, value): principals = {'mike': {'token': 'a' * 40, 'agents': ['forge'], 'services': []}} with pytest.raises(ValueError): @@ -68,3 +143,14 @@ def test_queue_timeout_environment(tmp_path, monkeypatch, env, expected): else: monkeypatch.setenv('FORGE_QUEUE_TIMEOUT', env) assert Settings.from_environment().queue_timeout == expected + + +@pytest.mark.parametrize('value', ['nan', 'inf', '-inf']) +def test_nonfinite_environment_queue_timeout_is_rejected(tmp_path, monkeypatch, value): + path = tmp_path / 'principals.json' + path.write_text(json.dumps({'mike': {'token': 'a' * 40, 'agents': ['forge'], 'services': []}})) + path.chmod(0o600) + monkeypatch.setenv('FORGE_PRINCIPALS_FILE', str(path)) + monkeypatch.setenv('FORGE_QUEUE_TIMEOUT', value) + with pytest.raises(ValueError, match='queue_timeout'): + Settings.from_environment()