diff --git a/.dockerignore b/.dockerignore index d3740d9..0867290 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,7 +5,14 @@ **/__pycache__ **/*.pyc .pytest_cache +.ruff_cache +coverage.json .venv +.env +.env.* +.qa-artifacts/ +testconfigkey.txt +backend/config/config.yaml tools/ data/ logs/ @@ -15,4 +22,4 @@ projects/ memory/ exports/ ipc_check.js -log.txt \ No newline at end of file +log.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad93ff3..394c878 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,10 +11,14 @@ on: permissions: contents: read +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: test: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 45 services: postgres: image: postgres:16-alpine @@ -46,16 +50,78 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -e ".[dev,docker]" + python -m pip install -e ".[dev,docker,browser]" + + - name: Set up Node for frontend behavior tests + uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Install explicit integration dependencies + run: | + sudo apt-get update + sudo apt-get install -y postgresql + + - name: Resolve Playwright cache version + id: playwright + run: python -c "import importlib.metadata; print('version='+importlib.metadata.version('playwright'))" >> "$GITHUB_OUTPUT" + + - name: Cache Playwright browsers + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }} + + - name: Install Playwright browser and system dependencies + run: | + python -m playwright install --with-deps chromium - - name: Run tests - run: python -m pytest -q + - name: Run tests with branch coverage + run: python -m pytest -q --require-postgres --require-browser --cov=backend --cov-branch --cov-report=term-missing --cov-report=json --junitxml=data/exports/pytest-ci.xml - - name: Static and deployment checks + - name: Enforce architectural coverage floors + run: python scripts/check_coverage.py + + - name: Verify temporary PostgreSQL isolation + run: | + native_postgres_ctl="$(pg_config --bindir)/pg_ctl" + test -x "$native_postgres_ctl" + env -u IPC_TEST_DATABASE_URL -u IPC_DATABASE_URL IPC_TEST_POSTGRES_EXECUTABLE="$native_postgres_ctl" \ + python -m pytest -q --require-postgres tests/test_postgres_isolation.py tests/test_goal_intent_postgres.py tests/test_ops_stream.py tests/test_competition_monitor.py tests/test_competition_migration_smoke.py + + - name: Full lint and deployment checks run: | + python -m ruff check backend scripts tests python -m compileall -q backend scripts tests python -m alembic upgrade head --sql > /tmp/ipc-migration.sql docker compose config --quiet + docker compose -f docker-compose.e2e.yml config --quiet + node tests/js/ops_chat_checks.mjs . + node tests/js/pentest_checks.mjs . + + - name: Upload test and coverage diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: ipc-tests-coverage + path: | + coverage.json + data/exports/pytest-ci.xml + data/exports/pentest + + package-smoke: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + cache: pip + - name: Install package build dependencies + run: python -m pip install -e ".[docker]" build + - name: Verify installed platform workflows + run: python scripts/check_workflow_package.py task-image: runs-on: ubuntu-latest @@ -64,8 +130,18 @@ jobs: - name: Checkout uses: actions/checkout@v5 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build task image - run: docker build -f docker/member/Dockerfile -t ipc-task:ci . + uses: docker/build-push-action@v6 + with: + context: . + file: docker/member/Dockerfile + tags: ipc-task:ci + load: true + cache-from: type=gha,scope=task-image + cache-to: type=gha,scope=task-image,mode=max - name: Run CTF tool acceptance run: docker run --rm -v "$PWD/scripts:/acceptance:ro" ipc-task:ci bash /acceptance/c5_task_acceptance.sh @@ -82,3 +158,51 @@ jobs: - name: Run AI CPU smoke run: docker run --rm -v "$PWD/scripts:/acceptance:ro" ipc-task:ci python3 /acceptance/ai_smoke.py + + e2e: + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + IPC_INSTALL_WSRX: "0" + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Start deterministic E2E stack + run: | + docker compose build ipc-app + docker compose -f docker-compose.e2e.yml up -d e2e-postgres e2e-platform e2e-app + + - name: Run Playwright competition acceptance + run: docker compose -f docker-compose.e2e.yml run --rm e2e-runner + + - name: Collect E2E stack logs + if: always() + run: | + mkdir -p data/exports/e2e + sudo chown -R "$(id -u):$(id -g)" data/exports/e2e + docker compose -f docker-compose.e2e.yml ps -a > data/exports/e2e/compose-status.txt + docker compose -f docker-compose.e2e.yml logs --no-color > data/exports/e2e/compose.log + + - name: Upload E2E diagnostics + # All validation jobs must be required by the repository's branch rules; + # declaring a job here alone does not configure branch protection. + if: always() + uses: actions/upload-artifact@v4 + with: + name: ipc-e2e-diagnostics + path: data/exports/e2e + + - name: Stop E2E stack + if: always() + run: docker compose -f docker-compose.e2e.yml down -v + + required-checks: + if: always() + needs: [test, package-smoke, task-image, e2e] + runs-on: ubuntu-latest + steps: + - name: Require every validation job to succeed + env: + JOB_RESULTS: ${{ toJSON(needs) }} + run: python -c "import json,os; results=json.loads(os.environ['JOB_RESULTS']); assert all(job['result']=='success' for job in results.values()), results" diff --git a/.gitignore b/.gitignore index 8c11a30..557d37c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ backend/config/config.yaml logs/ data/ .ipc/ -memory/ +/memory/ projects/ Wp/ docs/ @@ -64,6 +64,7 @@ htmlcov/ .cache nosetests.xml coverage.xml +coverage.json *.cover *.py.cover .hypothesis/ @@ -221,7 +222,7 @@ cython_debug/ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore # and can be added to the global gitignore or merged into this file. However, if you prefer, # you could uncomment the following to ignore the entire vscode folder -# .vscode/ +.vscode/ # Temporary file for partial code execution tempCodeRunnerFile.py diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 418a485..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "java.compile.nullAnalysis.mode": "automatic", - "python-envs.defaultEnvManager": "ms-python.python:system" -} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 4fe6f2d..5beb3c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,9 @@ FROM python:3.12-slim WORKDIR /app +# Debian's docker.io package provides the daemon; the app needs the docker.io +# client command, supplied by the split docker-cli package below. + RUN sed -i 's|http://deb.debian.org/debian|https://mirrors.tuna.tsinghua.edu.cn/debian|g; s|http://deb.debian.org/debian-security|https://mirrors.tuna.tsinghua.edu.cn/debian-security|g' /etc/apt/sources.list.d/debian.sources \ && printf '%s\n' \ 'Acquire::Retries "5";' \ @@ -12,7 +15,8 @@ RUN sed -i 's|http://deb.debian.org/debian|https://mirrors.tuna.tsinghua.edu.cn/ 'Acquire::BrokenProxy "true";' \ > /etc/apt/apt.conf.d/80-ipc-retries \ && apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates docker.io \ + ca-certificates docker-cli \ + && command -v docker \ && rm -rf /var/lib/apt/lists/* COPY pyproject.toml /app/pyproject.toml @@ -33,11 +37,17 @@ RUN python -c "from pathlib import Path; from backend.skills.store import SkillS # subprocesses are owned by the backend process (Members just connect to # ipc-app:). ARG WSRX_VERSION=0.6.1 -RUN python -c "import urllib.request; urllib.request.urlretrieve('https://github.com/XDSEC/WebSocketReflectorX/releases/download/${WSRX_VERSION}/wsrx-cli-${WSRX_VERSION}-linux-musl-x86_64.tar.gz', '/tmp/wsrx.tar.gz')" \ - && tar -xzf /tmp/wsrx.tar.gz -C /tmp \ - && install -m 0755 /tmp/wsrx /usr/local/bin/wsrx \ - && rm -f /tmp/wsrx /tmp/wsrx.tar.gz \ - && wsrx --version +ARG INSTALL_WSRX=1 +RUN if [ "$INSTALL_WSRX" = "1" ]; then \ + python -c "import urllib.request; urllib.request.urlretrieve('https://github.com/XDSEC/WebSocketReflectorX/releases/download/${WSRX_VERSION}/wsrx-cli-${WSRX_VERSION}-linux-musl-x86_64.tar.gz', '/tmp/wsrx.tar.gz')" \ + && tar -xzf /tmp/wsrx.tar.gz -C /tmp \ + && install -m 0755 /tmp/wsrx /usr/local/bin/wsrx \ + && rm -f /tmp/wsrx /tmp/wsrx.tar.gz \ + && wsrx --version; \ + else \ + printf '#!/bin/sh\necho wsrx disabled in this image\nexit 127\n' > /usr/local/bin/wsrx \ + && chmod +x /usr/local/bin/wsrx; \ + fi # /app/data is bind-mounted by docker-compose.yml. PostgreSQL owns runtime # facts; filesystem workspaces and generated files share one Artifact tree. diff --git a/README.md b/README.md index b612f31..0493003 100644 --- a/README.md +++ b/README.md @@ -8,18 +8,11 @@
- [![License: AGPL v3](https://img.shields.io/badge/License-AGPL_v3-blue.svg)](LICENSE) [![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-3776AB.svg?logo=python&logoColor=white)](https://www.python.org/) [![FastAPI](https://img.shields.io/badge/API-FastAPI-009688.svg?logo=fastapi&logoColor=white)](https://fastapi.tiangolo.com/) [![PostgreSQL 16](https://img.shields.io/badge/Runtime-PostgreSQL_16-4169E1.svg?logo=postgresql&logoColor=white)](#durable-runtime-state) [![Docker Compose](https://img.shields.io/badge/Deploy-Docker_Compose-2496ED.svg?logo=docker&logoColor=white)](#quick-start) - -
- -
-[🚀 快速开始](#quick-start) • [✨ 核心设计](#core-innovations) • [🖥️ 控制台](#agent-workbench) • [🏗️ 系统架构](#system-architecture) • [🧰 工具运行时](#tool-runtime) • [🧪 开发](#development) -
## Current Reward @@ -120,6 +113,12 @@ PostgreSQL 只保存运行时事实、协调状态和可查询元数据;文件 - 每个题目最多分配两个解题席位;WP 任务使用原作者 Member/session,远端实例与本地 sandbox 分开计量。 - `POST /api/workflows/{id}/preflight`、`POST /api/workflows/{id}/start` 和 `/api/runs/{id}/...` 提供平台预检、启动、同步、暂停、恢复、停止与事件查询。 +#### Built-in GZCTF workflow + +GZCTF is available as the built-in default platform workflow. Use **Competition → New GZCTF** or `POST /api/ops/workflows/gzctf` and provide the platform origin, game ID, and optional competition name, team ID, deadline, container concurrency, and attachment CDN origin. Choose either a participant token or username/password; credentials are stored in the workflow secret namespace and are never copied into the run snapshot. + +The same preset is available to the IPC MCP tool `ipc_gzctf_workflow`. After saving the draft, run the normal confirmation and preflight flow. Future GZCTF runs reuse the native challenge discovery, attachments, submissions, verdict polling, and container lifecycle adapter; no per-event HTTP field mapping is required. Other platforms and custom HTTP JSON workflows remain available through the generic workflow endpoint. + ### MCP Member 在题目容器中使用按分类注册的 CTF 工具,并可通过 MCP 访问: @@ -151,7 +150,6 @@ Docker Compose 将数据库放入命名卷,将需要人工访问或跨容器 | `data/artifacts/logs/` | 项目、LLM、工具与 Memory JSONL | | `data/artifacts/exports/` | Writeup、日志和 Obsidian/Markdown Memory 导出快照 | | `data/ops-agent/` | IPC 模型配置和工作流密钥等文件型机密 | -| `ipc_claude_home` Docker 卷 | `claudecode` 运行器的原生 JSONL 会话 | 导出采用无损编号命名;同名文件存在时会创建 `名称01`、`名称02` 等新文件,不覆盖历史导出。 @@ -165,7 +163,7 @@ Docker Compose 将数据库放入命名卷,将需要人工访问或跨容器 | Docker Compose v2 | 必需 | 标准部署与服务编排 | | Docker Socket | 必需 | App 需要创建题目容器;IPC Runner 同样依赖 Socket | | Linux Docker 主机 | 推荐/已验证 | Compose 会挂载 Docker Socket 与 Compose 插件 | -| LLM 端点 | 开始解题时需要 | Diamond 与已启用 Member 使用 OpenAI、Anthropic、Claude Code、DeepSeek、Pi 或 Mock 适配器 | +| LLM 端点 | 开始解题时需要 | Diamond 与已启用 Member 使用 OpenAI、Anthropic、DeepSeek、Pi 或 Mock 适配器 | | 浏览器 | 可选 | 只保留 Playwright Python 包;浏览器二进制不在任务镜像内,由部署环境显式提供路径 | > [!WARNING] @@ -199,7 +197,7 @@ docker compose logs -f ipc-app docker compose down ``` -此命令不会删除 `./data`、`ipc_postgres_data` 或 `ipc_claude_home`。只有主动删除它们或执行 `docker compose down -v` 才会清理命名卷。 +此命令不会删除 `./data` 或 `ipc_postgres_data`。只有主动删除它们或执行 `docker compose down -v` 才会清理命名卷。 ### 2. Configure the LLM runtime @@ -258,7 +256,7 @@ UI 运行在操作者可信的 Docker 网络边界内,默认不增加浏览器 IPC 是面向操作者的持续对话代理,可诊断运行环境、辅助题目沙箱、操作项目生命周期,并生成声明式平台工作流。工作流必须经人工确认才能执行。其对话、运行事件和工作流元数据存入 PostgreSQL,文件型密钥保存在 `data/ops-agent/`。 -`claudecode` 运行器会持久化原生会话,以便在容器替换后通过 `--resume` 延续上下文。无论使用哪种 Runner,内部 MCP 请求都应通过 `IPC_RUNNER_TOKEN` 鉴权。 +IPC 与比赛 Member 共用 `backend/agent/` 中的统一运行时:会话记录在 `agent_sessions` / `agent_events`,因此容器替换后可以从持久事件重建上下文,无需外部 Runner。 --- @@ -275,7 +273,7 @@ flowchart TB subgraph OPS["IPC 行动代理层"] IPC["IPC 对话代理"] - RUNNER["Claude Code Runner\n或 OpenAI-compatible API"] + RUNNER["统一 Agent 运行时\nagent_sessions / agent_events"] IMCP["内部 IPC MCP\n项目生命周期与诊断工具"] IPC <--> RUNNER RUNNER --> IMCP @@ -450,6 +448,16 @@ docker run --rm \ --- +## References + +Pentest 工作区、渗透图谱、证据与 Agent 工具说明见 [Pentest 使用文档](docs/pentest.md)。 +GZCTF 与 TSecBench 固定工作流的个性化配置见 [平台文档](backend/platform/README.md)。 +普通题目、独立渗透目标和平台多 Flag 题目分别遵循各自的完成条件,子任务继续使用本项目 Agent Harness。 + +[Cairn](https://github.com/oritera/Cairn) + +[ctf-agent](https://github.com/verialabs/ctf-agent) + ## 👥 Contributors

@@ -463,16 +471,6 @@ docker run --rm \ springbot

- - -## 🤝 Contribution - -欢迎提交 bug 报告、稳定性测试、工具集成、文档和架构改进。 - -1. 先在 [Issues](https://github.com/PureStream108/IPC_CTFAgent/issues) 描述问题或设计提案。 -2. 创建聚焦的分支,并为修改的运行时边界补充测试。 -3. 在 Pull Request 中说明行为变化、迁移影响和验证结果。 - --- ## 📝 License diff --git a/backend/agent/__init__.py b/backend/agent/__init__.py index 93af161..e69de29 100644 --- a/backend/agent/__init__.py +++ b/backend/agent/__init__.py @@ -1 +0,0 @@ -"""Unified agent runtime: one durable action loop for Ops, competition, projects.""" diff --git a/backend/agent/compaction.py b/backend/agent/compaction.py index 7367d33..2a35ded 100644 --- a/backend/agent/compaction.py +++ b/backend/agent/compaction.py @@ -1,20 +1,11 @@ -"""Semantic compaction: a structured summary, not a truncated transcript. - -Dropping the oldest messages loses exactly what a long investigation needs to -keep: which facts were verified, which approaches already failed, and where the -interesting bytes live. That is why the summary has an explicit schema with a -``do_not_retry`` marker on failed paths - the point is to stop the model from -re-running work it has already ruled out. - -Compaction is an optimisation, never a precondition for running. If the -summarizer fails, the caller falls back to :func:`deterministic_summary`, which -needs no model at all. -""" from __future__ import annotations import json +import logging from typing import Any +_recovery_logger = logging.getLogger(__name__) + SUMMARY_SCHEMA_HINT = { "verified_facts": [{"claim": "...", "evidence": "file:offset | command | artifact_id"}], "failed_paths": [{"approach": "...", "why_failed": "...", "do_not_retry": True}], @@ -96,8 +87,8 @@ def deterministic_summary( summary = empty_summary() if dropped: summary["open_questions"] = [ - "The earlier transcript was compacted without a semantic summary; " - "re-derive any detail you need instead of assuming it." + ("The earlier transcript was compacted without a semantic summary; " + "re-derive any detail you need instead of assuming it.") ] if signatures: seen: list[str] = [] @@ -110,9 +101,68 @@ def deterministic_summary( "evidence": "compacted transcript", } ] + for message in dropped: + if not isinstance(message, dict): + continue + content = message.get("content") + if not isinstance(content, str): + continue + try: + value = json.loads(content.split("\n", 1)[1]) if content.startswith("[compacted context summary]") else json.loads(content) + except (ValueError, IndexError): + continue + if isinstance(value, dict): + for key in _SUMMARY_KEYS: + if isinstance(value.get(key), list): + summary[key].extend(value[key]) + artifacts = (value.get("process") or {}).get("artifacts", {}) + if isinstance(artifacts, dict): + summary["artifacts"].extend(item for item in artifacts.values() if isinstance(item, dict)) return summary, [*head, *tail] +def fit_history(messages: list[dict[str, Any]], *, max_bytes: int, max_tokens: int = 0, + calibration: float = 1.0) -> list[dict[str, Any]]: + """Bound a projection while removing orphaned results with their old batch.""" + from backend.agent.context import estimate_tokens + from backend.sandbox.output import preview_text + + limit = max(1024, min(16384, max_bytes // 4)) + bounded = [] + for original in messages: + message = dict(original) + for key in ("content", "output"): + content = message.get(key) + if isinstance(content, str): + message[key] = preview_text(content, limit) + elif isinstance(content, list): + blocks = [] + for original_block in content: + if not isinstance(original_block, dict): + blocks.append(original_block) + continue + block = dict(original_block) + for field in ("text", "content"): + if isinstance(block.get(field), str): + block[field] = preview_text(block[field], limit) + blocks.append(block) + message[key] = blocks + bounded.append(message) + while bounded: + size = len(json.dumps(bounded, ensure_ascii=False).encode("utf-8")) + if size <= max_bytes and (not max_tokens or estimate_tokens(bounded, calibration=calibration) <= max_tokens): + return bounded + if len(bounded) > 1: + bounded = bounded[:1] + _balanced_tail(bounded[2:], len(bounded)) + continue + content = bounded[0].get("content") + if isinstance(content, str) and len(content.encode("utf-8")) > 512: + bounded[0]["content"] = preview_text(content, max(512, len(content.encode("utf-8")) // 2)) + continue + raise ValueError("opening context exceeds the configured projection budget") + return [] + + def _balanced_tail( messages: list[dict[str, Any]], keep_recent: int ) -> list[dict[str, Any]]: @@ -158,6 +208,7 @@ def summarize( *, request, keep_recent: int = 8, + on_degraded=None, ) -> tuple[dict[str, Any], list[dict[str, Any]]]: """Produce a structured summary plus the retained tail. @@ -175,9 +226,14 @@ def summarize( try: raw = request(_PROMPT, dropped) summary = normalize_summary(_parse_json(raw)) - except Exception: + except Exception as exc: + _recovery_logger.debug('Error handled in summarize', exc_info=True) + if on_degraded is not None: + on_degraded(f"{type(exc).__name__}: {exc}"[:1000]) return fallback_summary, retained if is_empty(summary): + if on_degraded is not None: + on_degraded("semantic summarizer returned no usable structured facts") return fallback_summary, retained return summary, retained diff --git a/backend/agent/context.py b/backend/agent/context.py index f25b6fd..78d9160 100644 --- a/backend/agent/context.py +++ b/backend/agent/context.py @@ -1,18 +1,3 @@ -"""Context projection: rebuild what the provider sees from durable events. - -The reconstruction rule is deliberately simple and total: the newest -compaction's summary leads, followed by every message the compaction retained -and every event recorded after it. Because ``agent_events`` is append-only, a -projection decision is never destructive - a bad compaction costs one turn of -context quality, not the evidence chain. - -Budgeting works in tokens rather than bytes. A byte threshold is wrong by a -factor of roughly two between CJK and ASCII text, which in practice means -compacting far too late (context overflow) or far too early (throwing away -usable evidence). The estimate is calibrated against the real -``usage.prompt_tokens`` the provider reports, so it converges on the tokenizer -actually in use instead of assuming one. -""" from __future__ import annotations import json @@ -45,7 +30,7 @@ ("gpt-4o", 128_000), ("o4", 200_000), ("o3", 200_000), - ("deepseek", 128_000), + ("deepseek", 1_000_000), ("kimi-k2", 256_000), ("kimi-for-coding", 256_000), ("qwen3", 128_000), diff --git a/backend/agent/job_service.py b/backend/agent/job_service.py index c394dfb..5f5ae3a 100644 --- a/backend/agent/job_service.py +++ b/backend/agent/job_service.py @@ -1,22 +1,12 @@ -"""Run a sandbox command that outlives the turn which started it. - -This is the orchestration half of P4a: :mod:`backend.agent.jobs` owns the -durable rows, the sandbox owns the process, and this module is what keeps the -two honest about each other. - -The distinction it exists to protect: a job whose process has vanished without -a recorded exit status is ``lost``, not ``exited 0``. Telling a model that a -brute force "finished successfully" when the container was actually restarted -underneath it produces confident, wrong conclusions, so the uncertainty is -reported instead of smoothed over. -""" from __future__ import annotations +import logging from typing import Any from backend.agent.jobs import INLINE_LIMIT, AgentJobStore, JobConflict, new_job_id from backend.sandbox.command_policy import host_path_violation +_recovery_logger = logging.getLogger(__name__) class JobService: """Bind a job store to one session's sandbox.""" @@ -49,6 +39,7 @@ def start(self, command: str, *, kind: str = "bash") -> dict[str, Any]: try: pid = self.sandbox.exec_detached(job_id, command) except Exception as exc: + _recovery_logger.debug('Error handled in start', exc_info=True) self.store.finish(job_id, "lost") return { "ok": False, @@ -80,11 +71,30 @@ def status(self, job_id: str) -> dict[str, Any]: return self._view(row) return self._view(self._reconcile(row)) + def wait(self, job_id: str, *, timeout: int = 5, cancel=None) -> dict[str, Any]: + import time + + deadline = time.monotonic() + max(1, min(15, timeout)) + while True: + result = self.status(job_id) + if result.get("state") != "running" or not result.get("ok", True): + return result + if cancel is not None and cancel.is_set(): + return {**result, "wait_cancelled": True} + remaining = deadline - time.monotonic() + if remaining <= 0: + return {**result, "wait_timed_out": True} + if cancel is not None: + cancel.wait(min(.5, remaining)) + else: + time.sleep(min(.5, remaining)) + def _reconcile(self, row: dict) -> dict: """Fold the sandbox's observation of a running job into its row.""" try: observed = self.sandbox.job_state(row["id"]) except Exception: + _recovery_logger.debug('Error handled in _reconcile', exc_info=True) return row state = observed.get("state") if state == "running": @@ -133,6 +143,7 @@ def logs( job_id, stream, offset=start, limit=capped ) except Exception as exc: + _recovery_logger.debug('Error handled in logs', exc_info=True) return { "ok": False, "job_id": job_id, @@ -166,6 +177,7 @@ def cancel(self, job_id: str) -> dict[str, Any]: try: self.sandbox.job_cancel(job_id) except Exception as exc: + _recovery_logger.debug('Error handled in cancel', exc_info=True) return { "ok": False, "job_id": job_id, @@ -194,4 +206,4 @@ def _view(row: dict) -> dict[str, Any]: return view -__all__ = ["JobService", "JobConflict", "AgentJobStore"] +__all__ = ["AgentJobStore", "JobConflict", "JobService"] diff --git a/backend/agent/jobs.py b/backend/agent/jobs.py index 95ad4df..c050e1e 100644 --- a/backend/agent/jobs.py +++ b/backend/agent/jobs.py @@ -1,19 +1,3 @@ -"""Durable records for sandbox work that outlives the turn that started it. - -Every Member command used to run under ``timeout -k 5s 60s``, so a brute force -pass, a large file analysis or a slow scan was killed rather than finished. -A job is started detached and polled instead; this module is the durable half -of that, and the reason a poll still works after the worker restarts. - -Two rules shape it: - -* A job's handle is its row plus the files in the workspace volume, never a - docker exec id. Exec ids die with the container; the volume does not. -* ``lost`` is a real state. A job whose process is gone with no recorded exit - status is reported as ``lost``, never as ``exited 0`` -- "unknown" and - "succeeded" are completely different evidence to a model deciding what to do - next. -""" from __future__ import annotations import uuid diff --git a/backend/agent/runtime.py b/backend/agent/runtime.py index 58a6231..4f200c9 100644 --- a/backend/agent/runtime.py +++ b/backend/agent/runtime.py @@ -1,32 +1,16 @@ -"""The single durable action loop shared by Ops, competition and projects. - -The loop has no step budget: it ends when the provider stops asking for tools, -when cancellation is requested, when the writer loses its fence, or when the -provider repeats an identical tool batch (a real loop, as opposed to slow -progress). Everything it needs from storage arrives through a -:class:`SessionWriter`, which is what lets one implementation serve a -competition Member seat and an Ops chat turn without either knowing about the -other's lease model. - -Durability rules the loop depends on: - -* Every provider message, tool call and tool result is appended before the next - step, keyed so a replay after a restart is idempotent. -* A tool call whose result never landed is repaired as ``unknown`` rather than - silently retried, because the side effect may already have happened. -* Executors receive a stable idempotency key and are responsible for returning - a prior result instead of repeating a side effect. -""" from __future__ import annotations import asyncio import hashlib import json +import logging import threading +import time +from collections.abc import Callable from dataclasses import dataclass -from typing import Any, Callable, Protocol +from typing import Any, Protocol -from backend.agent.compaction import deterministic_summary +from backend.agent.compaction import deterministic_summary, fit_history from backend.agent.context import ( calibrate, compacted_history, @@ -42,6 +26,8 @@ ) from backend.competition.store import CompetitionConflict +_recovery_logger = logging.getLogger(__name__) + _EVENT_KEY_REUSE_MESSAGE = "event key reused with different content" @@ -198,6 +184,16 @@ def save_calibration(self, calibration: float) -> None: budget["calibration_ratio"] = calibration self.store.save_token_budget(self.session_id, budget) + def record_provider_progress(self, turn: int, status: str, seconds: float) -> None: + if not getattr(self.store, "provider_progress_events", False): + return + if status == "provider.started": + self.store.record_turn(self.assignment, turn, "running") + self.append( + f"provider-progress:{turn}:{time.monotonic_ns()}", "status", + {"status": status, "turn": turn, "latency_seconds": round(seconds, 6)}, + ) + class AgentRuntime: """One provider-native tool loop, independent of who owns the session.""" @@ -404,6 +400,10 @@ def _compact(self, messages: list[dict[str, Any]], turn: int) -> list[dict[str, if not needed: return messages + started = time.monotonic() + self._append(f"compact-start:{self._writer_epoch()}:{turn}", "status", + {"status": "compaction.started", "turn": turn, **detail}) + summary: Any compacted: list[dict[str, Any]] if self.compressor is None: @@ -421,6 +421,7 @@ def _compact(self, messages: list[dict[str, Any]], turn: int) -> list[dict[str, if not compacted: raise ValueError("context compressor returned no provider messages") except Exception as exc: + _recovery_logger.debug('Error handled in _compact', exc_info=True) summary, compacted = deterministic_summary( messages, keep_recent=self.keep_recent_messages ) @@ -429,8 +430,27 @@ def _compact(self, messages: list[dict[str, Any]], turn: int) -> list[dict[str, {**detail, "status": "compressor_failed", "error": f"{type(exc).__name__}: {exc}"[:2000]}, ) - payload = {"summary": summary, "messages": compacted} + from backend.sandbox.output import preview_text + + summary_budget = max(1024, min(8192, self.context_bytes // 8)) + if len(json.dumps(summary, ensure_ascii=False).encode("utf-8")) > summary_budget: + summary = {"text": preview_text(json.dumps(summary, ensure_ascii=False), summary_budget - 128)} + compacted = fit_history( + compacted, max_bytes=max(2048, int(self.context_bytes * .6) - summary_budget), + max_tokens=max(512, int(self.context_token_limit * .5)) if self.context_token_limit else 0, + calibration=self.calibration, + ) + metadata = dict(getattr(self.compressor, "last_metadata", {})) + fallback = "compacted without a semantic summary" in json.dumps(summary, ensure_ascii=False) + payload = {"summary": summary, "messages": compacted, "metrics": { + "duration_seconds": round(time.monotonic() - started, 6), + "input_tokens": detail.get("tokens"), "output_tokens": estimate_tokens(compacted), + "output_bytes": len(json.dumps(compacted, ensure_ascii=False).encode("utf-8")), + "degraded": fallback or bool(metadata.get("degraded_reason")), **metadata, + }} self._append(f"compact:{self._writer_epoch()}:{turn}", "context_compacted", payload) + self._append(f"compact-end:{self._writer_epoch()}:{turn}", "status", + {"status": "compaction.completed", "turn": turn, **payload["metrics"]}) record = getattr(self.writer, "record_compaction", None) if callable(record): record(summary, token_estimate=detail.get("tokens")) @@ -731,6 +751,46 @@ def _turn_args(self, messages: list[dict[str, Any]], on_delta, on_reasoning): args.append(on_reasoning) return args + def _provider_callbacks(self, turn: int): + sequences = getattr(self, "recon_sequences", []) + if sequences: + self._append(f"recon-context:{self._writer_epoch()}:{turn}", "status", + {"status": "recon.in_provider_context", "turn": turn, "sequences": sequences}) + self.recon_sequences = [] + recorder = getattr(self.writer, "record_provider_progress", None) + if callable(recorder): + recorder(turn, "provider.started", 0.0) + started = time.monotonic() + responded = False + content_received = False + + def first_response(): + nonlocal responded + if not responded: + responded = True + if callable(recorder): + recorder(turn, "provider.first_response", time.monotonic() - started) + + def completed(): + first_response() + if callable(recorder): + recorder(turn, "provider.completed", time.monotonic() - started) + + def first_content(): + nonlocal content_received + if not content_received: + content_received = True + if callable(recorder): + recorder(turn, "provider.first_content", time.monotonic() - started) + + if hasattr(self.adapter, "on_first_response"): + self.adapter.on_first_response = first_response + if hasattr(self.adapter, "on_first_content"): + self.adapter.on_first_content = first_content + if hasattr(self.adapter, "on_progress") and callable(recorder): + self.adapter.on_progress = lambda: recorder(turn, "provider.progress", time.monotonic() - started) + return first_response, completed + def _record_tool_call(self, call: ToolCall) -> None: self._append( f"tool-call:{call.id}", "tool_call", @@ -845,14 +905,20 @@ def run(self, initial_message: str | None = None) -> SessionResult: messages.extend(self.before_turn(self)) messages = self._compact(messages, turn_number) deltas: list[str] = [] + first_response, provider_completed = self._provider_callbacks(turn_number) - def on_delta(text: str) -> None: + def on_delta(text: str, deltas=deltas, first_response=first_response) -> None: + first_response() deltas.append(text) self.emit(text) + def on_reasoning(text: str, first_response=first_response) -> None: + first_response() + self.emit_reasoning(text) + try: turn = self.adapter.turn( - *self._turn_args(messages, on_delta, self.emit_reasoning) + *self._turn_args(messages, on_delta, on_reasoning) ) except TurnTruncated as exc: truncated_attempts += 1 @@ -865,6 +931,7 @@ def on_delta(text: str) -> None: last_text = exc.turn.text or last_text continue except Exception as exc: + _recovery_logger.debug('Error handled in run', exc_info=True) messages = self._handle_turn_failure( exc, messages, turn_number, repaired_turn ) @@ -872,6 +939,7 @@ def on_delta(text: str) -> None: turn_number -= 1 continue repaired_turn = -1 + provider_completed() truncated_attempts = 0 last_text = turn.text self._append( @@ -932,17 +1000,23 @@ async def run_async(self, initial_message: str | None = None) -> SessionResult: return SessionResult("turn_budget", turn_number - 1, last_text) if self.before_turn is not None: messages.extend(self.before_turn(self)) - messages = self._compact(messages, turn_number) + messages = await asyncio.to_thread(self._compact, messages, turn_number) deltas: list[str] = [] + first_response, provider_completed = self._provider_callbacks(turn_number) - def on_delta(text: str) -> None: + def on_delta(text: str, deltas=deltas, first_response=first_response) -> None: + first_response() deltas.append(text) self.emit(text) + def on_reasoning(text: str, first_response=first_response) -> None: + first_response() + self.emit_reasoning(text) + try: turn = await asyncio.to_thread( self.adapter.turn, - *self._turn_args(messages, on_delta, self.emit_reasoning), + *self._turn_args(messages, on_delta, on_reasoning), ) except TurnTruncated as exc: truncated_attempts += 1 @@ -955,6 +1029,7 @@ def on_delta(text: str) -> None: last_text = exc.turn.text or last_text continue except Exception as exc: + _recovery_logger.debug('Error handled in run_async', exc_info=True) messages = self._handle_turn_failure( exc, messages, turn_number, repaired_turn ) @@ -962,6 +1037,7 @@ def on_delta(text: str) -> None: turn_number -= 1 continue repaired_turn = -1 + provider_completed() truncated_attempts = 0 last_text = turn.text self._append( diff --git a/backend/agent/schema.py b/backend/agent/schema.py index 462c0f0..51f0ceb 100644 --- a/backend/agent/schema.py +++ b/backend/agent/schema.py @@ -1,13 +1,3 @@ -"""Storage contract for the unified agent runtime. Keep migration-stable. - -The central invariant: ``agent_events`` is append-only. Nothing in this -contract modifies or deletes an event, so the evidence chain of a task -survives compaction, member handoff, and worker restarts. Context -reconstruction is a projection over these events (see -``backend.agent.context``), and a compaction is a *derived* summary recorded -in ``agent_compactions`` rather than a destructive rewrite of the transcript. -""" - AGENT_SCHEMA = ( """CREATE TABLE IF NOT EXISTS agent_sessions ( id TEXT PRIMARY KEY, @@ -136,24 +126,26 @@ -- any kind. A long provider turn and a long tool both count as -- progress; only genuine silence accumulates here. GREATEST( - COALESCE(e.last_event_at, s.created_at), + COALESCE(e.last_progress_event_at, s.created_at), COALESCE(t.current_turn_started_at, s.created_at), COALESCE(t.last_finished_turn_at, s.created_at), - COALESCE(j.last_job_progress_at, s.created_at), - COALESCE(w.last_renewed_at, s.created_at) + COALESCE(j.last_job_progress_at, s.created_at) ) AS last_progress_at, EXTRACT(EPOCH FROM now() - GREATEST( - COALESCE(e.last_event_at, s.created_at), + COALESCE(e.last_progress_event_at, s.created_at), COALESCE(t.current_turn_started_at, s.created_at), COALESCE(t.last_finished_turn_at, s.created_at), - COALESCE(j.last_job_progress_at, s.created_at), - COALESCE(w.last_renewed_at, s.created_at) + COALESCE(j.last_job_progress_at, s.created_at) ))::bigint AS quiet_seconds FROM agent_sessions s LEFT JOIN agent_session_writers w ON w.session_id = s.id LEFT JOIN ( SELECT session_id, max(created_at) AS last_event_at, + max(created_at) FILTER (WHERE kind IN ('tool_call','tool_result','context_compacted') + OR (kind='status' AND payload->>'status' IN + ('provider.started','provider.first_content','provider.progress','provider.completed', + 'compaction.started','compaction.completed'))) AS last_progress_event_at, count(*) AS event_count, (array_agg(kind ORDER BY sequence DESC))[1] AS last_event_kind FROM agent_events GROUP BY session_id diff --git a/backend/agent/session.py b/backend/agent/session.py index 601c928..ab58721 100644 --- a/backend/agent/session.py +++ b/backend/agent/session.py @@ -1,16 +1,3 @@ -"""Short-transaction access to the unified agent session tables. - -Two rules shape this module: - -* ``agent_events`` is append-only. A compaction records a derived summary in - ``agent_compactions``; it never edits or deletes the events it summarizes, so - the evidence chain of a task outlives every projection decision. -* Writing requires a live fence. ``append_event`` validates - ``agent_session_writers`` inside the same transaction as the insert, which is - what makes a member handoff safe: ownership moves to a new worker and the - epoch advances, and the previous owner's next write is rejected instead of - interleaving into the same transcript. -""" from __future__ import annotations import uuid diff --git a/backend/agent/watchdog.py b/backend/agent/watchdog.py index 06167e1..ec7a03b 100644 --- a/backend/agent/watchdog.py +++ b/backend/agent/watchdog.py @@ -1,32 +1,7 @@ -"""Graded response to a session that has stopped making progress (P5c). - -The failure this exists for: a Member's seat lease was renewed by the -coordinator's tick, so a worker that had silently stopped kept its challenge -until the deadline. P5b made the worker certify itself; this decides what to do -once a session goes quiet anyway. - -The response is graded rather than binary because the cost of being wrong is -asymmetric. Killing a Member that is thirty seconds into a legitimate three -minute provider turn destroys real work; waiting one extra interval on a -genuinely dead one costs almost nothing. So escalation runs: - - diagnose -> cancel the current turn -> hand the session to another Member - -Each step is recorded before it acts, which is what makes a later postmortem -able to distinguish "we killed a working Member" from "the Member was gone". - -One subtlety worth stating plainly, because it is the easiest thing to get -wrong here: competition writes are fenced by the **assignment** lease -(``competition_assignments.lease_owner``/``epoch``), while a handoff moves the -**writer** row. Advancing only one of them leaves the new Member unable to -write a single event. ``handoff_assignment`` therefore moves both inside one -transaction. -""" from __future__ import annotations from dataclasses import dataclass - DIAGNOSE_AFTER = 300 CANCEL_AFTER = 900 HANDOFF_AFTER = 1800 diff --git a/backend/api/agent.py b/backend/api/agent.py index a080761..d7945f9 100644 --- a/backend/api/agent.py +++ b/backend/api/agent.py @@ -1,14 +1,3 @@ -"""Agent session observability: tell "alive" apart from "making progress". - -The UI used to render four different facts as one green dot -- the API is up, -the coordinator is ticking, a worker holds the seat, and the task is actually -advancing. A run could look perfectly healthy while a Member had been silent -for twenty minutes. - -These endpoints expose each fact separately, straight from -``agent_session_health``, plus a merged event stream so a Member card can show -what its session is doing right now rather than only that it exists. -""" from __future__ import annotations import json @@ -23,15 +12,17 @@ from backend.api.deps import get_state from backend.core.state import AppState - router = APIRouter(prefix="/api/agent", tags=["agent"]) -def get_sessions(state: AppState = Depends(get_state)) -> AgentSessionStore: + +_GET_STATE_DEPENDENCY = Depends(get_state) + +def get_sessions(state: AppState = _GET_STATE_DEPENDENCY) -> AgentSessionStore: return AgentSessionStore(state.db) -def get_jobs(state: AppState = Depends(get_state)) -> AgentJobStore: +def get_jobs(state: AppState = _GET_STATE_DEPENDENCY) -> AgentJobStore: return AgentJobStore(state.db) @@ -49,13 +40,16 @@ def _view(row: dict) -> dict: } + +_GET_SESSIONS_DEPENDENCY = Depends(get_sessions) + @router.get("/sessions") def list_sessions( kind: str | None = Query(default=None), task_key: str | None = Query(default=None), state_filter: str | None = Query(default="active", alias="state"), limit: int = Query(default=100, ge=1, le=500), - sessions: AgentSessionStore = Depends(get_sessions), + sessions: AgentSessionStore = _GET_SESSIONS_DEPENDENCY, ): states: tuple[str, ...] = () if state_filter and state_filter != "all": @@ -68,11 +62,14 @@ def list_sessions( return {"sessions": [_view(row) for row in rows]} + +_GET_JOBS_DEPENDENCY = Depends(get_jobs) + @router.get("/sessions/{session_id}") def get_session( session_id: str, - sessions: AgentSessionStore = Depends(get_sessions), - jobs: AgentJobStore = Depends(get_jobs), + sessions: AgentSessionStore = _GET_SESSIONS_DEPENDENCY, + jobs: AgentJobStore = _GET_JOBS_DEPENDENCY, ): try: session = sessions.session(session_id) @@ -91,7 +88,7 @@ def get_session( def session_jobs( session_id: str, active_only: bool = Query(default=False), - jobs: AgentJobStore = Depends(get_jobs), + jobs: AgentJobStore = _GET_JOBS_DEPENDENCY, ): return {"jobs": jobs.jobs(session_id, active_only=active_only, limit=100)} @@ -101,8 +98,8 @@ def session_events( session_id: str, after: int = Query(default=0, ge=0), last_event_id: str | None = Header(default=None, alias="Last-Event-ID"), - sessions: AgentSessionStore = Depends(get_sessions), - state: AppState = Depends(get_state), + sessions: AgentSessionStore = _GET_SESSIONS_DEPENDENCY, + state: AppState = _GET_STATE_DEPENDENCY, ): """Stream one session's transcript, resumable by ``event_id``. @@ -157,7 +154,7 @@ def scope_events( task_key: str | None = Query(default=None), after: int = Query(default=0, ge=0), last_event_id: str | None = Header(default=None, alias="Last-Event-ID"), - sessions: AgentSessionStore = Depends(get_sessions), + sessions: AgentSessionStore = _GET_SESSIONS_DEPENDENCY, ): """Stream every session in a scope through one connection. diff --git a/backend/api/auth.py b/backend/api/auth.py index e99e564..74e196e 100644 --- a/backend/api/auth.py +++ b/backend/api/auth.py @@ -59,8 +59,11 @@ def _client_key(request: Request) -> str: return request.client.host if request.client else "unknown" + +_GET_AUTH_MANAGER_DEPENDENCY = Depends(get_auth_manager) + @router.get("/status") -def auth_status(request: Request, manager: AuthManager = Depends(get_auth_manager)) -> dict: +def auth_status(request: Request, manager: AuthManager = _GET_AUTH_MANAGER_DEPENDENCY) -> dict: return { "setup_required": False, "authenticated": True, @@ -73,7 +76,7 @@ def setup( payload: SetupRequest, request: Request, response: Response, - manager: AuthManager = Depends(get_auth_manager), + manager: AuthManager = _GET_AUTH_MANAGER_DEPENDENCY, ) -> dict: if manager.configuration_error: raise HTTPException( @@ -95,7 +98,7 @@ def login( payload: LoginRequest, request: Request, response: Response, - manager: AuthManager = Depends(get_auth_manager), + manager: AuthManager = _GET_AUTH_MANAGER_DEPENDENCY, ) -> dict: if manager.configuration_error: raise HTTPException( @@ -138,7 +141,7 @@ def login( def logout( request: Request, response: Response, - manager: AuthManager = Depends(get_auth_manager), + manager: AuthManager = _GET_AUTH_MANAGER_DEPENDENCY, ) -> None: manager.revoke_session(request.cookies.get(SESSION_COOKIE_NAME)) response.delete_cookie( diff --git a/backend/api/competition.py b/backend/api/competition.py index 098b9d3..8c00f40 100644 --- a/backend/api/competition.py +++ b/backend/api/competition.py @@ -12,7 +12,7 @@ from backend.competition.service import CompetitionService from backend.competition.store import CompetitionConflict from backend.core.state import AppState - +from backend.core.validation import InputValidationError router = APIRouter(prefix="/api", tags=["competition"]) @@ -30,7 +30,7 @@ def validate_select(cls, value: list[str] | None) -> list[str] | None: cleaned: list[str] = [] for item in value: if not isinstance(item, str): - raise ValueError("challenge ids must be strings") + raise InputValidationError("challenge ids must be strings") item = item.strip() if not item or len(item) > 256: raise ValueError("challenge ids must be 1 to 256 characters") @@ -44,6 +44,16 @@ def validate_select(cls, value: list[str] | None) -> list[str] | None: class ControlRequest(BaseModel): model_config = ConfigDict(extra="forbid") revision: int = Field(ge=1) + operation_id: str | None = Field(default=None, min_length=8, max_length=128, pattern=r"^[A-Za-z0-9_-]+$") + + +class LimitsRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + revision: int = Field(ge=1) + limits: dict[str, int] = Field(default_factory=dict) + max_concurrent_tasks: int | None = None + total_cpu: int | None = None + remote_concurrency: int | None = None class CandidateRequest(BaseModel): @@ -55,7 +65,10 @@ class CandidateRequest(BaseModel): instance_generation: int = Field(ge=0) -def get_competition(state: AppState = Depends(get_state)) -> CompetitionService: + +_GET_STATE_DEPENDENCY = Depends(get_state) + +def get_competition(state: AppState = _GET_STATE_DEPENDENCY) -> CompetitionService: service = getattr(state, "competition", None) if service is None: service = CompetitionService(state) @@ -72,15 +85,22 @@ def _call(function, *args, **kwargs): except PermissionError as exc: raise HTTPException(403, str(exc)) from exc except CompetitionConflict as exc: - raise HTTPException(409, str(exc)) from exc + headers = {} + retry_after = getattr(exc, "retry_after", None) + if retry_after is not None: + headers["Retry-After"] = str(retry_after) + raise HTTPException(409, str(exc), headers=headers) from exc except ValueError as exc: raise HTTPException(400, str(exc)) from exc + +_GET_COMPETITION_DEPENDENCY = Depends(get_competition) + @router.post("/workflows/{workflow_id}/preflight") def preflight( workflow_id: str, - service: CompetitionService = Depends(get_competition), + service: CompetitionService = _GET_COMPETITION_DEPENDENCY, ): return _call(service.preflight, workflow_id) @@ -89,7 +109,7 @@ def preflight( def start( workflow_id: str, body: StartRequest, - service: CompetitionService = Depends(get_competition), + service: CompetitionService = _GET_COMPETITION_DEPENDENCY, ): return _call(service.start, workflow_id, body.idempotency_key, body.select) @@ -97,18 +117,18 @@ def start( @router.get("/runs") def list_runs( limit: int = Query(default=50, ge=1, le=200), - service: CompetitionService = Depends(get_competition), + service: CompetitionService = _GET_COMPETITION_DEPENDENCY, ): return {"runs": service.store.list_runs(limit)} @router.get("/runs/{run_id}") -def get_run(run_id: str, service: CompetitionService = Depends(get_competition)): +def get_run(run_id: str, service: CompetitionService = _GET_COMPETITION_DEPENDENCY): return _call(service.store.run_snapshot, run_id) @router.get("/runs/{run_id}/members") -def get_members(run_id: str, service: CompetitionService = Depends(get_competition)): +def get_members(run_id: str, service: CompetitionService = _GET_COMPETITION_DEPENDENCY): return {"members": _call(service.store.member_snapshot, run_id)} @@ -117,17 +137,38 @@ def control_run( run_id: str, action: Literal["pause", "resume", "stop", "refresh"], body: ControlRequest, - service: CompetitionService = Depends(get_competition), + service: CompetitionService = _GET_COMPETITION_DEPENDENCY, ): + if body.operation_id: + return _call(service.control, run_id, action, body.revision, operation_id=body.operation_id) return _call(service.control, run_id, action, body.revision) +@router.get("/runs/{run_id}/operations/{operation_id}") +def control_operation(run_id: str, operation_id: str, service: CompetitionService = _GET_COMPETITION_DEPENDENCY): + return _call(service.store.control_operation, run_id, operation_id) + + +@router.patch("/runs/{run_id}/limits") +def patch_run_limits( + run_id: str, + body: LimitsRequest, + service: CompetitionService = _GET_COMPETITION_DEPENDENCY, +): + limits = {**body.limits} + for key in ("max_concurrent_tasks", "total_cpu", "remote_concurrency"): + value = getattr(body, key) + if value is not None: + limits[key] = value + return _call(service.update_limits, run_id, limits, body.revision) + + @router.get("/runs/{run_id}/events") def run_events( run_id: str, after: int = Query(default=0, ge=0), last_event_id: str | None = Header(default=None, alias="Last-Event-ID"), - service: CompetitionService = Depends(get_competition), + service: CompetitionService = _GET_COMPETITION_DEPENDENCY, ): cursor = after if last_event_id: @@ -170,7 +211,7 @@ def stream(): def run_observations( run_id: str, limit: int = Query(default=100, ge=1, le=1000), - service: CompetitionService = Depends(get_competition), + service: CompetitionService = _GET_COMPETITION_DEPENDENCY, ): return {"observations": _call(service.store.observations, run_id, limit=limit)} @@ -180,7 +221,7 @@ def session_events( session_id: str, after: int = Query(default=0, ge=0), limit: int = Query(default=200, ge=1, le=500), - service: CompetitionService = Depends(get_competition), + service: CompetitionService = _GET_COMPETITION_DEPENDENCY, ): _call(service.store.session, session_id) return {"events": service.store.events(session_id, after, limit)} @@ -190,7 +231,7 @@ def session_events( def submit_candidate( challenge_id: str, body: CandidateRequest, - service: CompetitionService = Depends(get_competition), + service: CompetitionService = _GET_COMPETITION_DEPENDENCY, ): return _call( service.submit_candidate, @@ -206,11 +247,11 @@ def submit_candidate( @router.get("/submissions/{submission_id}") def get_submission( submission_id: str, - service: CompetitionService = Depends(get_competition), + service: CompetitionService = _GET_COMPETITION_DEPENDENCY, ): return _call(service.store.submission, submission_id) @router.post("/wp-jobs/{job_id}/retry") -def retry_wp(job_id: str, service: CompetitionService = Depends(get_competition)): +def retry_wp(job_id: str, service: CompetitionService = _GET_COMPETITION_DEPENDENCY): return _call(service.retry_wp, job_id) diff --git a/backend/api/config.py b/backend/api/config.py index 3c75530..daef726 100644 --- a/backend/api/config.py +++ b/backend/api/config.py @@ -5,13 +5,13 @@ from backend.api.deps import get_state from backend.core.config import ( - AppConfig, - LLMConfig, + CATEGORIES, ApiFormat, ApiSurface, - CATEGORIES, - SelectedReasoningEffort, + AppConfig, + LLMConfig, RuntimeConfig, + SelectedReasoningEffort, ) from backend.core.state import AppState @@ -38,6 +38,12 @@ class RuntimeUpdate(BaseModel): browser_artifact_max_bytes: int | None = None +class LimitsUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + max_concurrent_tasks: int | None = None + total_cpu: int | None = None + + class ConfigUpdate(BaseModel): model_config = ConfigDict(extra="forbid") @@ -48,6 +54,7 @@ class ConfigUpdate(BaseModel): members: dict[str, LLMUpdate] | None = None remove_members: list[str] | None = None runtime: RuntimeUpdate | None = None + limits: LimitsUpdate | None = None def _redact(value: str) -> str: @@ -101,18 +108,22 @@ def _config_view(state: AppState) -> dict: "browser_allowed_origins": cfg.runtime.browser_allowed_origins, "browser_artifact_max_bytes": cfg.runtime.browser_artifact_max_bytes, }, + "limits": cfg.limits.model_dump(), "startup_errors": cfg.startup_errors(), "startup_warnings": cfg.startup_warnings(), } + +_GET_STATE_DEPENDENCY = Depends(get_state) + @router.get("/config") -def get_config(state: AppState = Depends(get_state)): +def get_config(state: AppState = _GET_STATE_DEPENDENCY): return _config_view(state) @router.get("/config/runtime") -def get_runtime_config(state: AppState = Depends(get_state)): +def get_runtime_config(state: AppState = _GET_STATE_DEPENDENCY): orchestrator = state.orchestrator active_members: dict[str, list[str]] = {} running_tasks: list[dict[str, str]] = [] @@ -161,7 +172,7 @@ def _apply(llm, upd: LLMUpdate) -> None: @router.put("/config") -def update_config(body: ConfigUpdate, state: AppState = Depends(get_state)): +def update_config(body: ConfigUpdate, state: AppState = _GET_STATE_DEPENDENCY): cfg = state.config.model_copy(deep=True) if body.remove_members: raise HTTPException(400, "The ten Member identities are fixed.") @@ -197,13 +208,26 @@ def update_config(body: ConfigUpdate, state: AppState = Depends(get_state)): for error in exc.errors(include_input=False, include_context=False, include_url=False) ) raise HTTPException(400, detail) from exc + if body.limits is not None: + limit_values = { + **cfg.limits.model_dump(), + **body.limits.model_dump(exclude_none=True), + } + try: + cfg.limits = type(cfg.limits).model_validate(limit_values) + except ValidationError as exc: + detail = "; ".join( + f"{'.'.join(str(part) for part in error['loc'])}: {error['msg']}" + for error in exc.errors(include_input=False, include_context=False, include_url=False) + ) + raise HTTPException(400, detail) from exc state.config = AppConfig.model_validate(cfg.model_dump()) state.save_config() return _config_view(state) @router.post("/config/health") -def health_check(state: AppState = Depends(get_state)): +def health_check(state: AppState = _GET_STATE_DEPENDENCY): """Validate each configured LLM endpoint (Seed.md 启动时需要校验每个 LLM 的 health).""" from backend.members.adapters import health_check as adapter_health diff --git a/backend/api/flags.py b/backend/api/flags.py index 58e260f..3ff6c04 100644 --- a/backend/api/flags.py +++ b/backend/api/flags.py @@ -4,8 +4,8 @@ from pydantic import BaseModel from backend.api.deps import get_state -from backend.core.state import AppState from backend.core.redaction import redact_flag +from backend.core.state import AppState router = APIRouter(prefix="/api/flags", tags=["flags"]) @@ -38,10 +38,13 @@ def _record(row, *, reveal: bool = False) -> FlagRecord: ) + +_GET_STATE_DEPENDENCY = Depends(get_state) + @router.get("", response_model=list[FlagRecord]) def list_flags( reveal: bool = Query(default=False), - state: AppState = Depends(get_state), + state: AppState = _GET_STATE_DEPENDENCY, ): with state.db.connect() as conn: rows = conn.execute( @@ -59,7 +62,7 @@ def list_flags( def get_flag( project_id: str, reveal: bool = Query(default=False), - state: AppState = Depends(get_state), + state: AppState = _GET_STATE_DEPENDENCY, ): with state.db.connect() as conn: row = conn.execute( diff --git a/backend/api/graph.py b/backend/api/graph.py index 0b8ace0..e84b213 100644 --- a/backend/api/graph.py +++ b/backend/api/graph.py @@ -1,20 +1,22 @@ from __future__ import annotations - from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import Response from backend.api.deps import get_state from backend.blackboard import graph_store -from backend.core.state import AppState -from backend.core.replay import build_timeline, export_yaml from backend.core.redaction import redact_object +from backend.core.replay import build_timeline, export_yaml +from backend.core.state import AppState router = APIRouter(tags=["graph"]) + +_GET_STATE_DEPENDENCY = Depends(get_state) + @router.get("/projects/{project_id}/export") -def export_project(project_id: str, format: str = "yaml", state: AppState = Depends(get_state)): +def export_project(project_id: str, format: str = "yaml", state: AppState = _GET_STATE_DEPENDENCY): if format not in ("yaml", "timeline"): raise HTTPException(400, "format must be yaml or timeline") with state.db.connect() as conn: @@ -39,7 +41,7 @@ def _timeline_text(detail) -> str: @router.get("/projects/{project_id}/replay") -def replay_timeline(project_id: str, state: AppState = Depends(get_state)): +def replay_timeline(project_id: str, state: AppState = _GET_STATE_DEPENDENCY): """Ordered event frames for the UI replay (Seed.md 重放).""" with state.db.connect() as conn: detail = graph_store.project_detail(conn, project_id) diff --git a/backend/api/logs.py b/backend/api/logs.py index d73545f..7d449e7 100644 --- a/backend/api/logs.py +++ b/backend/api/logs.py @@ -1,13 +1,13 @@ from __future__ import annotations -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Query from pydantic import BaseModel from backend.api.deps import get_state from backend.blackboard import graph_store from backend.core.archive import list_archived_projects, project_archive_id -from backend.core.state import AppState from backend.core.redaction import redact_object +from backend.core.state import AppState from backend.filename_util import numbered_filename router = APIRouter(tags=["logs"]) @@ -23,13 +23,16 @@ class LogToggle(BaseModel): enabled: bool + +_GET_STATE_DEPENDENCY = Depends(get_state) + @router.get("/logs/status") -def log_status(state: AppState = Depends(get_state)): +def log_status(state: AppState = _GET_STATE_DEPENDENCY): return {"enabled": state.logger.enabled} @router.put("/logs/status") -def set_log_status(body: LogToggle, state: AppState = Depends(get_state)): +def set_log_status(body: LogToggle, state: AppState = _GET_STATE_DEPENDENCY): state.logger.set_enabled(body.enabled) state.config.log_enabled = body.enabled state.save_config() @@ -37,7 +40,7 @@ def set_log_status(body: LogToggle, state: AppState = Depends(get_state)): @router.get("/logs/projects") -def read_project_logs(limit: int = 500, state: AppState = Depends(get_state)): +def read_project_logs(limit: int = 500, state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: projects = graph_store.project_summaries(conn) logs = [] @@ -86,7 +89,7 @@ def read_project_logs(limit: int = 500, state: AppState = Depends(get_state)): @router.post("/logs/derive") -def derive_project_logs(state: AppState = Depends(get_state)): +def derive_project_logs(state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: projects = graph_store.project_summaries(conn) target = state.log_export_dir @@ -143,8 +146,13 @@ def derive_project_logs(state: AppState = Depends(get_state)): @router.get("/logs/{project_id}") -def read_logs(project_id: str, kind: str = "project", limit: int = 500, state: AppState = Depends(get_state)): +def read_logs(project_id: str, kind: str = "project", limit: int = Query(default=500, ge=1, le=1000), + after: int | None = Query(default=None, ge=0), file_id: str = "", state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: row = graph_store.get_project_row(conn, project_id) secret = row["flag"] if row else None + if after is not None: + page = state.logger.read_since(kind, project_id, after=after, limit=limit, file_id=file_id) + page["entries"] = [redact_object(entry, secret) for entry in page["entries"]] + return page return {"entries": [redact_object(entry, secret) for entry in state.logger.read_log(kind, project_id, limit)]} diff --git a/backend/api/memory.py b/backend/api/memory.py index b31e667..e7d5782 100644 --- a/backend/api/memory.py +++ b/backend/api/memory.py @@ -5,8 +5,8 @@ from pydantic import BaseModel, Field from backend.api.deps import get_state -from backend.core.state import AppState from backend.core.redaction import redact_object +from backend.core.state import AppState from backend.memory.exporter.obsidian import export_obsidian from backend.memory.memory_search import search as mem_search from backend.memory.memory_store import CATEGORIES, Memory @@ -21,8 +21,11 @@ class AddMemoryRequest(BaseModel): tags: list[str] = Field(default_factory=list) + +_GET_STATE_DEPENDENCY = Depends(get_state) + @router.get("/memory", response_model=list[Memory]) -def list_memory(category: str | None = None, state: AppState = Depends(get_state)): +def list_memory(category: str | None = None, state: AppState = _GET_STATE_DEPENDENCY): memories = state.memory.list(category) with state.db.connect() as conn: flags = { @@ -36,7 +39,7 @@ def list_memory(category: str | None = None, state: AppState = Depends(get_state @router.post("/memory", response_model=Memory, status_code=201) -def add_memory(body: AddMemoryRequest, state: AppState = Depends(get_state)): +def add_memory(body: AddMemoryRequest, state: AppState = _GET_STATE_DEPENDENCY): if body.category not in CATEGORIES: raise HTTPException(400, f"category must be one of {CATEGORIES}") mem = state.memory.add(body.category, body.title, body.content, body.tags, source="human") @@ -45,13 +48,13 @@ def add_memory(body: AddMemoryRequest, state: AppState = Depends(get_state)): @router.delete("/memory/{memory_id}", status_code=204) -def delete_memory(memory_id: str, state: AppState = Depends(get_state)): +def delete_memory(memory_id: str, state: AppState = _GET_STATE_DEPENDENCY): if not state.memory.delete(memory_id): raise HTTPException(404, "Memory not found") @router.get("/memory/search") -def search_memory(q: str, category: str | None = None, limit: int = 5, state: AppState = Depends(get_state)): +def search_memory(q: str, category: str | None = None, limit: int = 5, state: AppState = _GET_STATE_DEPENDENCY): results = mem_search(state.memory, q, category=category, limit=limit) with state.db.connect() as conn: flags = { @@ -65,12 +68,12 @@ def search_memory(q: str, category: str | None = None, limit: int = 5, state: Ap @router.get("/memory/catalog") -def memory_catalog(state: AppState = Depends(get_state)): +def memory_catalog(state: AppState = _GET_STATE_DEPENDENCY): return {"children": state.catalog.tree()} @router.get("/memory/catalog/{entry_id}") -def memory_catalog_entry(entry_id: str, state: AppState = Depends(get_state)): +def memory_catalog_entry(entry_id: str, state: AppState = _GET_STATE_DEPENDENCY): entry = state.catalog.get(entry_id) if entry is None: raise HTTPException(404, "Catalog document not found") @@ -84,7 +87,7 @@ def memory_catalog_entry(entry_id: str, state: AppState = Depends(get_state)): @router.get("/memory/catalog/{entry_id}/document") def memory_catalog_document( entry_id: str, - state: AppState = Depends(get_state), + state: AppState = _GET_STATE_DEPENDENCY, ): entry = state.catalog.get(entry_id) if entry is None: @@ -99,6 +102,6 @@ def memory_catalog_document( @router.post("/memory/derive") -def derive_memory(state: AppState = Depends(get_state)): +def derive_memory(state: AppState = _GET_STATE_DEPENDENCY): vault = export_obsidian(state.memory, state.memory_export_dir / "vault") return {"vault": str(vault)} diff --git a/backend/api/ops_agent.py b/backend/api/ops_agent.py index e8192e6..e3f0e74 100644 --- a/backend/api/ops_agent.py +++ b/backend/api/ops_agent.py @@ -5,20 +5,34 @@ from typing import Any, Literal import requests -from fastapi import APIRouter, Depends, File, HTTPException, Response, UploadFile, status +from fastapi import ( + APIRouter, + Depends, + File, + HTTPException, + Response, + UploadFile, + status, +) from fastapi.responses import StreamingResponse from pydantic import BaseModel, ConfigDict, Field, field_validator from backend.api.deps import get_state from backend.core.config import ApiFormat, ApiSurface, SelectedReasoningEffort from backend.core.state import AppState -from backend.ops.network import NetworkPolicyError from backend.ops.attachments import MAX_UPLOAD_BYTES, OpsAttachmentStore +from backend.ops.network import NetworkPolicyError +from backend.ops.presets import ( + GZCTFWorkflowInput, + TSECBenchWorkflowInput, + workflow_templates, +) from backend.ops.service import ( OpsAgentNotConfigured, OpsAgentService, OpsAgentUpstreamError, WorkflowConfirmationError, + WorkflowSessionError, ) from backend.ops.store import WorkflowInUse @@ -98,7 +112,10 @@ def validate_select(cls, value: list[str] | None) -> list[str] | None: return value -def get_ops_service(state: AppState = Depends(get_state)) -> OpsAgentService: + +_GET_STATE_DEPENDENCY = Depends(get_state) + +def get_ops_service(state: AppState = _GET_STATE_DEPENDENCY) -> OpsAgentService: service = getattr(state, "ops_agent_service", None) if service is not None: return service @@ -110,38 +127,41 @@ def get_ops_service(state: AppState = Depends(get_state)) -> OpsAgentService: return service + +_GET_OPS_SERVICE_DEPENDENCY = Depends(get_ops_service) + @router.get("/config") -def get_config(service: OpsAgentService = Depends(get_ops_service)): +def get_config(service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return service.config_view() @router.put("/config") -def update_config(body: OpsConfigUpdate, service: OpsAgentService = Depends(get_ops_service)): +def update_config(body: OpsConfigUpdate, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return _call(service.update_config, **body.model_dump(exclude_unset=True)) @router.post("/config/health") -def health(service: OpsAgentService = Depends(get_ops_service)): +def health(service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return _call(service.health) @router.get("/tools") -def tools(service: OpsAgentService = Depends(get_ops_service)): +def tools(service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return {"tools": service.tool_catalog()} @router.get("/sessions") -def list_sessions(service: OpsAgentService = Depends(get_ops_service)): +def list_sessions(service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return {"sessions": service.list_sessions()} @router.get("/sessions/{session_id}") -def get_session(session_id: str, service: OpsAgentService = Depends(get_ops_service)): +def get_session(session_id: str, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return _call(service.session_view, session_id) @router.delete("/sessions/{session_id}", status_code=status.HTTP_204_NO_CONTENT) -def delete_session(session_id: str, service: OpsAgentService = Depends(get_ops_service)): +def delete_session(session_id: str, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): try: deleted = service.delete_session(session_id) except ValueError as exc: @@ -152,7 +172,7 @@ def delete_session(session_id: str, service: OpsAgentService = Depends(get_ops_s @router.post("/chat") -def chat(body: ChatRequest, service: OpsAgentService = Depends(get_ops_service)): +def chat(body: ChatRequest, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return _call( service.chat, message=body.message, @@ -164,7 +184,7 @@ def chat(body: ChatRequest, service: OpsAgentService = Depends(get_ops_service)) @router.post("/chat/stream") -def chat_stream(body: ChatRequest, service: OpsAgentService = Depends(get_ops_service)): +def chat_stream(body: ChatRequest, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): def events(): for event in service.chat_stream( message=body.message, @@ -182,10 +202,13 @@ def events(): ) + +_REQUIRED_UPLOAD_FILE = File(...) + @router.post("/uploads") async def upload_attachment( - file: UploadFile = File(...), - service: OpsAgentService = Depends(get_ops_service), + file: UploadFile = _REQUIRED_UPLOAD_FILE, + service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY, ): """Upload platform API documentation for the next IPC message.""" @@ -205,17 +228,44 @@ async def upload_attachment( @router.post("/chat/interrupt") -def interrupt_chat(body: ChatInterruptRequest, service: OpsAgentService = Depends(get_ops_service)): +def interrupt_chat(body: ChatInterruptRequest, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return _call(service.interrupt_chat, session_id=body.session_id, run_id=body.run_id) @router.get("/workflows") -def list_workflows(service: OpsAgentService = Depends(get_ops_service)): +def list_workflows(service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return {"workflows": service.list_workflows()} +@router.get("/workflow-templates") +def list_workflow_templates(): + return {"default": "gzctf", "templates": workflow_templates()} + + +@router.post("/workflows/gzctf", status_code=status.HTTP_201_CREATED) +def create_gzctf_workflow(body: GZCTFWorkflowInput, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): + return _call(service.configure_gzctf_workflow, {**body.model_dump(), "secrets": body.secrets}) + + +@router.put("/workflows/{workflow_id}/gzctf") +def update_gzctf_workflow(workflow_id: str, body: GZCTFWorkflowInput, + service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): + return _call(service.configure_gzctf_workflow, {**body.model_dump(), "secrets": body.secrets}, workflow_id=workflow_id) + + +@router.post("/workflows/tsecbench", status_code=status.HTTP_201_CREATED) +def create_tsecbench_workflow(body: TSECBenchWorkflowInput, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): + return _call(service.configure_tsecbench_workflow, {**body.model_dump(), "secrets": body.secrets}) + + +@router.put("/workflows/{workflow_id}/tsecbench") +def update_tsecbench_workflow(workflow_id: str, body: TSECBenchWorkflowInput, + service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): + return _call(service.configure_tsecbench_workflow, {**body.model_dump(), "secrets": body.secrets}, workflow_id=workflow_id) + + @router.post("/workflows", status_code=status.HTTP_201_CREATED) -def create_workflow(body: WorkflowDraftRequest, service: OpsAgentService = Depends(get_ops_service)): +def create_workflow(body: WorkflowDraftRequest, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return _call( service.create_workflow, body.workflow, @@ -224,7 +274,7 @@ def create_workflow(body: WorkflowDraftRequest, service: OpsAgentService = Depen @router.get("/workflows/{workflow_id}") -def get_workflow(workflow_id: str, service: OpsAgentService = Depends(get_ops_service)): +def get_workflow(workflow_id: str, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return _call(service.get_workflow, workflow_id) @@ -232,7 +282,7 @@ def get_workflow(workflow_id: str, service: OpsAgentService = Depends(get_ops_se def update_workflow( workflow_id: str, body: WorkflowDraftRequest, - service: OpsAgentService = Depends(get_ops_service), + service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY, ): return _call( service.update_workflow, @@ -243,7 +293,7 @@ def update_workflow( @router.delete("/workflows/{workflow_id}", status_code=status.HTTP_204_NO_CONTENT) -def delete_workflow(workflow_id: str, service: OpsAgentService = Depends(get_ops_service)): +def delete_workflow(workflow_id: str, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): try: deleted = service.delete_workflow(workflow_id) except WorkflowInUse as exc: @@ -257,13 +307,13 @@ def delete_workflow(workflow_id: str, service: OpsAgentService = Depends(get_ops def update_workflow_secrets( workflow_id: str, body: WorkflowSecretsUpdate, - service: OpsAgentService = Depends(get_ops_service), + service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY, ): return _call(service.set_workflow_secrets, workflow_id, body.secrets) @router.get("/workflows/{workflow_id}/confirmation") -def confirmation_preview(workflow_id: str, service: OpsAgentService = Depends(get_ops_service)): +def confirmation_preview(workflow_id: str, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return _call(service.confirmation_preview, workflow_id) @@ -271,13 +321,13 @@ def confirmation_preview(workflow_id: str, service: OpsAgentService = Depends(ge def confirm_workflow( workflow_id: str, body: WorkflowConfirmRequest, - service: OpsAgentService = Depends(get_ops_service), + service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY, ): return _call(service.confirm_workflow, workflow_id, body.confirmation_phrase) @router.post("/workflows/{workflow_id}/revoke") -def revoke_workflow(workflow_id: str, service: OpsAgentService = Depends(get_ops_service)): +def revoke_workflow(workflow_id: str, service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY): return _call(service.revoke_workflow, workflow_id) @@ -285,7 +335,7 @@ def revoke_workflow(workflow_id: str, service: OpsAgentService = Depends(get_ops def execute_workflow( workflow_id: str, body: WorkflowExecuteRequest, - service: OpsAgentService = Depends(get_ops_service), + service: OpsAgentService = _GET_OPS_SERVICE_DEPENDENCY, ): return _call( service.execute, @@ -312,6 +362,8 @@ def _call(function, *args: Any, **kwargs: Any): raise HTTPException(502, str(exc)) from exc except WorkflowConfirmationError as exc: raise HTTPException(409, str(exc)) from exc + except WorkflowSessionError as exc: + raise HTTPException(409, str(exc)) from exc except NetworkPolicyError as exc: raise HTTPException(400, str(exc)) from exc except requests.RequestException as exc: diff --git a/backend/api/pentest.py b/backend/api/pentest.py new file mode 100644 index 0000000..282c7ab --- /dev/null +++ b/backend/api/pentest.py @@ -0,0 +1,101 @@ +from typing import Literal + +from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi.responses import Response + +from backend.api.deps import get_state +from backend.core.redaction import redact_object +from backend.core.state import AppState +from backend.core.wp_writer import _redact_text +from backend.pentest import projection, store +from backend.pentest.evidence import archive_pages +from backend.pentest.models import Action +from backend.pentest.report import render_report +from backend.sandbox.output import read_page_file + +router = APIRouter(tags=["pentest"]) +_STATE = Depends(get_state) + + +def _public(value, secret=None): + if isinstance(value, str): + return _redact_text(redact_object(value, secret)) + if isinstance(value, dict): + return {key: _public(item, secret) for key, item in value.items()} + if isinstance(value, list): + return [_public(item, secret) for item in value] + return value + + +def _error(exc): + return HTTPException(404 if isinstance(exc, KeyError) else 409, str(exc)) + + +@router.get("/projects/{project_id}/pentest") +def read_board(project_id: str, after: int = Query(0, ge=0), limit: int = Query(500, ge=1, le=1000), + record_offset: int = Query(0, ge=0), record_limit: int = Query(200, ge=1, le=500), state: AppState = _STATE): + try: + with state.db.connect() as conn: + data = store.snapshot(conn, project_id, after=after, limit=limit, record_offset=record_offset, record_limit=record_limit) + return _public(data, store.project(conn, project_id).get("flag")) + except (KeyError, ValueError) as exc: + raise _error(exc) from exc + + +@router.post("/projects/{project_id}/pentest/actions") +def update_board(project_id: str, body: Action, state: AppState = _STATE): + root = (state.projects_dir / project_id).resolve() + + def read_page(path, **kwargs): + target = (root / path).resolve() + if root not in target.parents or not target.is_file(): + raise ValueError("evidence must refer to an existing artifact in this project") + return read_page_file(target, **kwargs) + + try: + with state.db.connect() as conn: + store.project(conn, project_id) + prepared = archive_pages(read_page, body.data.get("path", ""), root / "evidence") if body.operation == "record_evidence" else None + with state.db.connect() as conn: + result = store.apply(conn, project_id, body.operation, body.data, read_artifact=lambda _: prepared) + return _public(result) + except (KeyError, ValueError, OSError) as exc: + raise _error(exc) from exc + + +@router.get("/projects/{project_id}/pentest/evidence/{evidence_id}") +def read_evidence(project_id: str, evidence_id: str, offset: int = Query(0, ge=0), limit: int = Query(8192, ge=1, le=8192), state: AppState = _STATE): + try: + with state.db.connect() as conn: + item = store.record(conn, project_id, evidence_id, "evidence") + relative = item["data"].get("archive_path") + root = (state.projects_dir / project_id).resolve() + target = (root / (relative or "")).resolve() + if not relative or root not in target.parents: + raise ValueError("evidence has no readable archive") + # The artifact is exact bytes; the board/report only expose redacted summaries. + return read_page_file(target, offset=offset, limit=limit) + except (KeyError, ValueError, OSError) as exc: + raise _error(exc) from exc + + +@router.get("/projects/{project_id}/pentest/graph") +def read_graph(project_id: str, view: Literal["exploration", "assets"] = "exploration", offset: int = Query(0, ge=0), + limit: int = Query(250, ge=1, le=500), q: str = Query("", max_length=200), state: AppState = _STATE): + try: + with state.db.connect() as conn: + row = store.project(conn, project_id) + return _public(projection.graph(conn, project_id, view=view, offset=offset, limit=limit, query=q), row.get("flag")) + except (KeyError, ValueError) as exc: + raise _error(exc) from exc + + +@router.get("/projects/{project_id}/pentest/report") +def read_report(project_id: str, state: AppState = _STATE): + try: + with state.db.connect() as conn: + store.project(conn, project_id) + content = render_report(conn, project_id) + return Response(content, media_type="text/markdown") + except (KeyError, ValueError) as exc: + raise _error(exc) from exc diff --git a/backend/api/platform.py b/backend/api/platform.py index ec3de07..b636ca0 100644 --- a/backend/api/platform.py +++ b/backend/api/platform.py @@ -14,9 +14,9 @@ from backend.core.state import AppState from backend.platform.adapter import PlatformAdapter from backend.platform.factory import build_adapter +from backend.platform.gzctf import GZCTFError from backend.platform.mapping import FieldMapping, PlatformChallenge from backend.platform.ret2shell import Ret2ShellAdapter, Ret2ShellClient, Ret2ShellError -from backend.platform.gzctf import GZCTFError router = APIRouter(prefix="/api/platform", tags=["platform"]) @@ -69,8 +69,11 @@ def preview_challenges(body: ChallengeRequest): return {"challenges": [challenge.model_dump() for challenge in challenges]} + +_GET_STATE_DEPENDENCY = Depends(get_state) + @router.post("/import", status_code=201) -def import_challenges(body: ImportRequest, state: AppState = Depends(get_state)): +def import_challenges(body: ImportRequest, state: AppState = _GET_STATE_DEPENDENCY): adapter, challenges = _fetch(body.mapping) by_id = {challenge.external_id: challenge for challenge in challenges} if body.select is None: diff --git a/backend/api/project.py b/backend/api/project.py index fe00817..e52e0bd 100644 --- a/backend/api/project.py +++ b/backend/api/project.py @@ -1,10 +1,11 @@ from __future__ import annotations -from fastapi import APIRouter, Depends, File, HTTPException, UploadFile +from collections.abc import Callable from pathlib import Path -from typing import Any, Callable, TypeVar +from typing import TypeVar import psycopg.errors +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile from backend.api.deps import get_state from backend.blackboard import edge_store, graph_store, node_store @@ -22,8 +23,8 @@ ReasonClaimRequest, ) from backend.core.config import CATEGORIES -from backend.core.state import AppState from backend.core.redaction import redact_object +from backend.core.state import AppState router = APIRouter(tags=["projects"]) MAX_PROJECT_ATTACHMENT_BYTES = 100 * 1024 * 1024 @@ -62,8 +63,13 @@ def _public_detail(detail): return detail.__class__.model_validate(redact_object(detail.model_dump(mode="json"), secret)) + +_GET_STATE_DEPENDENCY = Depends(get_state) + @router.get("/projects", response_model=list[ProjectSummary]) -def list_projects(state: AppState = Depends(get_state)): +def list_projects(state: AppState = _GET_STATE_DEPENDENCY, category: str | None = None): + if category is not None and category not in CATEGORIES: + raise HTTPException(400, "unknown project category") def read() -> list[ProjectSummary]: with state.db.connect() as conn: _expire(state, conn) @@ -73,26 +79,34 @@ def read() -> list[ProjectSummary]: summaries = _with_deadlock_retry(read) except psycopg.errors.OperationalError: if "all" in _last_project_list: - return _last_project_list["all"] + return [p for p in _last_project_list["all"] if category is None or p.category == category] raise _last_project_list["all"] = summaries - return summaries + return [p for p in summaries if category is None or p.category == category] @router.post("/projects", response_model=ProjectDetail, status_code=201) -def create_project(body: CreateProjectRequest, state: AppState = Depends(get_state)): +def create_project(body: CreateProjectRequest, state: AppState = _GET_STATE_DEPENDENCY): if body.category not in CATEGORIES: raise HTTPException(400, f"category must be one of {CATEGORIES}") + if body.pentest is not None: + from backend.pentest.models import Setup + try: + Setup.model_validate(body.pentest) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + if body.category != "pentest": + raise HTTPException(400, "pentest setup requires pentest category") hints = [(h.content, h.creator) for h in (body.hints or [])] with state.db.connect() as conn: - pid = graph_store.create_project(conn, body.title, body.origin, body.goal, body.category, hints) + pid = graph_store.create_project(conn, body.title, body.origin, body.goal, body.category, hints, pentest_setup=body.pentest) state.logger.project("project_created", pid, title=body.title, category=body.category) with state.db.connect() as conn: return _public_detail(graph_store.project_detail(conn, pid)) @router.get("/projects/{project_id}", response_model=ProjectDetail) -def get_project(project_id: str, state: AppState = Depends(get_state)): +def get_project(project_id: str, state: AppState = _GET_STATE_DEPENDENCY): def read(): with state.db.connect() as conn: _expire(state, conn, project_id) @@ -113,7 +127,7 @@ def read(): @router.delete("/projects/{project_id}", status_code=204) -def delete_project(project_id: str, state: AppState = Depends(get_state)): +def delete_project(project_id: str, state: AppState = _GET_STATE_DEPENDENCY): state.pool.stop_project(project_id) state.network.stop(project_id) with state.db.connect() as conn: @@ -124,9 +138,12 @@ def delete_project(project_id: str, state: AppState = Depends(get_state)): graph_store.delete_project(conn, project_id) + +_REQUIRED_UPLOAD_FILE = File(...) + @router.post("/projects/{project_id}/attachments") async def upload_attachment( - project_id: str, file: UploadFile = File(...), state: AppState = Depends(get_state) + project_id: str, file: UploadFile = _REQUIRED_UPLOAD_FILE, state: AppState = _GET_STATE_DEPENDENCY ): with state.db.connect() as conn: if graph_store.get_project_row(conn, project_id) is None: @@ -165,7 +182,7 @@ async def upload_attachment( @router.post("/projects/{project_id}/hints", response_model=Hint, status_code=201) -def create_hint(project_id: str, body: CreateHintRequest, state: AppState = Depends(get_state)): +def create_hint(project_id: str, body: CreateHintRequest, state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: if graph_store.get_project_row(conn, project_id) is None: raise HTTPException(404, "Project not found") @@ -184,7 +201,7 @@ def _require_active(conn, project_id: str): @router.post("/projects/{project_id}/intents", response_model=Intent, status_code=201) -def create_intent(project_id: str, body: CreateIntentRequest, state: AppState = Depends(get_state)): +def create_intent(project_id: str, body: CreateIntentRequest, state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: _require_active(conn, project_id) for fid in body.from_: @@ -199,7 +216,7 @@ def create_intent(project_id: str, body: CreateIntentRequest, state: AppState = @router.post("/projects/{project_id}/intents/{intent_id}/heartbeat", response_model=Intent) def heartbeat_intent( - project_id: str, intent_id: str, body: HeartbeatRequest, state: AppState = Depends(get_state) + project_id: str, intent_id: str, body: HeartbeatRequest, state: AppState = _GET_STATE_DEPENDENCY ): with state.db.connect() as conn: _require_active(conn, project_id) @@ -226,7 +243,7 @@ def heartbeat_intent( @router.post("/projects/{project_id}/intents/{intent_id}/release", response_model=Intent) def release_intent( - project_id: str, intent_id: str, body: HeartbeatRequest, state: AppState = Depends(get_state) + project_id: str, intent_id: str, body: HeartbeatRequest, state: AppState = _GET_STATE_DEPENDENCY ): with state.db.connect() as conn: _require_active(conn, project_id) @@ -254,7 +271,7 @@ def release_intent( @router.post("/projects/{project_id}/reason/claim") -def claim_reason(project_id: str, body: ReasonClaimRequest, state: AppState = Depends(get_state)): +def claim_reason(project_id: str, body: ReasonClaimRequest, state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: _require_active(conn, project_id) _, reason_timeout = graph_store.get_timeouts(conn) @@ -266,7 +283,7 @@ def claim_reason(project_id: str, body: ReasonClaimRequest, state: AppState = De @router.post("/projects/{project_id}/reason/heartbeat") -def heartbeat_reason(project_id: str, body: HeartbeatRequest, state: AppState = Depends(get_state)): +def heartbeat_reason(project_id: str, body: HeartbeatRequest, state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: _require_active(conn, project_id) _, reason_timeout = graph_store.get_timeouts(conn) @@ -281,7 +298,7 @@ def heartbeat_reason(project_id: str, body: HeartbeatRequest, state: AppState = @router.post("/projects/{project_id}/reason/release") -def release_reason(project_id: str, body: HeartbeatRequest, state: AppState = Depends(get_state)): +def release_reason(project_id: str, body: HeartbeatRequest, state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: _require_active(conn, project_id) current = graph_store.reason_holder(conn, project_id) @@ -293,7 +310,7 @@ def release_reason(project_id: str, body: HeartbeatRequest, state: AppState = De @router.post("/projects/{project_id}/intents/{intent_id}/conclude", response_model=ConcludeResponse) def conclude_intent( - project_id: str, intent_id: str, body: ConcludeRequest, state: AppState = Depends(get_state) + project_id: str, intent_id: str, body: ConcludeRequest, state: AppState = _GET_STATE_DEPENDENCY ): with state.db.connect() as conn: _require_active(conn, project_id) diff --git a/backend/api/questions.py b/backend/api/questions.py index d688e15..74a7bc0 100644 --- a/backend/api/questions.py +++ b/backend/api/questions.py @@ -4,6 +4,7 @@ from backend.api.deps import get_state from backend.competition.store import CompetitionConflict from backend.ops.questions import QuestionStore +from backend.ops.store import WorkflowSessionConflict router = APIRouter(prefix="/questions", tags=["questions"]) @@ -13,18 +14,35 @@ class Answer(BaseModel): value: str = Field(min_length=1, max_length=16384) +class Dismiss(BaseModel): + session_id: str + + + +_GET_STATE_DEPENDENCY = Depends(get_state) + @router.get("") -def pending_questions(session_id: str, state=Depends(get_state)): +def pending_questions(session_id: str, state=_GET_STATE_DEPENDENCY): return {"questions": QuestionStore(state).list(session_id)} +@router.post("/{question_id}/dismiss") +def dismiss_question(question_id: str, body: Dismiss, state=_GET_STATE_DEPENDENCY): + try: + return QuestionStore(state).dismiss(question_id, body.session_id) + except KeyError as exc: + raise HTTPException(404, "Question not found") from exc + + @router.post("/{question_id}/answer") -def answer_question(question_id: str, body: Answer, state=Depends(get_state)): +def answer_question(question_id: str, body: Answer, state=_GET_STATE_DEPENDENCY): try: return QuestionStore(state).answer(question_id, body.session_id, body.value) except KeyError as exc: raise HTTPException(404, "Question not found") from exc except CompetitionConflict as exc: raise HTTPException(409, str(exc)) from exc + except WorkflowSessionConflict as exc: + raise HTTPException(409, str(exc)) from exc except ValueError as exc: raise HTTPException(400, str(exc)) from exc diff --git a/backend/api/skills.py b/backend/api/skills.py index 00dc462..a1967b4 100644 --- a/backend/api/skills.py +++ b/backend/api/skills.py @@ -44,13 +44,16 @@ def _view(state): } + +_GET_STATE_DEPENDENCY = Depends(get_state) + @router.get("") -def list_skills(state=Depends(get_state)): +def list_skills(state=_GET_STATE_DEPENDENCY): return _view(state) @router.post("") -def install_skill(body: InstallSkill, state=Depends(get_state)): +def install_skill(body: InstallSkill, state=_GET_STATE_DEPENDENCY): try: state.skills.install(body.repository, body.revision) except ValueError as exc: @@ -61,7 +64,7 @@ def install_skill(body: InstallSkill, state=Depends(get_state)): @router.post("/add") -def add_skill(body: AddSkill, state=Depends(get_state)): +def add_skill(body: AddSkill, state=_GET_STATE_DEPENDENCY): """Install a skill package with the ``skills`` CLI.""" try: result = state.skills.install_with_npx(body.spec) @@ -72,8 +75,11 @@ def add_skill(body: AddSkill, state=Depends(get_state)): return {**_view(state), "output": result["output"], "spec": result["spec"]} + +_REQUIRED_UPLOAD_FILE = File(...) + @router.post("/upload") -async def upload_skill(file: UploadFile = File(...), state=Depends(get_state)): +async def upload_skill(file: UploadFile = _REQUIRED_UPLOAD_FILE, state=_GET_STATE_DEPENDENCY): """Accept an operator-provided SKILL.md into the Custom folder.""" if not file.filename: raise HTTPException(400, "a SKILL.md filename is required") @@ -95,7 +101,7 @@ async def upload_skill(file: UploadFile = File(...), state=Depends(get_state)): @router.post("/move") -def move_skill(body: MoveSkill, state=Depends(get_state)): +def move_skill(body: MoveSkill, state=_GET_STATE_DEPENDENCY): """Reassign a skill to another folder (drag-and-drop in the UI).""" try: state.skills.move(body.repository, body.path, body.folder) @@ -107,7 +113,7 @@ def move_skill(body: MoveSkill, state=Depends(get_state)): @router.patch("") -def enable_skill(body: EnableSkill, state=Depends(get_state)): +def enable_skill(body: EnableSkill, state=_GET_STATE_DEPENDENCY): try: state.skills.enable(body.repository, body.enabled) except KeyError as exc: diff --git a/backend/api/solve.py b/backend/api/solve.py index ad872cc..feac118 100644 --- a/backend/api/solve.py +++ b/backend/api/solve.py @@ -11,21 +11,24 @@ Report, ReportRequest, ) -from backend.core.state import AppState -from backend.core.redaction import redact_flag from backend.core.ipc import FlagConflictError, submit_flag_candidate from backend.core.postprocess_store import enqueue_postprocess +from backend.core.redaction import redact_flag +from backend.core.state import AppState router = APIRouter(tags=["solve"]) + +_GET_STATE_DEPENDENCY = Depends(get_state) + @router.post("/projects/{project_id}/start") -def start_solving(project_id: str, state: AppState = Depends(get_state)): +def start_solving(project_id: str, state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: row = graph_store.get_project_row(conn, project_id) if row is None: raise HTTPException(404, "Project not found") - if row["status"] == "solved": + if row["status"] in {"solved", "completed"}: raise HTTPException(409, "Project already solved") errors = state.config.startup_errors() if errors: @@ -39,23 +42,23 @@ def start_solving(project_id: str, state: AppState = Depends(get_state)): @router.post("/projects/{project_id}/reopen") -def reopen_project(project_id: str, state: AppState = Depends(get_state)): +def reopen_project(project_id: str, state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: row = graph_store.get_project_row(conn, project_id) if row is None: raise HTTPException(404, "Project not found") - if row["status"] == "solved": + if row["status"] in {"solved", "completed"}: raise HTTPException(409, "Solved projects are immutable; create a new project to retry") raise HTTPException(409, "Only terminal failed projects can be resumed") @router.post("/projects/{project_id}/stop") -def stop_solving(project_id: str, state: AppState = Depends(get_state)): +def stop_solving(project_id: str, state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: row = graph_store.get_project_row(conn, project_id) if row is None: raise HTTPException(404, "Project not found") - if row["status"] == "solved": + if row["status"] in {"solved", "completed"}: raise HTTPException(409, "Solved projects cannot be stopped") graph_store.set_status(conn, project_id, "stopped") conn.execute( @@ -71,12 +74,20 @@ def stop_solving(project_id: str, state: AppState = Depends(get_state)): return {"status": "stopped", "project_id": project_id} +@router.post("/projects/{project_id}/pause") +def pause_solving(project_id: str, state: AppState = _GET_STATE_DEPENDENCY): + result = stop_solving(project_id, state) + with state.db.connect() as conn: + graph_store.set_runtime_phase(conn, project_id, "paused") + return {**result, "phase": "paused"} + + @router.post("/projects/stop-all") -def stop_all(state: AppState = Depends(get_state)): +def stop_all(state: AppState = _GET_STATE_DEPENDENCY): stopped = [] with state.db.connect() as conn: rows = conn.execute( - "SELECT id FROM projects WHERE status NOT IN ('solved','stopped')" + "SELECT id FROM projects WHERE status NOT IN ('solved','completed','stopped')" ).fetchall() ids = [r["id"] for r in rows] for pid in ids: @@ -89,7 +100,7 @@ def stop_all(state: AppState = Depends(get_state)): @router.post("/projects/{project_id}/resume") -def resume_solving(project_id: str, state: AppState = Depends(get_state)): +def resume_solving(project_id: str, state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: row = graph_store.get_project_row(conn, project_id) if row is None: @@ -110,7 +121,7 @@ def resume_solving(project_id: str, state: AppState = Depends(get_state)): @router.post("/projects/{project_id}/reports", response_model=Report, status_code=201) -def submit_report(project_id: str, body: ReportRequest, state: AppState = Depends(get_state)): +def submit_report(project_id: str, body: ReportRequest, state: AppState = _GET_STATE_DEPENDENCY): """A Member submits a difficulty report to Diamond (Seed.md 角色联动).""" with state.db.connect() as conn: if graph_store.get_project_row(conn, project_id) is None: @@ -130,7 +141,7 @@ def submit_report(project_id: str, body: ReportRequest, state: AppState = Depend @router.post("/projects/{project_id}/complete", response_model=Intent) -def complete_project(project_id: str, body: CompleteRequest, state: AppState = Depends(get_state)): +def complete_project(project_id: str, body: CompleteRequest, state: AppState = _GET_STATE_DEPENDENCY): """Mark a flag found (IPC verification entry). Creates the goal edge.""" with state.db.connect() as conn: row = graph_store.get_project_row(conn, project_id) @@ -170,7 +181,7 @@ def complete_project(project_id: str, body: CompleteRequest, state: AppState = D @router.get("/broadcasts", response_model=list[Broadcast]) -def list_broadcasts(state: AppState = Depends(get_state)): +def list_broadcasts(state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: rows = graph_store.list_broadcasts(conn) return [item.model_copy(update={"flag": redact_flag(item.flag) or ""}) for item in rows] diff --git a/backend/api/wp.py b/backend/api/wp.py index fac8bfb..e72f8cc 100644 --- a/backend/api/wp.py +++ b/backend/api/wp.py @@ -20,7 +20,7 @@ def _completed_writeups(state: AppState) -> list[dict]: """ SELECT id, title, status, wp_path, updated_at, created_at FROM projects - WHERE status = 'solved' AND wp_path IS NOT NULL + WHERE status IN ('solved','completed') AND wp_path IS NOT NULL ORDER BY created_at, id """ ).fetchall() @@ -65,8 +65,11 @@ def _completed_writeups(state: AppState) -> list[dict]: return out + +_GET_STATE_DEPENDENCY = Depends(get_state) + @router.get("/projects/{project_id}/wp") -def get_wp(project_id: str, state: AppState = Depends(get_state)): +def get_wp(project_id: str, state: AppState = _GET_STATE_DEPENDENCY): with state.db.connect() as conn: row = graph_store.get_project_row(conn, project_id) if row is None: @@ -81,12 +84,12 @@ def get_wp(project_id: str, state: AppState = Depends(get_state)): @router.get("/wp/completed") -def completed_wp(state: AppState = Depends(get_state)): +def completed_wp(state: AppState = _GET_STATE_DEPENDENCY): return {"writeups": _completed_writeups(state)} @router.post("/wp/derive") -def derive_wp(state: AppState = Depends(get_state)): +def derive_wp(state: AppState = _GET_STATE_DEPENDENCY): target = state.wp_export_dir target.mkdir(parents=True, exist_ok=True) writeups = _completed_writeups(state) @@ -111,6 +114,6 @@ def derive_wp(state: AppState = Depends(get_state)): @router.get("/wp") -def list_wp(state: AppState = Depends(get_state)): +def list_wp(state: AppState = _GET_STATE_DEPENDENCY): files = sorted(state.wp_dir.glob("*.md")) return {"writeups": [f.name for f in files]} diff --git a/backend/auth/manager.py b/backend/auth/manager.py index 98375a4..92cebef 100644 --- a/backend/auth/manager.py +++ b/backend/auth/manager.py @@ -9,12 +9,14 @@ import threading import time from collections import OrderedDict +from collections.abc import Callable from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from typing import Any, Callable +from typing import Any from backend.auth.session_store import ActiveSessionStore +from backend.core.validation import InputValidationError AUTH_FILE_ENV = "IPC_AUTH_FILE" SESSION_COOKIE_NAME = "ipc_session" @@ -328,7 +330,7 @@ def _restrict_permissions(self) -> None: @staticmethod def _validate_new_password(password: str) -> None: if not isinstance(password, str): - raise ValueError("password must be a string") + raise InputValidationError("password must be a string") if len(password) < _MIN_PASSWORD_LENGTH: raise ValueError(f"password must contain at least {_MIN_PASSWORD_LENGTH} characters") if len(password) > _MAX_PASSWORD_LENGTH: diff --git a/backend/blackboard/db.py b/backend/blackboard/db.py index cfc2fb6..78c22b2 100644 --- a/backend/blackboard/db.py +++ b/backend/blackboard/db.py @@ -2,7 +2,6 @@ from backend.persistence.database import Database, PostgresDatabase - PROJECT_STATES = ( "created", "running", @@ -14,4 +13,4 @@ "stopped", ) -__all__ = ["Database", "PostgresDatabase", "PROJECT_STATES"] +__all__ = ["PROJECT_STATES", "Database", "PostgresDatabase"] diff --git a/backend/blackboard/edge_store.py b/backend/blackboard/edge_store.py index 308f19f..2736e19 100644 --- a/backend/blackboard/edge_store.py +++ b/backend/blackboard/edge_store.py @@ -8,7 +8,6 @@ from backend.blackboard.ids import next_intent_id, utcnow from backend.blackboard.models import Intent - _WS_RE = re.compile(r"\s+") _PUNCT_RE = re.compile(r"[^a-z0-9_./:-]+") diff --git a/backend/blackboard/graph_store.py b/backend/blackboard/graph_store.py index e873173..b918277 100644 --- a/backend/blackboard/graph_store.py +++ b/backend/blackboard/graph_store.py @@ -127,6 +127,7 @@ def project_meta(row: Any) -> ProjectMeta: external_id=row["external_id"], title=row["title"], category=row["category"], + completion_policy=row.get("completion_policy", "verified_flag"), status=row["status"], postprocess_status=row.get("postprocess_status", "not_started"), terminal_reason=row.get("terminal_reason"), @@ -155,6 +156,7 @@ def create_project( hints: list[tuple[str, str]] | None = None, external_id: str | None = None, platform: str | None = None, + pentest_setup=None, ) -> str: pid = next_project_id(conn) now = utcnow() @@ -179,6 +181,12 @@ def create_project( "INSERT INTO hints (id, project_id, content, creator, created_at) VALUES (%s, %s, %s, %s, %s)", (hid, pid, content, creator, now), ) + policy = "platform_flags" if platform else ("goals" if category == "pentest" else "verified_flag") + conn.execute("UPDATE projects SET completion_policy=%s WHERE id=%s", (policy, pid)) + if category == "pentest": + from backend.pentest.models import Setup + from backend.pentest.store import initialize + initialize(conn, pid, goal, origin, Setup.model_validate(pentest_setup or {})) add_agent(conn, pid, "ipc", "ipc", state="active") add_agent(conn, pid, "diamond", "diamond", state="idle") return pid diff --git a/backend/blackboard/models.py b/backend/blackboard/models.py index 5d30c07..3858600 100644 --- a/backend/blackboard/models.py +++ b/backend/blackboard/models.py @@ -9,6 +9,7 @@ "running", "flag_found", "solved", + "completed", "timeout", "infra_error", "failed", @@ -101,6 +102,7 @@ class ProjectMeta(BaseModel): external_id: str | None = None title: str category: str + completion_policy: str = "verified_flag" status: ProjectStatus postprocess_status: str = "not_started" terminal_reason: str | None = None @@ -157,6 +159,7 @@ class CreateProjectRequest(BaseModel): goal: str category: str = "misc" hints: list[CreateHintInline] | None = None + pentest: dict | None = None _v = field_validator("title", "origin", "goal")(lambda cls, v: _non_empty(v)) diff --git a/backend/blackboard/node_store.py b/backend/blackboard/node_store.py index 69d4c03..e04e697 100644 --- a/backend/blackboard/node_store.py +++ b/backend/blackboard/node_store.py @@ -6,7 +6,6 @@ from backend.blackboard.ids import next_fact_id, utcnow from backend.blackboard.models import Fact - _FINDING_RE = re.compile( r"(?P[A-Za-z0-9_./-]+\.(?:c|cc|cpp|cxx|h|hpp|py|js|ts|java))" r"(?::|\s+lines?\s+)(?P\d+(?:-\d+)?)" diff --git a/backend/competition/__init__.py b/backend/competition/__init__.py index d492435..e69de29 100644 --- a/backend/competition/__init__.py +++ b/backend/competition/__init__.py @@ -1 +0,0 @@ -"""Durable competition contracts and scheduling primitives.""" diff --git a/backend/competition/control_schema.py b/backend/competition/control_schema.py new file mode 100644 index 0000000..49831bc --- /dev/null +++ b/backend/competition/control_schema.py @@ -0,0 +1,18 @@ + +CONTROL_SCHEMA = ( + """CREATE TABLE IF NOT EXISTS competition_control_operations ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL REFERENCES competition_runs(id), + action TEXT NOT NULL CHECK(action IN ('pause','resume','stop','refresh')), + requested_revision BIGINT NOT NULL, + status TEXT NOT NULL DEFAULT 'queued' CHECK(status IN ('queued','running','completed','failed')), + owner TEXT, + epoch BIGINT NOT NULL DEFAULT 0, + lease_expires_at TIMESTAMPTZ, + error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finished_at TIMESTAMPTZ + )""", + """CREATE INDEX IF NOT EXISTS ix_competition_control_pending + ON competition_control_operations(created_at) WHERE status IN ('queued','running')""", +) diff --git a/backend/competition/conversation.py b/backend/competition/conversation.py index b3240fb..10490a6 100644 --- a/backend/competition/conversation.py +++ b/backend/competition/conversation.py @@ -1,24 +1,25 @@ -"""Native streamed tool conversations. Persist returned provider messages intact. - -The runtime owns replay and side-effect fencing; this adapter performs one -model turn and never silently retries after receiving part of a response. -""" from __future__ import annotations import json +import logging +import socket import threading +import time import uuid +from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any, Callable +from typing import Any import requests from backend.members.adapters import ( _anthropic_endpoint, + _is_reasoning_model, _openai_endpoint, opencode_provider_headers, ) +_recovery_logger = logging.getLogger(__name__) @dataclass class ToolCall: @@ -158,17 +159,24 @@ def __init__( request=requests.post, max_output_tokens: int = 32768, read_timeout: int = 300, + turn_timeout: float = 900, + progress_timeout: float = 300, provider_session_id: str | None = None, ) -> None: self.config = config self.request = request self.max_output_tokens = max_output_tokens - self.read_timeout = max(30, read_timeout) + self.read_timeout = max(.1, float(read_timeout)) + self.turn_timeout = max(.1, float(turn_timeout)) + self.progress_timeout = max(.1, float(progress_timeout)) self.surface = "anthropic" if config.api_format == "anthropic" else ( "responses" if config.api_surface == "responses" else "chat_completions" ) self._provider_session = provider_session_id or f"ipc-session-{uuid.uuid4().hex[:16]}" self._stream_usage_supported = True + self.on_first_response: Callable[[], None] | None = None + self.on_first_content: Callable[[], None] | None = None + self.on_progress: Callable[[], None] | None = None @property def provider_session_id(self) -> str: @@ -186,6 +194,7 @@ def turn(self, messages: list[dict], system: str, tools: list[dict], emit: Callable[[str], None], cancel: threading.Event, emit_reasoning: Callable[[str], None] | None = None) -> Turn: config = self.config + self._request_started_at = time.monotonic() if config.api_format == "mock": latest = "" for message in reversed(messages): @@ -209,6 +218,7 @@ def turn(self, messages: list[dict], system: str, tools: list[dict], elif self.surface == "responses": endpoint = _openai_endpoint(config.base_url, "responses") body.update(input=messages, instructions=system, store=False, + max_output_tokens=self.max_output_tokens, include=["reasoning.encrypted_content"], tools=[{"type": "function", **t, "strict": False} for t in tools]) if config.reasoning_effort not in {"auto", "none", "minimal"}: @@ -217,6 +227,8 @@ def turn(self, messages: list[dict], system: str, tools: list[dict], endpoint = _openai_endpoint(config.base_url, "chat_completions") body.update(messages=[{"role": "system", "content": system}, *messages], tools=[{"type": "function", "function": t} for t in tools]) + token_key = "max_completion_tokens" if config.api_format != "deepseek" and _is_reasoning_model(config.model) else "max_tokens" + body[token_key] = self.max_output_tokens if config.reasoning_effort not in {"auto", "none", "minimal"} and config.api_format != "deepseek": body["reasoning_effort"] = config.reasoning_effort if self._stream_usage_supported: @@ -227,7 +239,7 @@ def turn(self, messages: list[dict], system: str, tools: list[dict], try: with self.request( endpoint, headers=headers, json=body, stream=True, - timeout=(15, self.read_timeout), + timeout=(min(15, self.turn_timeout), min(self.read_timeout, self.turn_timeout, self.progress_timeout)), ) as response: response.raise_for_status() return self._read(response, emit, cancel, think) @@ -238,7 +250,7 @@ def turn(self, messages: list[dict], system: str, tools: list[dict], body.pop("stream_options", None) with self.request( endpoint, headers=headers, json=body, stream=True, - timeout=(15, self.read_timeout), + timeout=(min(15, self.turn_timeout), min(self.read_timeout, self.turn_timeout, self.progress_timeout)), ) as response: response.raise_for_status() return self._read(response, emit, cancel, think) @@ -254,10 +266,87 @@ def _should_drop_stream_options(exc: requests.HTTPError, body: dict) -> bool: try: detail = response.text or "" except Exception: + _recovery_logger.debug('Error handled in _should_drop_stream_options', exc_info=True) detail = "" return "stream_options" in detail or "include_usage" in detail def _read(self, response, emit, cancel, emit_reasoning=None): + finished = threading.Event() + started = getattr(self, "_request_started_at", time.monotonic()) + progress = [started] + interrupted = [] + + def abort(): + raw = getattr(response, "raw", None) + connection = getattr(raw, "_connection", None) + sock = getattr(connection, "sock", None) + if sock is None: + sock = getattr(getattr(getattr(getattr(raw, "_fp", None), "fp", None), "raw", None), "_sock", None) + if sock is not None: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + close = getattr(response, "close", None) + if callable(close): + close() + + def watch(): + while not finished.wait(.05): + now = time.monotonic() + reason = ("cancelled" if cancel.is_set() else "turn deadline" if now-started >= self.turn_timeout + else "no effective progress" if now-progress[0] >= self.progress_timeout else None) + if reason: + interrupted.append(reason) + try: + abort() + except Exception: + _recovery_logger.debug("stream abort failed", exc_info=True) + return + + first_content = [False] + last_noted = [started] + def progressed(value): + progress[0] = time.monotonic() + if not first_content[0]: + first_content[0] = True + if self.on_first_content is not None: + self.on_first_content() + if progress[0] - last_noted[0] >= 15: + last_noted[0] = progress[0] + if self.on_progress is not None: + self.on_progress() + return value + + def text(value): + if value: + progressed(value) + emit(value) + + def thinking(value): + if value: + progressed(value) + if emit_reasoning: + emit_reasoning(value) + + watcher = threading.Thread(target=watch, name="ipc-provider-deadline", daemon=True) + watcher.start() + try: + result = self._read_stream(response, text, cancel, thinking, progressed) + if interrupted: + raise TimeoutError(interrupted[0]) + return result + except Exception: + if cancel.is_set(): + raise InterruptedError("session cancelled") from None + if interrupted: + raise TimeoutError(f"provider exceeded {interrupted[0]}") from None + raise + finally: + finished.set() + watcher.join(timeout=1) + + def _read_stream(self, response, emit, cancel, emit_reasoning=None, progressed=None): think = emit_reasoning or (lambda _text: None) text = "" calls: dict[int, dict] = {} @@ -266,7 +355,12 @@ def _read(self, response, emit, cancel, emit_reasoning=None): complete = False truncated: str | None = None usage: dict | None = None + first_event = True for event in sse(response): + if first_event: + first_event = False + if self.on_first_response is not None: + self.on_first_response() if cancel.is_set(): raise InterruptedError("session cancelled") delta_text = "" @@ -294,6 +388,9 @@ def _read(self, response, emit, cancel, emit_reasoning=None): raise RuntimeError( "provider stream did not complete successfully" ) + elif (kind in {"response.function_call_arguments.delta", "response.output_item.added"} + and progressed and (event.get("delta") or (event.get("item") or {}).get("type") == "function_call")): + progressed(event) elif self.surface == "anthropic": index = event.get("index", 0) if kind == "content_block_start": @@ -307,6 +404,8 @@ def _read(self, response, emit, cancel, emit_reasoning=None): blocks[index]["text"] = blocks[index].get("text", "") + delta_text elif delta.get("type") == "input_json_delta": blocks[index]["_json"] += delta.get("partial_json", "") + if delta.get("partial_json") and progressed: + progressed(event) elif delta.get("type") == "thinking_delta": reasoning_text = delta.get("thinking", "") blocks[index]["thinking"] = blocks[index].get("thinking", "") + reasoning_text @@ -338,6 +437,8 @@ def _read(self, response, emit, cancel, emit_reasoning=None): if isinstance(value, str): reasoning_text += value for item in delta.get("tool_calls") or []: + if progressed: + progressed(item) item = item or {} entry = calls.setdefault(item.get("index", 0), {"id": "", "type": "function", "function": {"name": "", "arguments": ""}}) if item.get("id"): diff --git a/backend/competition/fixture.py b/backend/competition/fixture.py index ceef374..c3a8135 100644 --- a/backend/competition/fixture.py +++ b/backend/competition/fixture.py @@ -90,6 +90,7 @@ def start_instance(self, external_id: str) -> dict: "state": "ready", "generation": 1, "renewals": 0, + "entry": f"http://{external_id}.fixture.invalid/instance/1", } self._instances[external_id] = instance return deepcopy(instance) @@ -118,6 +119,7 @@ def rebuild_instance(self, external_id: str) -> dict: "generation": generation, "renewals": 0, "renew_after": 300, + "entry": f"http://{external_id}.fixture.invalid/instance/{generation}", } self._instances[external_id] = replacement self.rebuilds.append(external_id) diff --git a/backend/competition/platform.py b/backend/competition/platform.py index 974e318..16dd8f2 100644 --- a/backend/competition/platform.py +++ b/backend/competition/platform.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from urllib.parse import quote, urlsplit from backend.ops.models import PlatformWorkflowSpec @@ -7,6 +8,7 @@ from backend.platform.adapter import PlatformAdapter from backend.platform.verdict import interpret_response +_recovery_logger = logging.getLogger(__name__) class CompetitionPlatform: """Freeze one Workflow revision; refresh only its referenced credentials.""" @@ -23,9 +25,7 @@ def identity(self): base = str(adapter_identity).rstrip("/") team = quote(str(self.spec.team_id), safe="") team_marker = f"/team/{team}" - if base.endswith(f"/{team}") or base.endswith(team_marker) or ( - team_marker in base and "/account/" in base - ): + if base.endswith((f"/{team}", team_marker)) or team_marker in base and "/account/" in base: return base return f"{base}/{quote(str(self.spec.competition_id), safe='')}/{team}" base = self.spec.challenges.list_url @@ -58,6 +58,7 @@ def instance_limit(self) -> int | None: try: limit = method() except Exception: + _recovery_logger.debug('Error handled in instance_limit', exc_info=True) return None try: limit = int(limit) if limit is not None else None @@ -87,6 +88,10 @@ def challenges(self): result = native() if callable(native) else self.adapter.fetch_challenges() return result + def network_diagnostics(self): + method = getattr(self.adapter, "network_diagnostics", None) + return method() if callable(method) else {} + def submit(self, external_id: str, flag: str): native = getattr(self.adapter, "submit", None) if callable(native): diff --git a/backend/competition/policy.py b/backend/competition/policy.py index 6648a4a..e0391fb 100644 --- a/backend/competition/policy.py +++ b/backend/competition/policy.py @@ -1,8 +1,8 @@ from __future__ import annotations +from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime, timedelta -from collections.abc import Iterable from backend.core.config import MEMBER_NAMES @@ -10,6 +10,10 @@ WP_DURATION = timedelta(minutes=10) SYNC_INTERVAL_SECONDS = 120 +# Easy/medium work is cheaper per point, so it is drained before hard targets. +# Unknown labels sort last instead of masquerading as "easy". +_DIFFICULTY_RANK = {"easy": 0, "medium": 1, "hard": 2} + @dataclass(frozen=True) class Challenge: @@ -23,6 +27,7 @@ class Challenge: members: tuple[str, ...] = () state: str = "ready" solve_count: int | None = None + difficulty: str | None = None @dataclass(frozen=True) @@ -64,10 +69,13 @@ def plan_assignments( free = [name for name in member_order if name not in occupied] eligible = [c for c in challenges if c.state in {"ready", "solving"} and (c.deadline_at is None or c.deadline_at > now)] - running = sum(bool(c.members) for c in challenges) - available_local = max(0, local_capacity - running) + running_local = sum(bool(c.members) and not c.remote for c in challenges) + running_remote = sum(bool(c.members) and c.remote for c in challenges) + available_local = max(0, local_capacity - running_local) + available_remote = max(0, remote_capacity - running_remote) candidates = [c for c in eligible if not c.members] candidates.sort(key=lambda c: ( + _DIFFICULTY_RANK.get((c.difficulty or "").strip().lower(), 3), -(c.solve_count if c.solve_count is not None else -1), 0 if c.remote else 1, c.ready_at, @@ -76,21 +84,45 @@ def plan_assignments( result: list[Assignment] = [] assigned: set[str] = set() for challenge in candidates: - if not free or not available_local: + if not free: break needs_instance = challenge.remote and not challenge.instance_ready - if needs_instance and remote_capacity == 0: - continue + if challenge.remote: + if available_remote <= 0: + continue + elif available_local <= 0: + break result.append(Assignment(free.pop(0), challenge.id, "primary", needs_instance)) assigned.add(challenge.id) - available_local -= 1 - if needs_instance: - remote_capacity -= 1 - if len(eligible) < 10: - helpers = [c for c in eligible if len(c.members) == 1 and c.id not in assigned] - helpers.sort(key=lambda c: (c.first_assigned_at or c.ready_at, c.id)) - for challenge in helpers: + if challenge.remote: + available_remote -= 1 + else: + available_local -= 1 + # A free Member has nothing useful to start when the platform has no + # capacity left: at that point helping a running challenge is the only + # work available. Short runs keep the original whole-run behaviour. + startable_left = any( + c.id not in assigned + and not c.members + and ((c.remote and available_remote > 0) or (not c.remote and available_local > 0)) + for c in candidates + ) + helpers = [c for c in eligible if len(c.members) == 1 and c.id not in assigned] + helpers.sort(key=lambda c: (c.first_assigned_at or c.ready_at, c.id)) + pentest_helpers = [c for c in helpers if c.category == "pentest"] + other_helpers = [c for c in helpers if c.category != "pentest"] + helped: set[str] = set() + # A pentest board is designed for two solvers on distinct work items, so + # fill its second seat before ordinary helper work. + for challenge in pentest_helpers: + if not free: + break + result.append(Assignment(free.pop(0), challenge.id, "helper")) + helped.add(challenge.id) + if len(eligible) < 10 or not startable_left: + for challenge in other_helpers: if not free: break result.append(Assignment(free.pop(0), challenge.id, "helper")) + helped.add(challenge.id) return result diff --git a/backend/competition/proto/__init__.py b/backend/competition/proto/__init__.py index 20486f2..e69de29 100644 --- a/backend/competition/proto/__init__.py +++ b/backend/competition/proto/__init__.py @@ -1 +0,0 @@ -"""Generated Protobuf contracts for competition coordination.""" diff --git a/backend/competition/proto/pair_pb2.py b/backend/competition/proto/pair_pb2.py index a1b1424..e1d8bf3 100644 --- a/backend/competition/proto/pair_pb2.py +++ b/backend/competition/proto/pair_pb2.py @@ -1,10 +1,9 @@ -# -*- coding: utf-8 -*- -"""Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder + _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 6, diff --git a/backend/competition/runtime.py b/backend/competition/runtime.py index d75e420..b4ace57 100644 --- a/backend/competition/runtime.py +++ b/backend/competition/runtime.py @@ -1,14 +1,8 @@ -"""Compatibility shim: the durable loop now lives in :mod:`backend.agent.runtime`. - -``SessionRunner`` keeps its assignment-shaped constructor so existing -competition callers are unchanged, while the loop itself is the shared -:class:`~backend.agent.runtime.AgentRuntime`. Removed once the Member bridge -constructs the runtime directly. -""" from __future__ import annotations import threading -from typing import Any, Callable +from collections.abc import Callable +from typing import Any from backend.agent.runtime import ( AgentRuntime, @@ -36,7 +30,7 @@ def __init__( cancel: threading.Event | None = None, emit: Callable[[str], None] | None = None, compressor: Callable[[list[dict[str, Any]]], tuple[str, list[dict[str, Any]]]] | None = None, - before_turn: Callable[["SessionRunner"], list[dict[str, Any]]] | None = None, + before_turn: Callable[[SessionRunner], list[dict[str, Any]]] | None = None, context_bytes: int = 1_000_000, repeated_batch_limit: int = 8, truncation_retry_limit: int = 3, diff --git a/backend/competition/schema.py b/backend/competition/schema.py index 63da861..8d4f4dc 100644 --- a/backend/competition/schema.py +++ b/backend/competition/schema.py @@ -1,4 +1,3 @@ -"""Version 1 additive storage contract. Keep these statements migration-stable.""" COMPETITION_SCHEMA = ( """CREATE TABLE IF NOT EXISTS workflow_challenges ( @@ -15,6 +14,7 @@ status TEXT NOT NULL DEFAULT 'preflight' CHECK (status IN ('preflight','importing','running','paused','blocked','draining','finished','stopped')), config_snapshot JSONB NOT NULL, + limits JSONB NOT NULL DEFAULT '{}'::jsonb, revision BIGINT NOT NULL DEFAULT 1, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now() @@ -125,6 +125,7 @@ "ALTER TABLE competition_runs ADD COLUMN IF NOT EXISTS last_error TEXT", "ALTER TABLE competition_runs ADD COLUMN IF NOT EXISTS lease_owner TEXT", "ALTER TABLE competition_runs ADD COLUMN IF NOT EXISTS lease_expires_at TIMESTAMPTZ", + "ALTER TABLE competition_runs ADD COLUMN IF NOT EXISTS limits JSONB NOT NULL DEFAULT '{}'::jsonb", "ALTER TABLE competition_session_events ADD COLUMN IF NOT EXISTS event_id BIGSERIAL", "CREATE UNIQUE INDEX IF NOT EXISTS uq_competition_event_id ON competition_session_events(event_id)", """CREATE TABLE IF NOT EXISTS competition_run_events ( diff --git a/backend/competition/service.py b/backend/competition/service.py index 96d4fbc..483c949 100644 --- a/backend/competition/service.py +++ b/backend/competition/service.py @@ -1,24 +1,30 @@ from __future__ import annotations -import os -import json import hashlib import inspect +import json +import logging +import os import sys import threading import time +from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from contextlib import suppress from datetime import datetime, timedelta, timezone -from typing import Any, Callable +from typing import Any from psycopg.errors import UndefinedColumn, UndefinedTable - from psycopg.types.json import Jsonb +from backend.blackboard import edge_store, node_store from backend.competition.platform import CompetitionPlatform from backend.competition.policy import Challenge, plan_assignments -from backend.competition.store import SERVICE_MEMBER_ID, CompetitionConflict, CompetitionStore +from backend.competition.store import ( + SERVICE_MEMBER_ID, + CompetitionConflict, + CompetitionStore, +) from backend.competition.transport import Envelope, PairCoordinator, ReconPublisher from backend.core.ipc import accept_verified_flag from backend.core.wp_writer import ( @@ -30,6 +36,7 @@ from backend.ops.models import PlatformWorkflowSpec from backend.ops.service import OpsAgentService +_recovery_logger = logging.getLogger(__name__) TERMINAL_RUN_STATES = {"finished", "stopped"} @@ -130,7 +137,7 @@ def __init__( self._wp_pool = ThreadPoolExecutor(max_workers=2, thread_name_prefix="ipc-wp") self._wp_futures: dict[str, Any] = {} self._instance_capability_warnings: set[tuple[str, str]] = set() - self._published_instance_entries: set[tuple[str, str, str]] = set() + self._published_instance_entries: set[tuple[str, str, str, int]] = set() self._engine_conflicts: set[tuple[str, str]] = set() self._engine_epochs: dict[str, int] = {} self.observation_interval = 10.0 @@ -141,6 +148,18 @@ def __init__( self._undelivered_injected: set[int] = set() self._receiver_cursors: dict[str, int] = {} self._stale_job_seconds = 1800 + self._resource_capacity: dict[str, dict] = {} + self.phase_metrics: dict[str, dict] = {} + + def _phase(self, name: str, operation, *args, **kwargs): + started = time.monotonic() + try: + return operation(*args, **kwargs) + finally: + previous = self.phase_metrics.get(name, {}) + duration = round(time.monotonic()-started, 6) + self.phase_metrics[name] = {"last_seconds": duration, "max_seconds": max(duration, previous.get("max_seconds", 0)), + "calls": previous.get("calls", 0)+1} @property def ops(self) -> OpsAgentService: @@ -184,6 +203,7 @@ def start_worker(self) -> None: ) self._recon_endpoint = endpoint except Exception: + _recovery_logger.debug('Error handled in start_worker', exc_info=True) self._recon = None self._recon_endpoint = None @@ -215,6 +235,7 @@ def _supervise(self) -> None: try: self._start_loop_thread() except Exception as exc: + _recovery_logger.debug('Error handled in _supervise', exc_info=True) print( f"[ipc-competition] supervisor restart failed:" f" {type(exc).__name__}: {exc}", @@ -326,11 +347,31 @@ def preflight(self, workflow_id: str) -> dict: supports_instances = getattr(platform, "client", None) is not None elif callable(supports_instances): supports_instances = supports_instances() + preflight_remote_limit, remote_source = self._resolve_remote_limit( + workflow["spec"].model_dump(mode="json"), platform, + ) return { "workflow_id": workflow_id, "identity_key": platform.identity, + "network": platform.network_diagnostics() if callable(getattr(platform, "network_diagnostics", None)) else {}, "challenge_count": len(challenges), "remote_count": sum(bool(item.remote) for item in challenges), + "ordinary_count": sum(not bool(item.remote) for item in challenges), + "challenges": [ + { + "external_id": str(item.external_id), + "title": item.title, + "category": item.category, + "remote": bool(item.remote), + "solved": bool(item.solved), + } + for item in challenges + ], + "limits": { + "local_concurrency": self.state.config.limits.max_concurrent_tasks, + "remote_concurrency": preflight_remote_limit, + "source": remote_source, + }, "capabilities": { "submit": bool(supports_submit), "instances": bool(supports_instances), @@ -367,11 +408,20 @@ def start( raise CompetitionConflict("idempotency key belongs to another run request") return self.store.run_snapshot(existing["id"]) discovered = _select_discovered(platform.preflight(), select) + snapshot_limits = { + "max_concurrent_tasks": self.state.config.limits.max_concurrent_tasks, + "total_cpu": getattr(self.state.config.limits, "total_cpu", 0), + } + snapshot_limits["remote_concurrency"], snapshot_limits["remote_source"] = self._resolve_remote_limit( + workflow["spec"].model_dump(mode="json"), platform, + ) snapshot = { "workflow_spec": workflow["spec"].model_dump(mode="json"), "spec_digest": workflow["spec_digest"], "member_config": _safe_member_config(self.state.config.member), "member_config_source": "runtime-config", + "limits_snapshot": snapshot_limits, + "limits": {}, } if select is not None: snapshot["selected_external_ids"] = [ @@ -446,37 +496,91 @@ def _challenge_metadata(item) -> dict[str, Any]: } if solve_count is not None: metadata["solve_count"] = solve_count + difficulty = str(platform_data.get("difficulty") or "").strip().lower() + if difficulty: + metadata["difficulty"] = difficulty container_type = platform_data.get("container_type") if container_type: metadata["container_type"] = str(container_type) + try: + score = float(platform_data.get("score")) + except (TypeError, ValueError): + score = None + if score is not None and 0 <= score < float("inf"): + metadata["score"] = score return metadata - def control(self, run_id: str, action: str, revision: int) -> dict: + def control(self, run_id: str, action: str, revision: int, operation_id: str | None = None) -> dict: if not self._tick_lock.acquire(blocking=False): - raise CompetitionConflict("another competition operation is in progress") + if not callable(getattr(self.store, "queue_control", None)): + error = CompetitionConflict("another competition operation is in progress; retry after 20 seconds") + error.retry_after = 20 + raise error + operation = self.store.queue_control(run_id, action, revision, operation_id) + self.wake() + return {**self.store.run_snapshot(run_id), "control_operation": operation} try: - run = self.store.get_run(run_id) - if run["revision"] != revision: - raise CompetitionConflict("stale run revision") - if action == "refresh": - self._refresh_unlocked(run_id) - return self.store.run_snapshot(run_id) - target = {"pause": "paused", "resume": "running", "stop": "stopped"}.get(action) - if target is None: - raise ValueError("unsupported run action") - changed = self.store.transition_run( - run_id, target, revision=revision, - permit_active=action == "stop", - ) - if action == "stop": - self._teardown_stopped_run(run_id, run) - elif action == "pause": - self._stop_run_projects(run_id) - self.store.release_run_assignments(run_id) - self.store.append_run_event(run_id, f"run.{target}", {}) - if action == "resume": + if operation_id: + operation = self.store.queue_control(run_id, action, revision, operation_id) self.wake() - return self.store.run_snapshot(changed["id"]) + return {**self.store.run_snapshot(run_id), "control_operation": operation} + return self._control_unlocked(run_id, action, revision) + finally: + self._tick_lock.release() + + def _control_unlocked(self, run_id: str, action: str, revision: int) -> dict: + run = self.store.get_run(run_id) + if run["revision"] != revision: + raise CompetitionConflict("stale run revision") + if action == "refresh": + self._refresh_unlocked(run_id) + return self.store.run_snapshot(run_id) + target = {"pause": "paused", "resume": "running", "stop": "stopped"}.get(action) + if target is None: + raise ValueError("unsupported run action") + if run["status"] != target: + self.store.transition_run(run_id, target, revision=revision, permit_active=action == "stop") + if action == "stop": + self._teardown_stopped_run(run_id, run) + elif action == "pause": + self._stop_run_projects(run_id) + self.store.release_run_assignments(run_id) + self.store.append_run_event(run_id, f"run.{target}", {}) + if action == "resume": + self.wake() + return self.store.run_snapshot(run_id) + + def _process_controls(self) -> bool: + if not callable(getattr(self.store, "claim_control", None)): + return False + processed = False + for _ in range(10): + operation = self.store.claim_control(self.owner) + if operation is None: + break + error = None + try: + run = self.store.get_run(operation["run_id"]) + self._control_unlocked(run["id"], operation["action"], run["revision"]) + except Exception as exc: + _recovery_logger.debug("Control operation failed", exc_info=True) + error = f"{type(exc).__name__}: {exc}"[:2000] + if self.store.finish_control(operation, error=error): + self.store.append_run_event(operation["run_id"], "run.control_failed" if error else "run.control_completed", + {"operation_id": operation["id"], "action": operation["action"], "error": error}) + processed = True + return processed + + def update_limits(self, run_id: str, limits: dict, revision: int) -> dict: + if not self._tick_lock.acquire(timeout=20): + error = CompetitionConflict("another competition operation is in progress; retry after 20 seconds") + error.retry_after = 20 + raise error + try: + updated = self.store.update_limits(run_id, limits, revision=revision) + self.store.append_run_event(run_id, "run.limits_updated", {"limits": limits}) + self.wake() + return self.store.run_snapshot(updated["id"]) finally: self._tick_lock.release() @@ -502,6 +606,7 @@ def _teardown_stopped_run(self, run_id: str, run: dict) -> None: try: self._run_teardown_step(step) except Exception as exc: + _recovery_logger.debug('Error handled in _teardown_stopped_run', exc_info=True) self.state.logger.project( "competition_teardown_step_failed", run_id, step=label, error=f"{type(exc).__name__}: {exc}", @@ -528,6 +633,7 @@ def _run_teardown_step(step, *, attempts: int = 3, base_delay: float = 0.2) -> N step() return except Exception as exc: + _recovery_logger.debug('Error handled in _run_teardown_step', exc_info=True) last = exc if attempt >= attempts: break @@ -582,6 +688,7 @@ def _sweep_stale_jobs(self, run: dict) -> None: jobs = AgentJobStore(self.state.db) swept = jobs.sweep_stale(idle_seconds=self._stale_job_seconds) except Exception as exc: + _recovery_logger.debug('Error handled in _sweep_stale_jobs', exc_info=True) self.state.logger.project( "competition_job_sweep_failed", run["id"], error=f"{type(exc).__name__}: {exc}", @@ -606,6 +713,7 @@ def _claim_project_engine(self, run: dict, project_id: str) -> bool: except (UndefinedColumn, UndefinedTable): return True except Exception as exc: + _recovery_logger.debug('Error handled in _claim_project_engine', exc_info=True) self.store.append_run_event( run["id"], "engine.lease_error", {"project_id": project_id, "error": str(exc)}, @@ -643,6 +751,7 @@ def _renew_project_engines(self, run: dict) -> None: epoch=epoch, seconds=60 ) except Exception: + _recovery_logger.debug('Error handled in _renew_project_engines', exc_info=True) renewed = True if not renewed: self._claim_project_engine(run, project_id) @@ -674,9 +783,11 @@ def _stop_owned_instances(self, run: dict) -> None: def _stop_instance(self, run: dict, challenge: dict, *, platform=None) -> bool: platform = platform or self._platform_for_run(run) + self.store.append_run_event(run["id"], "instance.stop_requested", {"challenge_id": challenge["id"]}) try: platform.stop_instance(challenge["external_id"]) except Exception as exc: + _recovery_logger.debug('Error handled in _stop_instance', exc_info=True) with self.state.db.connect() as connection: connection.execute( """UPDATE competition_instances @@ -695,6 +806,7 @@ def _stop_instance(self, run: dict, challenge: dict, *, platform=None) -> bool: WHERE challenge_id=%s""", (challenge["id"],), ) + self.store.append_run_event(run["id"], "instance.stopped", {"challenge_id": challenge["id"]}) return True def _retry_pending_instance_stops(self) -> None: @@ -715,6 +827,7 @@ def _retry_pending_instance_stops(self) -> None: {"id": row["challenge_id"], "external_id": row["external_id"]}, ) except Exception as exc: + _recovery_logger.debug('Error handled in _retry_pending_instance_stops', exc_info=True) self.store.append_run_event( row["run_id"], "instance.stop_retry_failed", {"challenge_id": row["challenge_id"], "error": str(exc)}, @@ -800,7 +913,7 @@ def retry_wp(self, job_id: str) -> dict: with self.state.db.connect() as connection: row = connection.execute( """UPDATE competition_wp_jobs SET status='pending',next_attempt_at=now(),last_error=NULL - WHERE challenge_id=%s AND status IN ('deferred','failed') RETURNING *""", + WHERE challenge_id=%s AND (status='deferred' OR (status='failed' AND attempts<5)) RETURNING *""", (job_id,), ).fetchone() if row is None: @@ -813,9 +926,11 @@ def _run_loop(self) -> None: try: self.tick() except BaseException as exc: + _recovery_logger.debug('Error handled in _run_loop', exc_info=True) try: run = self.store.active_run() except Exception: + _recovery_logger.debug('Error handled in _run_loop', exc_info=True) run = None if run: with suppress(Exception): @@ -838,10 +953,32 @@ def tick(self) -> None: if not self._tick_lock.acquire(blocking=False): return try: + self._process_controls() self._retry_pending_instance_stops() run = self.store.active_run() if run is None or not self.store.claim_run_lease(run["id"], self.owner): return + if run["status"] == "blocked": + due = _timestamp(run.get("next_sync_at")) + if due is None or due <= datetime.now(timezone.utc): + try: + self._platform_for_run(run).preflight() + current = self.store.get_run(run["id"]) + resumed = self.store.transition_run( + run["id"], "running", revision=current["revision"] + ) + self.store.append_run_event( + run["id"], "run.recovered", + {"reason": "startup lease takeover and preflight"}, + ) + run = resumed + except Exception as exc: + _recovery_logger.debug('Error handled in tick', exc_info=True) + self.store.schedule_next_sync(run["id"], seconds=self.sync_interval, error=str(exc)) + self.store.append_run_event( + run["id"], "run.recovery_failed", {"error": str(exc)} + ) + return if run["status"] in {"running", "paused", "draining"}: self.store.heartbeat_run_assignments(run["id"], self.owner) if run["status"] in {"running", "paused"}: @@ -850,6 +987,7 @@ def tick(self) -> None: try: self._refresh_unlocked(run["id"]) except Exception as exc: + _recovery_logger.debug('Error handled in tick', exc_info=True) self.store.schedule_next_sync( run["id"], seconds=self.sync_interval, error=str(exc) ) @@ -878,11 +1016,16 @@ def tick(self) -> None: self._check_message_delivery(run) self._sweep_stale_jobs(run) self._renew_project_engines(run) + if self._process_controls(): + return if run["status"] == "running": self._sync_orchestrator_assignments(run) - self._renew_instances(run) + self._phase("instance_renewal", self._renew_instances, run) + self._phase("instance_repair", self._repair_assigned_instances, run) self._publish_instance_entries(run) - self._process_submissions( + if self._process_controls(): + return + self._phase("submissions", self._process_submissions, run, allow_new=run["status"] == "running", ) @@ -913,6 +1056,7 @@ def _report_stalled_assignments(self, run: dict) -> None: try: stalled = self.store.stalled_run_assignments(run["id"]) except Exception as exc: + _recovery_logger.debug('Error handled in _report_stalled_assignments', exc_info=True) self.state.logger.project( "competition_stall_probe_failed", run["id"], error=f"{type(exc).__name__}: {exc}", @@ -961,6 +1105,7 @@ def _check_message_delivery(self, run: dict) -> None: run["id"], older_than_seconds=self._message_delivery_window ) except Exception as exc: + _recovery_logger.debug('Error handled in _check_message_delivery', exc_info=True) self.state.logger.project( "competition_message_probe_failed", run["id"], error=f"{type(exc).__name__}: {exc}", @@ -984,6 +1129,7 @@ def _cursor_or_zero(self, session_id: str) -> int: try: return self.store.message_cursor(session_id) except Exception: + _recovery_logger.debug('Error handled in _cursor_or_zero', exc_info=True) return 0 def _note_cursor(self, run: dict, session_id: str) -> None: @@ -991,6 +1137,7 @@ def _note_cursor(self, run: dict, session_id: str) -> None: try: current = self.store.message_cursor(session_id) except Exception: + _recovery_logger.debug('Error handled in _note_cursor', exc_info=True) return previous = self._receiver_cursors.get(session_id, 0) if current > previous: @@ -1060,6 +1207,7 @@ def _degrade_recon_to_blackboard(self, run: dict, message: dict) -> bool: if not recorded: return True except Exception as exc: + _recovery_logger.debug('Error handled in _degrade_recon_to_blackboard', exc_info=True) self.state.logger.project( "competition_message_degrade_failed", run["id"], error=f"{type(exc).__name__}: {exc}", challenge_id=str(challenge_id), @@ -1086,6 +1234,7 @@ def _run_session_watchdog(self, run: dict) -> None: try: rows = self.store.run_session_health(run["id"]) except Exception as exc: + _recovery_logger.debug('Error handled in _run_session_watchdog', exc_info=True) self.state.logger.project( "competition_health_probe_failed", run["id"], error=f"{type(exc).__name__}: {exc}", @@ -1135,6 +1284,7 @@ def _cancel_stalled_member(self, run: dict, row: dict) -> None: try: stop(member) except Exception as exc: + _recovery_logger.debug('Error handled in _cancel_stalled_member', exc_info=True) self.state.logger.project( "competition_turn_cancel_failed", run["id"], member=member, error=f"{type(exc).__name__}: {exc}", @@ -1235,6 +1385,7 @@ def _restart_seat_project(self, run: dict, assignment: dict) -> None: (assignment["challenge_id"],), ).fetchone() except Exception: + _recovery_logger.debug('Error handled in _restart_seat_project', exc_info=True) return project_id = challenge["project_id"] if challenge else None if not project_id: @@ -1242,6 +1393,7 @@ def _restart_seat_project(self, run: dict, assignment: dict) -> None: try: start(project_id) except Exception as exc: + _recovery_logger.debug('Error handled in _restart_seat_project', exc_info=True) self.store.append_run_event( run["id"], "session.recovery_start_failed", { @@ -1287,7 +1439,7 @@ def _idle_member(self, run: dict) -> str | None: ).fetchall() taken.update(row["member"] for row in rows) except Exception: - pass + _recovery_logger.debug('Error handled in _idle_member', exc_info=True) for name in MEMBER_NAMES: if name not in taken: return name @@ -1312,35 +1464,55 @@ def _record_observation(self, run: dict) -> None: if now - self._last_observation_at < self.observation_interval: return try: - self.store.observe_run(run["id"], self.owner) + self.store.observe_run( + run["id"], self.owner, resource_capacity=self._resource_capacity.get(run["id"]), + ) self._last_observation_at = now except Exception as exc: + _recovery_logger.debug('Error handled in _record_observation', exc_info=True) self.store.append_run_event( run["id"], "observation.failed", {"error": str(exc)} ) def _stop_expired_challenges(self, run: dict) -> None: - expired_ids = { - item["id"] for item in self.store.challenges(run["id"]) - if item["state"] == "expired" - } - if not expired_ids: - return orchestrator = self.state.orchestrator platform = self._platform_for_run(run) for challenge in self.store.challenges(run["id"]): - if challenge["id"] not in expired_ids: + if challenge["state"] != "expired": continue - if challenge.get("project_id"): - if orchestrator is not None: - with suppress(Exception): + with self.state.db.connect() as connection: + claimed = connection.execute( + """UPDATE competition_challenges SET metadata=metadata || %s + WHERE id=%s AND state='expired' + AND COALESCE(metadata->>'expiry_cleanup_state','')<>'done' + AND (metadata->>'expiry_cleanup_after' IS NULL + OR (metadata->>'expiry_cleanup_after')::timestamptz<=now()) + RETURNING id""", + (Jsonb({"expiry_cleanup_state": "running", "expiry_cleanup_after": ( + datetime.now(timezone.utc) + timedelta(seconds=60) + ).isoformat()}), challenge["id"]), + ).fetchone() + if claimed is None: + continue + try: + if challenge.get("project_id"): + if orchestrator is not None: orchestrator.stop_project(challenge["project_id"]) - self._release_project_engine(challenge["project_id"]) - if self._instance_ready(challenge["id"]): - self._stop_instance(run, challenge, platform=platform) - self.store.append_run_event( - run["id"], "challenge.expired", {"challenge_id": challenge["id"]} - ) + self._release_project_engine(challenge["project_id"]) + if self._instance_ready(challenge["id"]): + self._stop_instance(run, challenge, platform=platform) + except Exception as exc: + _recovery_logger.debug("Challenge cleanup failed", exc_info=True) + self.store.append_run_event( + run["id"], "challenge.cleanup_failed", + {"challenge_id": challenge["id"], "error": str(exc), "retry_after": 60}, + ) + continue + with self.state.db.connect() as connection: + connection.execute( + "UPDATE competition_challenges SET metadata=metadata || %s WHERE id=%s", + (Jsonb({"expiry_cleanup_state": "done"}), challenge["id"]), + ) def _release_external_solved(self, run: dict, challenge: dict) -> None: """Release local resources for a challenge solved outside this run.""" @@ -1378,7 +1550,7 @@ def _advance_run_completion(self, run: dict) -> None: with self.state.db.connect() as connection: outstanding = connection.execute( """SELECT 1 FROM competition_wp_jobs - WHERE run_id=%s AND status<>'done' LIMIT 1""", + WHERE run_id=%s AND status NOT IN ('done','failed') LIMIT 1""", (run["id"],), ).fetchone() occupied = connection.execute( @@ -1443,6 +1615,7 @@ def _sync_orchestrator_assignments(self, run: dict) -> None: try: orchestrator.start_project_async(project_id) except Exception as exc: + _recovery_logger.debug('Error handled in _sync_orchestrator_assignments', exc_info=True) self._starting_members = { key for key in self._starting_members if key[0] != project_id @@ -1510,15 +1683,10 @@ def _schedule_projects(self, run: dict) -> None: members=tuple(members_by_challenge.get(row["id"], [])), state=row["state"], solve_count=solve_count, + difficulty=(str(metadata.get("difficulty") or "").strip().lower() or None), )) remote_limit = self._remote_instance_limit(run, platform) with self.state.db.connect() as connection: - used_rows = connection.execute( - """SELECT count(*) AS count FROM competition_instances - WHERE run_id=%s AND state NOT IN ('stopped','expired')""", - (run["id"],), - ).fetchone() - local_used = int(used_rows["count"]) local_ids = { str(row["external_id"]) for row in connection.execute( @@ -1542,6 +1710,7 @@ def _schedule_projects(self, run: dict) -> None: if instance_active and external_id not in local_ids: external_used += 1 except Exception as exc: + _recovery_logger.debug('Error handled in _schedule_projects', exc_info=True) external_used = remote_limit self.store.append_run_event( run["id"], "instance.quota_unknown", {"error": str(exc)} @@ -1549,11 +1718,42 @@ def _schedule_projects(self, run: dict) -> None: plans = plan_assignments( challenges, occupied_members={item["member"] for item in active}, - remote_capacity=max(0, remote_limit - local_used - external_used), + remote_capacity=max(0, remote_limit - external_used), now=datetime.now(timezone.utc), - local_capacity=self.state.config.limits.max_concurrent_tasks, + local_capacity=self._local_task_limit(run), available_members={member.name for member in self.state.config.available_members()}, ) + planned_ids = {plan.challenge_id for plan in plans} + local_limit = self._local_task_limit(run) + local_used = sum( + bool(item.members) and not item.remote for item in challenges + ) + remote_used = sum( + bool(item.members) and item.remote for item in challenges + ) + _, remote_source = self._resolve_remote_limit( + run["config_snapshot"]["workflow_spec"], platform, run.get("limits"), + ) + self._resource_capacity[run["id"]] = { + "local": {"limit": local_limit, "source": "run" if "max_concurrent_tasks" in (run.get("limits") or {}) else "config"}, + "remote": {"limit": remote_limit, "source": remote_source, "external_used": external_used}, + } + for item in challenges: + if item.id in planned_ids or item.members or item.state not in {"ready", "solving"}: + continue + if item.remote and remote_used >= remote_limit: + reason = "waiting for remote container capacity" + elif not item.remote and local_used >= local_limit: + reason = "waiting for local task capacity" + elif not self.state.config.available_members(): + reason = "waiting for configured Member seat" + else: + reason = "waiting for scheduler tick" + with self.state.db.connect() as connection: + connection.execute( + "UPDATE competition_challenges SET metadata=metadata || %s WHERE id=%s", + (Jsonb({"queue_reason": reason}), item.id), + ) by_id = {item["id"]: item for item in rows} for plan in plans: challenge = by_id[plan.challenge_id] @@ -1586,28 +1786,49 @@ def _schedule_projects(self, run: dict) -> None: continue if self.store.find_assignment(run["id"], challenge["id"], plan.member) is None: try: - self.store.assign( + assignment = self.store.assign( run["id"], challenge["id"], plan.member, self.owner, role="primary", ) except CompetitionConflict: continue + self.store.append_run_event( + run["id"], "assignment.started", + { + "challenge_id": challenge["id"], + "member": plan.member, + "role": "primary", + "session_id": assignment["session_id"], + }, + ) elif plan.role == "helper": if self.store.find_assignment(run["id"], challenge["id"], plan.member) is None: try: - self.store.assign( + assignment = self.store.assign( run["id"], challenge["id"], plan.member, self.owner, role="helper", ) except CompetitionConflict: continue + self.store.append_run_event( + run["id"], "assignment.started", + { + "challenge_id": challenge["id"], + "member": plan.member, + "role": "helper", + "session_id": assignment["session_id"], + }, + ) self._ensure_helper_intent(run, challenge, project_id, plan.member) + if (challenge.get("metadata") or {}).get("remote"): + self._publish_instance_entries(run) try: self._starting_members.add(start_key) if project_id not in self._starting_projects: self._starting_projects.add(project_id) orchestrator.start_project_async(project_id) except Exception as exc: + _recovery_logger.debug('Error handled in _schedule_projects', exc_info=True) self._starting_members = { key for key in self._starting_members if key[0] != project_id } @@ -1617,11 +1838,14 @@ def _schedule_projects(self, run: dict) -> None: {"challenge_id": challenge["id"], "member": plan.member, "error": str(exc)}, ) continue + if (challenge.get("metadata") or {}).get("remote"): + self._publish_instance_entries(run) self._starting_members.add(start_key) self._starting_projects.add(project_id) try: orchestrator.start_project_async(project_id) except Exception as exc: + _recovery_logger.debug('Error handled in _schedule_projects', exc_info=True) self._starting_members = { key for key in self._starting_members if key[0] != project_id } @@ -1680,10 +1904,14 @@ def _ensure_helper_intent( start_node = latest.node_id if not direction: direction = ( - "Cross-check the primary solver's findings on this challenge: " - "read the blackboard facts and the latest report first, then " - "independently verify the current hypothesis and explore a " - "different approach instead of repeating the main line." + "Helper contract: the primary owns the main exploit/solver. " + "Read the blackboard and /workspace/shared handoffs first. " + "Own one complementary subtask: verify a claimed property or test a distinct fallback hypothesis. " + "Do not independently redo the full primary analysis. Deliver concrete evidence and a runnable " + "artifact under /workspace/shared/// with its SHA-256 and run command. " + "Use publish_fact for verified milestones and send_message to notify the primary of the " + "handoff or a blocker; reply_to identifies the request being addressed. " + "Finish when that subtask is verified or its failure is evidenced." ) existing = edge_store.find_similar_open_intent( connection, project_id, [start_node], direction @@ -1708,6 +1936,7 @@ def _ensure_helper_intent( connection, project_id, member, f"intent:{intent_id}", "explore" ) except Exception as exc: + _recovery_logger.debug('Error handled in _ensure_helper_intent', exc_info=True) self.store.append_run_event( run["id"], "assignment.helper_intent_failed", { @@ -1761,35 +1990,37 @@ def _restore_paused_seats(self, run: dict, rows: list[dict]) -> None: {"challenge_id": challenge["id"], "member": member, "role": seat["role"]}, ) - def _remote_instance_limit(self, run: dict, platform: Any) -> int: - """How many containers this run may hold at once. - - The platform wins when it publishes a per-team limit, because that is - the number actually enforced; the workflow's ``remote_instance_limit`` - is the manual override for platforms that stay silent. With neither, - one container is assumed and the decision is recorded, which is safer - than unbounded allocation and more useful than refusing to start a - container challenge at all. - """ - - configured = run["config_snapshot"]["workflow_spec"].get("remote_instance_limit", 0) - try: - configured = max(0, int(configured)) - except (TypeError, ValueError): - configured = 0 + @staticmethod + def _resolve_remote_limit(configured_spec: dict, platform: Any, run_limits=None) -> tuple[int, str]: + """Share the same precedence between preflight, run snapshots and ticks.""" + run_limits = run_limits or {} + if isinstance(run_limits, str): + try: + run_limits = json.loads(run_limits) + except (TypeError, ValueError): + run_limits = {} + override = run_limits.get("remote_concurrency") if isinstance(run_limits, dict) else None + configured = override if override is not None else configured_spec.get("remote_concurrency", 0) + if override is not None or configured: + return max(0, int(configured)), "run" if override is not None else "workflow.remote_concurrency" reported = None reader = getattr(platform, "instance_limit", None) if callable(reader): try: reported = reader() except Exception: + _recovery_logger.debug('Error handled in _resolve_remote_limit', exc_info=True) reported = None - if reported: - return int(reported) - if configured: - return configured - if not self._supports_instance_lifecycle(platform): - return 0 + if reported is not None: + return max(0, int(reported)), "platform.reported" + if configured_spec.get("remote_instance_limit"): + return int(configured_spec["remote_instance_limit"]), "workflow.remote_instance_limit" + return (1, "platform.default") if CompetitionService._supports_instance_lifecycle(platform) else (0, "platform.unsupported") + + def _remote_instance_limit(self, run: dict, platform: Any) -> int: + limit, source = self._resolve_remote_limit(run["config_snapshot"]["workflow_spec"], platform, run.get("limits")) + if source != "platform.default": + return limit warning_key = (run["id"], "instance_limit_default") if warning_key not in self._instance_capability_warnings: self._instance_capability_warnings.add(warning_key) @@ -1801,7 +2032,23 @@ def _remote_instance_limit(self, run: dict, platform: Any) -> int: "the workflow sets no remote_instance_limit", }, ) - return 1 + return limit + + def _local_task_limit(self, run: dict) -> int: + """Read the mutable local capacity on every scheduler tick.""" + limits = run.get("limits") or {} + if isinstance(limits, str): + try: + limits = json.loads(limits) + except (TypeError, ValueError): + limits = {} + value = limits.get("max_concurrent_tasks") if isinstance(limits, dict) else None + if value is None: + value = self.state.config.limits.max_concurrent_tasks + try: + return max(0, int(value)) + except (TypeError, ValueError): + return max(0, int(self.state.config.limits.max_concurrent_tasks)) def _instance_ready(self, challenge_id: str) -> bool: with self.state.db.connect() as connection: @@ -1847,6 +2094,9 @@ def _instance_next_renewal(value: Any, *, default_seconds: int = 300) -> datetim expiry = None if expiry is not None: remaining = max(5, int((expiry - now).total_seconds())) + margin = value.get("renew_before_seconds") + if isinstance(margin, (int, float)) and margin > 0: + return now + timedelta(seconds=max(5, remaining - margin)) return now + timedelta( seconds=max(5, min(3600, remaining // 2)) ) @@ -1863,8 +2113,9 @@ def _publish_instance_entries(self, run: dict) -> None: """ with self.state.db.connect() as connection: instances = connection.execute( - """SELECT challenge_id, external_id, metadata FROM competition_instances - WHERE run_id=%s AND state IN ('ready','running','active')""", + """SELECT i.challenge_id,i.external_id,i.metadata,c.instance_generation + FROM competition_instances i JOIN competition_challenges c ON c.id=i.challenge_id + WHERE i.run_id=%s AND i.state IN ('ready','running','active')""", (run["id"],), ).fetchall() if not instances: @@ -1880,7 +2131,8 @@ def _publish_instance_entries(self, run: dict) -> None: continue if assignment["role"] == "wp": continue - key = (instance["challenge_id"], assignment["session_id"], str(entry)) + generation = int(instance["instance_generation"]) + key = (instance["challenge_id"], assignment["session_id"], str(entry), generation) if key in self._published_instance_entries: continue message = Envelope( @@ -1890,7 +2142,11 @@ def _publish_instance_entries(self, run: dict) -> None: receiver_member_id=assignment["member"], sender_session_id=assignment["session_id"], receiver_session_id=assignment["session_id"], - request_id=f"target-instance:{instance['challenge_id']}:{assignment['session_id']}:{entry}"[:128], + request_id=( + f"target-instance:{instance['challenge_id']}:{assignment['session_id']}" + f":g{generation}:e{int(assignment['epoch'])}:" + f"{hashlib.sha256(str(entry).encode()).hexdigest()[:12]}" + ), correlation_id=f"target-instance:{instance['challenge_id']}", lease_epoch=int(assignment["epoch"]), created_at=now, @@ -1900,6 +2156,7 @@ def _publish_instance_entries(self, run: dict) -> None: "kind": "target_instance", "external_id": instance["external_id"], "entry": str(entry), + "instance_generation": generation, "instruction": ( "This challenge's live target is the per-team instance at this entry. " "Attack only this address. Never probe, scan, or submit to the " @@ -1911,11 +2168,28 @@ def _publish_instance_entries(self, run: dict) -> None: self.publish_recon(message) self._published_instance_entries.add(key) except Exception as exc: + _recovery_logger.debug('Error handled in _publish_instance_entries', exc_info=True) self.store.append_run_event( run["id"], "instance.entry_publish_failed", {"challenge_id": instance["challenge_id"], "error": str(exc)}, ) + def _repair_assigned_instances(self, run: dict) -> None: + """Repair an expired target that still belongs to an admitted solver.""" + assigned = {row["challenge_id"] for row in self.store.assignments(run["id"], active_only=True) if row["role"] != "wp"} + with self.state.db.connect() as connection: + expired = {row["challenge_id"] for row in connection.execute( + """SELECT challenge_id FROM competition_instances WHERE run_id=%s AND state='expired' + AND metadata->>'repair_required'='true'""", (run["id"],), + ).fetchall()} + for challenge in self.store.challenges(run["id"]): + if ( + challenge["id"] in assigned & expired + and challenge["state"] == "solving" + and self._start_instance(run, challenge) + ): + self.store.append_run_event(run["id"], "instance.repaired", {"challenge_id": challenge["id"]}) + def _renew_instances(self, run: dict) -> None: platform = self._platform_for_run(run) with self.state.db.connect() as connection: @@ -1926,68 +2200,44 @@ def _renew_instances(self, run: dict) -> None: (run["id"],), ).fetchall() for instance in due: + self.store.append_run_event(run["id"], "instance.renew_requested", {"challenge_id": instance["challenge_id"]}) try: renewed = platform.renew_instance(instance["external_id"]) except Exception as exc: - rebuilt = getattr(platform, "rebuild_instance", None) - if callable(rebuilt): - try: - replacement = rebuilt(instance["external_id"]) - except Exception as rebuild_exc: - replacement = None - self.store.append_run_event( - run["id"], "instance.rebuild_failed", - { - "challenge_id": instance["challenge_id"], - "error": str(rebuild_exc), - }, - ) - if replacement is not None: - if not isinstance(replacement, dict): - replacement = {"state": "ready"} - with self.state.db.connect() as connection: - connection.execute( - """UPDATE competition_instances - SET state='ready',metadata=%s,next_renew_at=%s,updated_at=now() - WHERE challenge_id=%s""", - ( - Jsonb(replacement), - self._instance_next_renewal(replacement), - instance["challenge_id"], - ), - ) - connection.execute( - """UPDATE competition_challenges - SET instance_generation=instance_generation+1 - WHERE id=%s""", - (instance["challenge_id"],), - ) - self.store.append_run_event( - run["id"], "instance.rebuilt", - { - "challenge_id": instance["challenge_id"], - "external_id": instance["external_id"], - }, + _recovery_logger.debug('Error handled in _renew_instances', exc_info=True) + message = str(exc).casefold() + reason = getattr(exc, "reason", "") + if reason in {"expired", "missing"} or (not reason and any( + token in message for token in ("not created", "does not exist", "expired", "not found") + )): + with self.state.db.connect() as connection: + connection.execute( + """UPDATE competition_instances SET state='expired',next_renew_at=NULL, + metadata=metadata || %s,updated_at=now() WHERE challenge_id=%s""", + (Jsonb({"renew_error": str(exc), "repair_required": True, + "repair_after": (datetime.now(timezone.utc) + timedelta(seconds=60)).isoformat()}), instance["challenge_id"]), ) - logger = getattr(self.state, "logger", None) - log_project = getattr(logger, "project", None) - if callable(log_project): - log_project( - "instance.rebuilt", - str(instance["challenge_id"]), - run_id=run["id"], - external_id=instance["external_id"], - ) - continue + self.store.append_run_event( + run["id"], "instance.expired", + {"challenge_id": instance["challenge_id"], "reason": reason or "unavailable", + "renew_error": str(exc), "external_id": instance["external_id"]}, + ) + continue + try: + retry_after = max(5, min(3600, int(getattr(exc, "retry_after", 60)))) + except (TypeError, ValueError): + retry_after = 60 with self.state.db.connect() as connection: connection.execute( - """UPDATE competition_instances SET next_renew_at=now()+interval '60 seconds', - updated_at=now() WHERE challenge_id=%s""", - (instance["challenge_id"],), + """UPDATE competition_instances SET next_renew_at=now()+(%s*interval '1 second'), + metadata=metadata || %s,updated_at=now() WHERE challenge_id=%s""", + (retry_after, Jsonb({"renew_error": str(exc), "renew_error_kind": reason or "transient"}), + instance["challenge_id"]), ) self.store.append_run_event( run["id"], "instance.renew_failed", - {"challenge_id": instance["challenge_id"], "error": str(exc)}, + {"challenge_id": instance["challenge_id"], "external_id": instance["external_id"], + "error": str(exc), "reason": reason or "transient", "retry_after": retry_after}, ) continue next_renewal = self._instance_next_renewal(renewed) @@ -2004,6 +2254,7 @@ def _renew_instances(self, run: dict) -> None: updated_at=now() WHERE challenge_id=%s""", (next_renewal, instance["challenge_id"]), ) + self.store.append_run_event(run["id"], "instance.renewed", {"challenge_id": instance["challenge_id"]}) def _start_instance(self, run: dict, challenge: dict) -> bool: platform = self._platform_for_run(run) @@ -2011,16 +2262,37 @@ def _start_instance(self, run: dict, challenge: dict) -> bool: return False with self.state.db.connect() as connection: existing = connection.execute( - "SELECT state FROM competition_instances WHERE challenge_id=%s", + "SELECT state,metadata FROM competition_instances WHERE challenge_id=%s", (challenge["id"],), ).fetchone() + if existing is not None and existing["state"] == "expired": + repair_after = (existing.get("metadata") or {}).get("repair_after") + if repair_after: + try: + if _timestamp(repair_after) and _timestamp(repair_after) > datetime.now(timezone.utc): + return False + except (TypeError, ValueError): + pass if existing is not None and existing["state"] in { "ready", "running", "active", "start_pending", "stop_pending" }: return existing["state"] in {"ready", "running", "active"} try: + self.store.append_run_event(run["id"], "instance.start_requested", {"challenge_id": challenge["id"]}) result = platform.start_instance(challenge["external_id"]) except Exception as exc: + _recovery_logger.debug('Error handled in _start_instance', exc_info=True) + with self.state.db.connect() as connection: + connection.execute( + """INSERT INTO competition_instances + (challenge_id,run_id,external_id,state,metadata,updated_at) + VALUES (%s,%s,%s,'expired',%s,now()) + ON CONFLICT (challenge_id) DO UPDATE SET state='expired', + metadata=competition_instances.metadata || EXCLUDED.metadata, + updated_at=now()""", + (challenge["id"], run["id"], challenge["external_id"], + Jsonb({"start_error": str(exc), "repair_after": (datetime.now(timezone.utc) + timedelta(seconds=60)).isoformat()})), + ) self.store.append_run_event( run["id"], "instance.start_failed", {"challenge_id": challenge["id"], "error": str(exc)}, @@ -2064,6 +2336,7 @@ def _start_instance(self, run: dict, challenge: dict) -> bool: (challenge["id"],), ) except Exception as exc: + _recovery_logger.debug('Error handled in _start_instance', exc_info=True) try: with self.state.db.connect() as connection: connection.execute( @@ -2083,11 +2356,31 @@ def _start_instance(self, run: dict, challenge: dict) -> bool: {"challenge_id": challenge["id"], "error": str(exc)}, ) return False + self.store.append_run_event(run["id"], "instance.started", {"challenge_id": challenge["id"], "repair": bool(existing and existing["state"] == "expired")}) return True def _process_submissions(self, run: dict, *, allow_new: bool = True) -> None: platform = self._platform_for_run(run) self._ingest_project_candidates(run) + with self.state.db.connect() as connection: + closeout_rows = connection.execute( + """SELECT s.*,c.external_id,c.project_id FROM competition_submissions s + JOIN competition_challenges c ON c.id=s.challenge_id + LEFT JOIN projects p ON p.id=c.project_id + WHERE s.run_id=%s AND s.status='correct' AND p.id IS NOT NULL + AND COALESCE((s.verdict->>'challenge_complete')::boolean, true) + AND (p.status IS DISTINCT FROM 'solved' + OR EXISTS (SELECT 1 FROM competition_assignments a + WHERE a.challenge_id=c.id AND a.released_at IS NULL)) + ORDER BY s.created_at""", + (run["id"],), + ).fetchall() + for submission in closeout_rows: + challenge = next( + item for item in self.store.challenges(run["id"]) + if item["id"] == submission["challenge_id"] + ) + self._on_correct(run, challenge, submission["candidate"]) with self.state.db.connect() as connection: unresolved = connection.execute( """SELECT s.*,c.external_id FROM competition_submissions s @@ -2102,9 +2395,15 @@ def _process_submissions(self, run: dict, *, allow_new: bool = True) -> None: submission["external_id"], submission["platform_submission_id"] ) except Exception: + _recovery_logger.debug('Error handled in _process_submissions', exc_info=True) continue finished = self.store.finish_submission(submission["id"], result) - if finished["status"] == "correct": + self.store.append_run_event( + run["id"], "submission.verdict", + {"challenge_id": submission["challenge_id"], "submission_id": submission["id"], + "verdict": finished["status"], "source": "query"}, + ) + if finished["status"] == "correct" and (finished.get("verdict") or {}).get("challenge_complete", True): challenge = next( item for item in self.store.challenges(run["id"]) if item["id"] == submission["challenge_id"] @@ -2123,6 +2422,7 @@ def _process_submissions(self, run: dict, *, allow_new: bool = True) -> None: try: result = platform.submit(challenge["external_id"], submission["candidate"]) except Exception: + _recovery_logger.debug('Error handled in _process_submissions', exc_info=True) result = {"verdict": "unknown"} finished = self.store.finish_submission(submission["id"], result) self.store.append_run_event( @@ -2130,11 +2430,19 @@ def _process_submissions(self, run: dict, *, allow_new: bool = True) -> None: {"challenge_id": challenge["id"], "submission_id": submission["id"], "verdict": finished["status"]}, ) - if finished["status"] == "correct": + if finished["status"] == "correct" and (finished.get("verdict") or {}).get("challenge_complete", True): self._on_correct(run, challenge, submission["candidate"]) self._mark_project_candidate( challenge, submission["candidate"], "verified" ) + elif finished["status"] == "correct": + self._mark_project_candidate(challenge, submission["candidate"], "verified") + self.store.append_run_event(run["id"], "challenge.flag_progress", + {"challenge_id": challenge["id"], "progress": finished.get("verdict")}) + orchestrator = self.state.orchestrator + if challenge.get("project_id") and orchestrator is not None: + with suppress(Exception): + orchestrator._resume_after_verdict(challenge["project_id"], reason="flag accepted; remaining platform objectives are incomplete") elif finished["status"] in {"wrong", "rejected"}: self._mark_project_candidate( challenge, submission["candidate"], "rejected" @@ -2186,11 +2494,35 @@ def _mark_project_candidate(self, challenge: dict, flag: str, status: str) -> No ) def _on_correct(self, run: dict, challenge: dict, flag: str) -> None: + with self.state.db.connect() as connection: + partial = connection.execute( + """SELECT verdict FROM competition_submissions WHERE run_id=%s AND challenge_id=%s + AND candidate=%s AND status='correct' ORDER BY created_at DESC LIMIT 1""", + (run["id"], challenge["id"], flag), + ).fetchone() + if partial and (partial.get("verdict") or {}).get("challenge_complete") is False: + return project_id = challenge.get("project_id") orchestrator = self.state.orchestrator if project_id: try: with self.state.db.connect() as connection: + self.store._lock(connection, run["id"]) + connection.execute("SELECT id FROM projects WHERE id=%s FOR UPDATE", (project_id,)) + goal = connection.execute( + "SELECT 1 FROM intents WHERE project_id=%s AND to_fact_id='goal' LIMIT 1", + (project_id,), + ).fetchone() + if goal is None: + fact = node_store.reserve_fact( + connection, project_id, + "Verified flag submitted through the competition closeout API.", + ) + node_store.insert_fact(connection, project_id, fact.id, fact.description) + edge_store.complete_goal_intent( + connection, project_id, [fact.id], + "verified flag submission", SERVICE_MEMBER_ID, + ) accept_verified_flag( connection, project_id, flag, source=f"competition:{run['workflow_id']}", @@ -2198,7 +2530,9 @@ def _on_correct(self, run: dict, challenge: dict, flag: str) -> None: from backend.core.postprocess_store import enqueue_postprocess enqueue_postprocess(connection, project_id) + self.store.finalize_correct_challenge(connection, run["id"], challenge["id"]) except Exception as exc: + _recovery_logger.debug('Error handled in _on_correct', exc_info=True) self.store.append_run_event( run["id"], "challenge.closeout_failed", {"challenge_id": challenge["id"], "error": str(exc)}, @@ -2227,7 +2561,7 @@ def _dispatch_wp(self, run: dict) -> None: for key in finished: self._wp_futures.pop(key, None) while len(self._wp_futures) < 2: - claimed = self.store.claim_wp_job(run["id"], self.owner) + claimed = self.store.claim_wp_job(run["id"], self.owner, require_verified=True) if claimed is None: return challenge_id = claimed["job"]["challenge_id"] @@ -2288,10 +2622,12 @@ def generator(prompt: str) -> str: "artifact_path": path}, ) except Exception as exc: + _recovery_logger.debug('Error handled in _run_wp_job', exc_info=True) + finished_job = None with suppress(CompetitionConflict): - self.store.finish_wp_job(assignment, error=str(exc)) + finished_job = self.store.finish_wp_job(assignment, error=str(exc)) self.store.append_run_event( - job["run_id"], "wp.deferred", + job["run_id"], "wp.failed" if finished_job and finished_job.get("status") == "failed" else "wp.deferred", {"challenge_id": job["challenge_id"], "member": job["member"], "error": str(exc)}, ) diff --git a/backend/competition/store.py b/backend/competition/store.py index 8451605..bb0f32c 100644 --- a/backend/competition/store.py +++ b/backend/competition/store.py @@ -1,12 +1,14 @@ from __future__ import annotations -import uuid import hashlib +import json +import uuid from typing import Any from psycopg.types.json import Jsonb from backend.core.config import MEMBER_NAMES +from backend.core.validation import InputValidationError from backend.platform.verdict import cooldown_seconds @@ -53,9 +55,71 @@ class CompetitionStore: deliberately not held during model calls, downloads, or platform requests. """ + provider_progress_events = True + def __init__(self, db): self.db = db + def queue_control(self, run_id: str, action: str, revision: int, operation_id: str | None = None) -> dict: + if action not in {"pause", "resume", "stop", "refresh"}: + raise ValueError("unsupported run action") + operation_id = operation_id or new_id() + with self.db.connect() as connection: + previous = connection.execute("SELECT * FROM competition_control_operations WHERE id=%s", (operation_id,)).fetchone() + if previous: + if (previous["run_id"], previous["action"], previous["requested_revision"]) != (run_id, action, revision): + raise CompetitionConflict("control operation id reused with different content") + return previous + run = connection.execute("SELECT revision FROM competition_runs WHERE id=%s", (run_id,)).fetchone() + if run is None: + raise KeyError(run_id) + if run["revision"] != revision: + raise CompetitionConflict("stale run revision") + connection.execute( + """INSERT INTO competition_control_operations(id,run_id,action,requested_revision) + VALUES (%s,%s,%s,%s) ON CONFLICT(id) DO NOTHING""", (operation_id, run_id, action, revision), + ) + operation = connection.execute("SELECT * FROM competition_control_operations WHERE id=%s", (operation_id,)).fetchone() + if (operation["run_id"], operation["action"], operation["requested_revision"]) != (run_id, action, revision): + raise CompetitionConflict("control operation id reused with different content") + return operation + + def claim_control(self, owner: str) -> dict | None: + with self.db.connect() as connection: + operation = connection.execute( + """UPDATE competition_control_operations SET status='running',owner=%s,epoch=epoch+1, + lease_expires_at=now()+interval '90 seconds' + WHERE id=(SELECT o.id FROM competition_control_operations o JOIN competition_runs r ON r.id=o.run_id + WHERE (o.status='queued' OR (o.status='running' AND o.lease_expires_at<=now())) + AND (r.status IN ('stopped','finished') OR r.lease_owner=%s + OR r.lease_expires_at IS NULL OR r.lease_expires_at<=now()) + ORDER BY o.created_at,o.id FOR UPDATE OF o SKIP LOCKED LIMIT 1) + RETURNING *""", (owner, owner), + ).fetchone() + if operation: + connection.execute( + """UPDATE competition_runs SET lease_owner=%s,lease_expires_at=now()+interval '90 seconds' + WHERE id=%s AND status NOT IN ('finished','stopped')""", (owner, operation["run_id"]), + ) + return operation + + def finish_control(self, operation: dict, *, error: str | None = None) -> bool: + with self.db.connect() as connection: + return connection.execute( + """UPDATE competition_control_operations SET status=%s,error=%s,finished_at=now(),lease_expires_at=NULL + WHERE id=%s AND owner=%s AND epoch=%s AND status='running' RETURNING id""", + ("failed" if error else "completed", error, operation["id"], operation["owner"], operation["epoch"]), + ).fetchone() is not None + + def control_operation(self, run_id: str, operation_id: str) -> dict: + with self.db.connect() as connection: + result = connection.execute( + "SELECT * FROM competition_control_operations WHERE id=%s AND run_id=%s", (operation_id, run_id), + ).fetchone() + if result is None: + raise KeyError(operation_id) + return result + def claim_engine_lease( self, project_id: str, kind: str, owner: str, *, seconds: int = 60 ) -> dict | None: @@ -137,11 +201,14 @@ def create_run(self, workflow_id: str, identity_key: str, idempotency_key: str, ).fetchone() if active: raise CompetitionConflict("another competition is active") + limits = snapshot.get("limits") if isinstance(snapshot, dict) else None + if not isinstance(limits, dict): + limits = {} return connection.execute( """INSERT INTO competition_runs - (id,workflow_id,identity_key,idempotency_key,config_snapshot) - VALUES (%s,%s,%s,%s,%s) RETURNING *""", - (new_id(), workflow_id, identity_key, idempotency_key, Jsonb(snapshot)), + (id,workflow_id,identity_key,idempotency_key,config_snapshot,limits) + VALUES (%s,%s,%s,%s,%s,%s) RETURNING *""", + (new_id(), workflow_id, identity_key, idempotency_key, Jsonb(snapshot), Jsonb(limits)), ).fetchone() def run_by_idempotency(self, idempotency_key: str) -> dict | None: @@ -710,6 +777,22 @@ def finish_submission(self, submission_id: str, result: dict) -> dict: """UPDATE competition_submissions SET status=%s,verdict=%s,platform_submission_id=%s WHERE id=%s""", (status, Jsonb(result), result.get("submission_id"), submission_id), ) + if status == "correct" and "challenge_complete" in result: + progress = dict((challenge.get("metadata") or {}).get("flag_progress") or {}) + for key in ("correct_flag_count", "total_flag_count", "challenge_complete", "cumulative_score"): + if result.get(key) is not None: + if key == "correct_flag_count": + progress[key] = max(int(progress.get(key) or 0), int(result[key])) + elif key == "challenge_complete": + progress[key] = bool(progress.get(key)) or bool(result[key]) + else: + progress[key] = result[key] + indices = set(progress.get("matched_indices", [])) + if result.get("matched_flag_index") is not None: + indices.add(result["matched_flag_index"]) + progress["matched_indices"] = sorted(indices) + connection.execute("UPDATE competition_challenges SET metadata=metadata || %s WHERE id=%s", + (Jsonb({"flag_progress": progress}), challenge["id"])) if status == "wrong" and challenge["state"] != "solved": wrong_count = challenge["wrong_count"] + 1 connection.execute( @@ -725,14 +808,28 @@ def finish_submission(self, submission_id: str, result: dict) -> dict: (max(1, int(result.get("retry_after") or 60)), row["run_id"]), ) connection.execute("UPDATE competition_submissions SET status='queued' WHERE id=%s", (submission_id,)) - elif status == "correct" and challenge["state"] != "solved": - connection.execute("UPDATE competition_challenges SET state='solved' WHERE id=%s", (challenge["id"],)) - connection.execute( - """INSERT INTO competition_wp_jobs (challenge_id,run_id,member,session_id) - VALUES (%s,%s,%s,%s) ON CONFLICT DO NOTHING""", - (challenge["id"], row["run_id"], row["member"], row["session_id"]), - ) - return {**row, "status": status} + return {**row, "status": status, "verdict": result} + + def finalize_correct_challenge(self, connection, run_id: str, challenge_id: str) -> None: + """Commit solved/WP in the caller's verified-flag closeout transaction.""" + self._lock(connection, run_id) + submission = connection.execute( + """SELECT * FROM competition_submissions + WHERE run_id=%s AND challenge_id=%s AND status='correct' + AND COALESCE((verdict->>'challenge_complete')::boolean, true) + ORDER BY created_at,id LIMIT 1 FOR UPDATE""", + (run_id, challenge_id), + ).fetchone() + if submission is None: + raise CompetitionConflict("closeout requires a correct platform verdict") + connection.execute( + "UPDATE competition_challenges SET state='solved' WHERE id=%s", (challenge_id,), + ) + connection.execute( + """INSERT INTO competition_wp_jobs (challenge_id,run_id,member,session_id) + VALUES (%s,%s,%s,%s) ON CONFLICT DO NOTHING""", + (challenge_id, run_id, submission["member"], submission["session_id"]), + ) def reconcile_inflight_submissions(self, run_id: str, *, grace_seconds: int = 30) -> int: """Move abandoned ``submitting`` rows to queryable ``unknown``. @@ -807,7 +904,7 @@ def run_events(self, run_id: str, after: int = 0, limit: int = 200) -> list[dict (run_id, max(0, after), max(1, min(limit, 1000))), ).fetchall() - def observe_run(self, run_id: str, owner: str) -> dict: + def observe_run(self, run_id: str, owner: str, *, resource_capacity: dict | None = None) -> dict: """Persist a compact, redacted health sample for a competition run. The sample is intentionally derived from durable rows rather than @@ -907,6 +1004,8 @@ def observe_run(self, run_id: str, owner: str) -> dict: "sync_lag_seconds": round(float(sync_lag or 0), 3), "artifact_bytes": int(artifact_bytes or 0), } + if resource_capacity is not None: + payload["resource_capacity"] = resource_capacity return connection.execute( """INSERT INTO competition_run_observations (run_id,owner,payload) VALUES (%s,%s,%s) RETURNING *""", @@ -977,6 +1076,45 @@ def run_snapshot(self, run_id: str) -> dict: challenge_views.append(item) solving = sum(a["role"] in {"primary", "helper"} for a in assignments) writing = sum(a["role"] == "wp" for a in assignments) + remote_challenge_ids = { + item["id"] for item in challenges if bool((item.get("metadata") or {}).get("remote")) + } + local_used = len({a["challenge_id"] for a in assignments + if a["role"] != "wp" and a["challenge_id"] not in remote_challenge_ids}) + with self.db.connect() as connection: + remote_used = int(connection.execute( + """SELECT count(*) AS count FROM competition_instances WHERE run_id=%s + AND state NOT IN ('stopped','expired')""", (run_id,), + ).fetchone()["count"]) + observation = connection.execute( + """SELECT payload->'resource_capacity' AS capacity FROM competition_run_observations + WHERE run_id=%s AND payload ? 'resource_capacity' + ORDER BY observation_id DESC LIMIT 1""", (run_id,), + ).fetchone() + observed = observation["capacity"] if observation else {} + raw_limits = run.get("limits") or {} + if isinstance(raw_limits, str): + try: + raw_limits = json.loads(raw_limits) + except (TypeError, ValueError): + raw_limits = {} + snapshot_limits = (run.get("config_snapshot") or {}).get("limits_snapshot", {}) + spec = (run.get("config_snapshot") or {}).get("workflow_spec", {}) + local_limit = int( + (raw_limits or {}).get( + "max_concurrent_tasks", + (observed.get("local") or {}).get("limit", (snapshot_limits or {}).get("max_concurrent_tasks", 0)), + ) + or 0 + ) + remote_limit = int(raw_limits.get("remote_concurrency", (observed.get("remote") or {}).get( + "limit", spec.get("remote_concurrency") or snapshot_limits.get("remote_concurrency", spec.get("remote_instance_limit", 0)), + )) or 0) + remote_source = "run" if "remote_concurrency" in raw_limits else (observed.get("remote") or {}).get( + "source", "workflow.remote_concurrency" if spec.get("remote_concurrency") else snapshot_limits.get( + "remote_source", "workflow.remote_instance_limit" if spec.get("remote_instance_limit") else "unreported", + ), + ) return { "run": run, "challenges": challenge_views, @@ -987,8 +1125,58 @@ def run_snapshot(self, run_id: str) -> dict: "collaborating": sum(a["role"] == "helper" for a in assignments), "idle": max(0, len(MEMBER_NAMES) - len(assignments)), }, + "resource_capacity": { + "local": {"used": local_used, "limit": local_limit, "remaining": max(0, local_limit - local_used), + "source": "run" if "max_concurrent_tasks" in raw_limits else (observed.get("local") or {}).get("source", "config_snapshot"), + "challenge_count": len(challenges) - len(remote_challenge_ids)}, + "remote": {"used": remote_used, "limit": remote_limit, "remaining": max(0, remote_limit - remote_used), + "source": remote_source, "challenge_count": len(remote_challenge_ids)}, + }, } + def update_limits(self, run_id: str, limits: dict, *, revision: int) -> dict: + """Patch mutable run limits with optimistic revision control.""" + if not isinstance(limits, dict): + raise InputValidationError("limits must be an object") + allowed = {"max_concurrent_tasks", "total_cpu", "remote_concurrency"} + unknown = set(limits) - allowed + if unknown: + raise ValueError(f"unknown run limit(s): {sorted(unknown)}") + cleaned: dict[str, int] = {} + for key, value in limits.items(): + try: + number = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{key} must be an integer") from exc + if number < 0 or number > 1000: + raise ValueError(f"{key} must be between 0 and 1000") + cleaned[key] = number + with self.db.connect() as connection: + self._lock(connection, run_id) + row = connection.execute( + "SELECT revision,limits FROM competition_runs WHERE id=%s FOR UPDATE", + (run_id,), + ).fetchone() + if row is None: + raise KeyError(run_id) + if int(row["revision"]) != int(revision): + raise CompetitionConflict("stale run revision") + current = row.get("limits") or {} + if isinstance(current, str): + try: + current = json.loads(current) + except (TypeError, ValueError): + current = {} + merged = {**(current if isinstance(current, dict) else {}), **cleaned} + updated = connection.execute( + """UPDATE competition_runs SET limits=%s,revision=revision+1,updated_at=now() + WHERE id=%s AND revision=%s RETURNING *""", + (Jsonb(merged), run_id, revision), + ).fetchone() + if updated is None: + raise CompetitionConflict("stale run revision") + return updated + def member_snapshot(self, run_id: str) -> list[dict]: active = {row["member"]: row for row in self.assignments(run_id, active_only=True)} result = [] @@ -1062,9 +1250,9 @@ def release_expired_assignments(self, run_id: str) -> list[dict]: with self.db.connect() as connection: self._lock(connection, run_id) connection.execute( - """UPDATE competition_wp_jobs w SET status='deferred', + """UPDATE competition_wp_jobs w SET status=CASE WHEN w.attempts >= 5 THEN 'failed' ELSE 'deferred' END, last_error='WP attempt exceeded the 10 minute seat deadline', - next_attempt_at=now()+interval '5 minutes' + next_attempt_at=CASE WHEN w.attempts >= 5 THEN now() ELSE now()+interval '5 minutes' END FROM competition_assignments a WHERE a.run_id=%s AND a.role='wp' AND a.challenge_id=w.challenge_id AND a.released_at IS NULL AND a.wp_deadline_at<=now() @@ -1084,12 +1272,19 @@ def release_expired_assignments(self, run_id: str) -> list[dict]: self._expire_assignment_writers( connection, [row["session_id"] for row in rows] ) - connection.execute( + expired = connection.execute( """UPDATE competition_challenges SET state='expired' WHERE id IN (SELECT challenge_id FROM competition_run_challenges WHERE run_id=%s) - AND deadline_at IS NOT NULL AND deadline_at<=now() AND state NOT IN ('solved','cancelled')""", + AND deadline_at IS NOT NULL AND deadline_at<=now() + AND state NOT IN ('solved','cancelled','expired') RETURNING id""", (run_id,), - ) + ).fetchall() + for challenge in expired: + connection.execute( + """INSERT INTO competition_run_events(run_id,kind,payload) + VALUES (%s,'challenge.expired',%s)""", + (run_id, Jsonb({"challenge_id": challenge["id"]})), + ) return rows def recover_expired_assignments(self, run_id: str, owner: str) -> list[dict]: @@ -1491,6 +1686,13 @@ def persist_recon_message(self, message) -> dict: previous.pop("sequence", None) candidate = dict(data) candidate.pop("sequence", None) + # Callers stamp created_at/deadline_at at publish time; a retry after + # a restart regenerates them. They are not part of the message's + # identity, so comparing them would turn every retry into a + # spurious "request id reused" conflict. + for field in ("created_at", "deadline_at"): + previous.pop(field, None) + candidate.pop(field, None) if previous != candidate: raise CompetitionConflict("request id reused with a different message") if existing["status"] != "done": @@ -1595,13 +1797,19 @@ def undelivered_messages(self, run_id: str, *, older_than_seconds: int) -> list[ (run_id, max(0, older_than_seconds)), ).fetchall() - def claim_wp_job(self, run_id: str, owner: str) -> dict | None: + def claim_wp_job(self, run_id: str, owner: str, *, require_verified: bool = False) -> dict | None: with self.db.connect() as connection: self._lock(connection, run_id) + verified_filter = """ + AND EXISTS (SELECT 1 FROM competition_challenges c + JOIN projects p ON p.id=c.project_id + WHERE c.id=w.challenge_id AND p.status='solved' AND p.flag IS NOT NULL)""" if require_verified else "" job = connection.execute( - """SELECT w.* FROM competition_wp_jobs w + f"""SELECT w.* FROM competition_wp_jobs w JOIN competition_runs r ON r.id=w.run_id WHERE w.run_id=%s AND w.status IN ('pending','deferred') + AND w.attempts < 5 + {verified_filter} AND w.next_attempt_at<=now() AND NOT EXISTS (SELECT 1 FROM competition_assignments a WHERE a.member=w.member AND a.released_at IS NULL) @@ -1663,13 +1871,20 @@ def finish_wp_job( "SELECT attempts FROM competition_wp_jobs WHERE challenge_id=%s", (assignment["challenge_id"],), ).fetchone()["attempts"] - delay = min(3600, 60 * (2 ** min(attempts - 1, 6))) - job = connection.execute( - """UPDATE competition_wp_jobs SET status='deferred',last_error=%s, - next_attempt_at=now()+(%s*interval '1 second') - WHERE challenge_id=%s RETURNING *""", - ((error or "WP attempt failed")[:2000], delay, assignment["challenge_id"]), - ).fetchone() + if attempts >= 5: + job = connection.execute( + """UPDATE competition_wp_jobs SET status='failed',last_error=%s, + next_attempt_at=now() WHERE challenge_id=%s RETURNING *""", + ((error or "WP attempt failed; retry limit exceeded")[:2000], assignment["challenge_id"]), + ).fetchone() + else: + delay = min(3600, 60 * (2 ** min(attempts - 1, 6))) + job = connection.execute( + """UPDATE competition_wp_jobs SET status='deferred',last_error=%s, + next_attempt_at=now()+(%s*interval '1 second') + WHERE challenge_id=%s RETURNING *""", + ((error or "WP attempt failed")[:2000], delay, assignment["challenge_id"]), + ).fetchone() connection.execute( """UPDATE competition_assignments SET released_at=now(),epoch=epoch+1 WHERE id=%s""", diff --git a/backend/competition/transport.py b/backend/competition/transport.py index f0806d4..205307f 100644 --- a/backend/competition/transport.py +++ b/backend/competition/transport.py @@ -1,27 +1,26 @@ -"""Single-host gRPC/UDS control and ZeroMQ reconnaissance transport. - -The coordinator is hosted by a Linux worker, never by a challenge sandbox. -Protobuf is the gRPC source-of-truth contract. JSON remains the persisted and -ZeroMQ representation, with explicit conversion through ``Envelope``. -""" from __future__ import annotations import json +import logging import os import queue import threading +import time +from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from contextlib import suppress from datetime import datetime, timezone from pathlib import Path -from typing import Any, Callable, Literal +from typing import Any, Literal import grpc import zmq from pydantic import BaseModel, ConfigDict, Field, model_validator from backend.competition.proto import pair_pb2 +from backend.core.validation import InputValidationError +_recovery_logger = logging.getLogger(__name__) class Envelope(BaseModel): model_config = ConfigDict(extra="forbid") @@ -126,7 +125,7 @@ def result_from_proto(raw: pair_pb2.Result) -> dict: except (TypeError, ValueError) as exc: raise ValueError("result payload_json is not valid JSON") from exc if not isinstance(value, dict): - raise ValueError("result payload must be a JSON object") + raise InputValidationError("result payload must be a JSON object") value.setdefault("status", raw.status) value.setdefault("request_id", raw.request_id) return value @@ -227,6 +226,7 @@ def _call(self, raw, context): try: result = self.execute(message, cancel) except Exception as exc: + _recovery_logger.debug('Error handled in _call', exc_info=True) result = { "status": "failed", "request_id": message.request_id, @@ -340,6 +340,7 @@ def _run(self) -> None: self._socket = socket self._bound = True except BaseException as exc: + _recovery_logger.debug('Error handled in _run', exc_info=True) self._error = exc self._ready.set() if socket is not None: @@ -358,6 +359,7 @@ def _run(self) -> None: try: socket.send_multipart([topic, payload], flags=zmq.NOBLOCK) except Exception as exc: + _recovery_logger.debug('Error handled in _run', exc_info=True) self._error = exc break finally: @@ -443,10 +445,10 @@ def reliable(self) -> bool: def _normalize(self, value: Any) -> dict: if not isinstance(value, dict): - raise ValueError("recon record must be a JSON object") + raise InputValidationError("recon record must be a JSON object") envelope_value = value.get("envelope", value) if not isinstance(envelope_value, dict): - raise ValueError("recon envelope must be a JSON object") + raise InputValidationError("recon envelope must be a JSON object") sequence = value.get("sequence", envelope_value.get("sequence")) if not isinstance(sequence, int) or isinstance(sequence, bool) or sequence <= 0: raise ValueError("recon record requires a positive persisted sequence") @@ -484,6 +486,22 @@ def _catch_up(self) -> None: for record in records: self._queue(record) + def _addressed_elsewhere(self, value: Any) -> bool: + """Return True when a broadcast record belongs to a peer's session. + + Recon is published on one challenge topic, so every subscriber sees + every receiver's mail. Only this session's records belong in this + transcript; rejecting a peer's record here would crash an ordinary + teammate's turn instead of simply ignoring traffic that is not ours. + """ + if self.session_id is None or not isinstance(value, dict): + return False + envelope = value.get("envelope", value) + if not isinstance(envelope, dict): + return False + receiver = envelope.get("receiver_session_id") + return receiver is not None and receiver != self.session_id + def ack(self, sequence: int | None = None) -> int: if not self.reliable or self.session_id is None or self.advance is None: raise RuntimeError("persistent acknowledgement is not configured") @@ -510,25 +528,31 @@ def receive(self, timeout_ms: int = 1000) -> dict | None: if self._pending: self._inflight = self._pending[min(self._pending)] return self._inflight - if self.socket is None or not self.socket.poll(timeout_ms): - return None - topic, payload = self.socket.recv_multipart() - if topic.decode() != self.challenge_id: - return None - try: - value = json.loads(payload) - except (TypeError, ValueError) as exc: - raise ValueError("recon payload is not valid JSON") from exc - if not self.reliable: - if isinstance(value, dict) and "envelope" in value: - self._normalize(value) - return value - self._queue(value) - self._catch_up() - if not self._pending: + if self.socket is None: return None - self._inflight = self._pending[min(self._pending)] - return self._inflight + deadline = time.monotonic() + max(0, timeout_ms) / 1000.0 + while True: + remaining_ms = max(0, int((deadline - time.monotonic()) * 1000)) + if not self.socket.poll(remaining_ms): + return None + topic, payload = self.socket.recv_multipart() + if topic.decode() != self.challenge_id: + continue + try: + value = json.loads(payload) + except (TypeError, ValueError) as exc: + raise ValueError("recon payload is not valid JSON") from exc + if not self.reliable: + if isinstance(value, dict) and "envelope" in value: + self._normalize(value) + return value + if self._addressed_elsewhere(value): + continue + self._queue(value) + self._catch_up() + if self._pending: + self._inflight = self._pending[min(self._pending)] + return self._inflight def close(self): if self.socket is not None: diff --git a/backend/core/archive.py b/backend/core/archive.py index 9d8ac2a..4499436 100644 --- a/backend/core/archive.py +++ b/backend/core/archive.py @@ -37,16 +37,20 @@ def archive_completed_project( row = graph_store.get_project_row(connection, project_id) if row is None: raise ValueError(f"project not found: {project_id}") - if row["status"] != "solved": + if row["status"] not in {"solved", "completed"}: raise ValueError(f"project {project_id} is not solved") wp_path = Path(str(row["wp_path"] or "")) if not wp_path.is_file(): raise ValueError(f"project {project_id} does not have a readable writeup") - writeup_errors = validate_writeup( - wp_path.read_text(encoding="utf-8"), - expected_flag=str(row["flag"] or "") or None, - require_complete=True, - ) + if row.get("completion_policy") == "goals": + from backend.pentest.report import validate_report + writeup_errors = validate_report(wp_path.read_text(encoding="utf-8")) + else: + writeup_errors = validate_writeup( + wp_path.read_text(encoding="utf-8"), + expected_flag=str(row["flag"] or "") or None, + require_complete=True, + ) if writeup_errors: raise ValueError( f"project {project_id} does not have a valid final writeup: {'; '.join(writeup_errors)}" diff --git a/backend/core/config.py b/backend/core/config.py index 2fe6c6c..2a469ad 100644 --- a/backend/core/config.py +++ b/backend/core/config.py @@ -10,7 +10,7 @@ import yaml from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -CATEGORIES: tuple[str, ...] = ("pwn", "reverse", "crypto", "web", "misc", "ai", "osint") +CATEGORIES: tuple[str, ...] = ("pwn", "reverse", "crypto", "web", "misc", "ai", "osint", "pentest") ApiFormat = Literal["openai", "anthropic", "deepseek", "pi", "mock"] ApiSurface = Literal["auto", "chat_completions", "responses"] @@ -86,6 +86,12 @@ class RuntimeConfig(BaseModel): compaction_trigger_ratio: float = Field(default=0.75, gt=0.0, le=0.95) max_turns: int = Field(default=0, ge=0) provider_read_timeout: int = Field(default=300, ge=30) + provider_turn_timeout: int = Field(default=900, ge=30, le=7200) + provider_progress_timeout: int = Field(default=300, ge=10, le=3600) + tool_output_bytes: int = Field(default=16384, ge=1024, le=65536) + context_bytes: int = Field(default=1_000_000, ge=16384, le=16_000_000) + compaction_timeout: int = Field(default=30, ge=1, le=120) + compaction_max_output_tokens: int = Field(default=2048, ge=256, le=8192) browser_event_limit: int = Field(default=200, gt=0, le=1000) browser_console_limit: int = Field(default=100, gt=0, le=1000) browser_error_limit: int = Field(default=50, gt=0, le=1000) diff --git a/backend/core/diamond.py b/backend/core/diamond.py index 00d3413..7bc6d5b 100644 --- a/backend/core/diamond.py +++ b/backend/core/diamond.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from dataclasses import dataclass from backend.blackboard import edge_store, graph_store, node_store @@ -9,6 +10,8 @@ from backend.core.wp_writer import WRITEUP_SYSTEM_PROMPT, generate_wp_content, write_wp from backend.members.adapters import make_adapter +_recovery_logger = logging.getLogger(__name__) + BOOTSTRAP_DESC = "Bootstrap: Starting" @@ -68,6 +71,7 @@ def _members_seated_elsewhere(self, project_id: str) -> set[str]: (project_id,), ).fetchall() except Exception: + _recovery_logger.debug('Error handled in _members_seated_elsewhere', exc_info=True) return set() return {row["member"] for row in rows} @@ -91,7 +95,12 @@ def assign_initial( graph_store.set_agent_state(conn, project_id, "diamond", "active") if not graph_store.link_exists(conn, project_id, "ipc", "diamond", "start"): graph_store.add_link(conn, project_id, "ipc", "diamond", "start") - intent = edge_store.create_intent(conn, project_id, ["origin"], BOOTSTRAP_DESC, "diamond") + row = graph_store.get_project_row(conn, project_id) + description = ("Read pentest_board, inspect the explicit scope and constraints, decompose the user objectives, " + "then persist focused work with pentest_plan_work. Record discovered assets and actual evidence. " + "Use only the existing task harness and do not treat a candidate finding as verified." + if row and row["category"] == "pentest" else BOOTSTRAP_DESC) + intent = edge_store.create_intent(conn, project_id, ["origin"], description, "diamond") graph_store.add_agent(conn, project_id, initial.name, "member", state="active", start_fact_id="origin") graph_store.set_agent_state(conn, project_id, initial.name, "active") graph_store.add_link(conn, project_id, "diamond", initial.name, "assign") @@ -166,6 +175,10 @@ def _dedupe_directions(self, directions: list[str]) -> list[str]: def plan_next_intent(self, project_id: str, detail, trigger: str): """Create one follow-up intent after a real graph-state checkpoint change.""" + if detail.project.category == "pentest": + from backend.pentest.store import plan_next + with self.db.connect() as conn: + return plan_next(conn, project_id, detail, trigger, max_open=min(2, self.config.runtime.max_members_per_report)) open_intents = [intent for intent in detail.intents if intent.to is None] if open_intents: self.logger.project( @@ -255,6 +268,11 @@ def generator(prompt: str) -> str: max_tokens=8192, ) + with self.db.connect() as conn: + row = graph_store.get_project_row(conn, project_id) + if row and row.get("completion_policy") == "goals": + from backend.pentest.report import render_report + return render_report(conn, project_id), "" return generate_wp_content( self.db, project_id, @@ -271,8 +289,10 @@ def draw_completion(self, project_id: str, connection=None) -> None: """ def _draw(conn) -> None: - if not graph_store.link_exists(conn, project_id, "flag", "wp", "wp"): - graph_store.add_link(conn, project_id, "flag", "wp", "wp") + row = graph_store.get_project_row(conn, project_id) + source = "goal" if row and row.get("completion_policy") == "goals" else "flag" + if not graph_store.link_exists(conn, project_id, source, "wp", "wp"): + graph_store.add_link(conn, project_id, source, "wp", "wp") if not graph_store.link_exists(conn, project_id, "wp", "diamond", "wp"): graph_store.add_link(conn, project_id, "wp", "diamond", "wp") if not graph_store.link_exists(conn, project_id, "diamond", "ipc", "return"): diff --git a/backend/core/engine_lease.py b/backend/core/engine_lease.py index 9876cf2..f9586a5 100644 --- a/backend/core/engine_lease.py +++ b/backend/core/engine_lease.py @@ -1,10 +1,10 @@ -"""Database-backed fencing between the legacy and competition engines.""" from __future__ import annotations from collections.abc import Mapping from typing import Any +from backend.core.validation import InputValidationError ENGINE_KINDS = {"legacy", "competition"} @@ -117,7 +117,7 @@ def release_engine_leases_for_owner( _validate(kind, owner, 60) if not isinstance(epochs, Mapping): - raise ValueError("engine lease epochs are required") + raise InputValidationError("engine lease epochs are required") released = 0 for project_id, epoch in epochs.items(): if not isinstance(epoch, int) or epoch < 1: diff --git a/backend/core/ipc.py b/backend/core/ipc.py index 2d5c87a..ccadc9e 100644 --- a/backend/core/ipc.py +++ b/backend/core/ipc.py @@ -37,13 +37,21 @@ def assert_flag_compatible(connection, project_id: str, flag: str) -> dict: if not normalized: raise ValueError("flag must not be empty") project = connection.execute( - "SELECT flag, status, flag_verified_at FROM projects WHERE id = %s FOR UPDATE", + "SELECT flag, status, flag_verified_at, completion_policy FROM projects WHERE id = %s FOR UPDATE", (project_id,), ).fetchone() if project is None: raise ValueError("project not found") + policy = str(project.get("completion_policy") or "verified_flag") + if policy == "goals": + raise FlagConflictError("independent pentest projects complete through evidence-backed goal proofs") + # A platform_flags project is judged per challenge: one challenge may carry + # several flags, and each accepted candidate is a distinct verified row. + # Only a settled project is locked to the flag that closed it. + platform_flags = policy == "platform_flags" + settled = str(project.get("status") or "") in {"solved", "completed"} existing_flag = str(project["flag"] or "").strip() - if existing_flag and existing_flag != normalized: + if existing_flag and existing_flag != normalized and (settled or not platform_flags): raise FlagConflictError("project already has a different flag") verified = connection.execute( """ @@ -53,7 +61,9 @@ def assert_flag_compatible(connection, project_id: str, flag: str) -> dict: """, (project_id,), ).fetchall() - if any(str(row["normalized_flag"] or "").strip() != normalized for row in verified): + if not platform_flags and any( + str(row["normalized_flag"] or "").strip() != normalized for row in verified + ): raise FlagConflictError("project already has a different verified flag submission") return project @@ -100,7 +110,7 @@ def accept_verified_flag( ).fetchone() if has_goal_edge is None: raise ValueError("cannot verify flag without a completion edge") - key = hashlib.sha256(f"{project_id}\0{normalized}".encode("utf-8")).hexdigest() + key = hashlib.sha256(f"{project_id}\0{normalized}".encode()).hexdigest() now = utcnow() connection.execute( """ @@ -156,7 +166,7 @@ def record_pending_flag( normalized = flag.strip() project = assert_flag_compatible(connection, project_id, normalized) - key = hashlib.sha256(f"{project_id}\0{normalized}".encode("utf-8")).hexdigest() + key = hashlib.sha256(f"{project_id}\0{normalized}".encode()).hexdigest() now = utcnow() row = connection.execute( """ diff --git a/backend/core/lifecycle.py b/backend/core/lifecycle.py index caf10a3..e74cfce 100644 --- a/backend/core/lifecycle.py +++ b/backend/core/lifecycle.py @@ -4,9 +4,10 @@ TRANSITIONS = { "created": {"running", "stopped"}, - "running": {"flag_found", "stopped", "timeout", "infra_error", "failed"}, + "running": {"completed", "flag_found", "stopped", "timeout", "infra_error", "failed"}, "flag_found": {"solved", "running", "failed"}, "solved": set(), + "completed": set(), "timeout": {"running"}, "infra_error": {"running", "stopped"}, "failed": {"running", "stopped"}, diff --git a/backend/core/logging_util.py b/backend/core/logging_util.py index 740e555..253bedc 100644 --- a/backend/core/logging_util.py +++ b/backend/core/logging_util.py @@ -5,6 +5,7 @@ from collections.abc import Callable from datetime import UTC, datetime from pathlib import Path +from typing import ClassVar from backend.core.redaction import redact_flag @@ -14,7 +15,7 @@ def _utcnow() -> str: class IPCLogger: - KINDS = { + KINDS: ClassVar[dict] = { "project": "project_logs", "tool": "tool_logs", "llm": "llm_logs", @@ -134,8 +135,57 @@ def _read_jsonl(text: str) -> list[dict]: @staticmethod def _tail(path: Path, limit: int | None) -> list[dict]: - entries = IPCLogger._read_array(path) - return entries if limit is None else entries[-limit:] + if not path.is_file(): + return [] + if limit is None or path.suffix == ".json": + entries = IPCLogger._read_array(path) + return entries if limit is None else entries[-limit:] + if limit <= 0: + return [] + with path.open("rb") as handle: + handle.seek(0, 2) + position = handle.tell() + chunks = [] + newlines = 0 + while position and newlines <= limit: + size = min(position, 16384) + position -= size + handle.seek(position) + chunk = handle.read(size) + chunks.append(chunk) + newlines += chunk.count(b"\n") + data = b"".join(reversed(chunks)) + if position: + data = data.partition(b"\n")[2] + return IPCLogger._read_jsonl(data.decode("utf-8", "replace"))[-limit:] + + def read_since(self, kind: str, project_id: str, *, after: int = 0, + limit: int = 500, file_id: str = "") -> dict: + path = self._file(kind, project_id) + if not path.is_file(): + return {"entries": [], "next_offset": 0, "file_id": "", "has_more": False} + stat = path.stat() + identity = f"{stat.st_dev}:{stat.st_ino}" + if file_id and file_id != identity or after > stat.st_size: + after = 0 + records = [] + with path.open("rb") as handle: + handle.seek(max(0, after)) + while len(records) < max(1, min(1000, limit)): + position = handle.tell() + line = handle.readline() + if not line or not line.endswith(b"\n"): + handle.seek(position) + break + try: + record = json.loads(line) + except (ValueError, UnicodeError): + continue + if isinstance(record, dict): + records.append(record) + offset = handle.tell() + return {"entries": records, "next_offset": offset, "file_id": identity, + "has_more": offset < path.stat().st_size} @staticmethod def jsonl_text(entries: list[dict]) -> str: diff --git a/backend/core/orchestrator.py b/backend/core/orchestrator.py index 080d5ad..dfef619 100644 --- a/backend/core/orchestrator.py +++ b/backend/core/orchestrator.py @@ -1,28 +1,32 @@ from __future__ import annotations import asyncio -from collections import deque -from contextlib import suppress -from datetime import datetime, timedelta, timezone -import threading -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass import hashlib import json +import logging +import re import sys +import threading +import uuid +from collections import deque +from concurrent.futures import ThreadPoolExecutor +from contextlib import suppress +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone from time import monotonic from typing import Any -import uuid from psycopg.errors import UndefinedColumn, UndefinedTable +from psycopg.types.json import Jsonb from backend.blackboard import edge_store, graph_store, node_store +from backend.competition.transport import Envelope +from backend.core.archive import archive_completed_project +from backend.core.diamond import Diamond from backend.core.engine_lease import ( claim_engine_lease, release_engine_lease, ) -from backend.core.archive import archive_completed_project -from backend.core.diamond import Diamond from backend.core.ipc import ( accept_verified_flag, lock_project_completion, @@ -42,6 +46,7 @@ ) from backend.core.project_manager import ProjectManager from backend.core.resource_manager import ResourceManager +from backend.core.wp_writer import persist_validated_writeup from backend.mcp.mcp_client import MCPClient, MCPRegistryTarget from backend.members.base_member import MemberDeps from backend.members.factory import create_member @@ -52,9 +57,8 @@ ) from backend.sandbox.errors import SandboxStartupError from backend.sandbox.task_sandbox import member_workdir, task_container_name -from backend.core.wp_writer import persist_validated_writeup -from backend.competition.transport import Envelope +_recovery_logger = logging.getLogger(__name__) @dataclass(slots=True) class ReasonCheckpoint: @@ -177,7 +181,7 @@ def start_project_async(self, project_id: str) -> dict[str, str | None]: status = self.lifecycle.status(project_id) if status is None: raise ValueError(f"project not found: {project_id}") - if status in ("flag_found", "solved"): + if status in ("flag_found", "solved", "completed"): return self.runtime_status(project_id) if status == "running" and not self._startup_in_progress(project_id): with self._lock: @@ -225,6 +229,7 @@ def _run_project_startup(self, project_id: str, resume: bool) -> None: else: self.start_project(project_id) except Exception as exc: + _recovery_logger.debug('Error handled in _run_project_startup', exc_info=True) self._mark_project_startup_failure( project_id, exc, @@ -286,6 +291,7 @@ def _release_project_engine_lease(self, project_id: str) -> None: self.state.db, project_id, context[0], context[1], epoch=context[2] ) except Exception: + _recovery_logger.debug('Error handled in _release_project_engine_lease', exc_info=True) return def _ensure_project_lease(self, project_id: str) -> bool: @@ -444,7 +450,7 @@ def _mark_project_startup_failure( row = graph_store.get_project_row(connection, project_id) if row is not None: preserved_status = str(row["status"]) - if preserved_status not in ("flag_found", "solved"): + if preserved_status not in ("flag_found", "solved", "completed"): changed = ( preserved_status != target or row["runtime_phase"] != phase @@ -453,7 +459,7 @@ def _mark_project_startup_failure( connection.execute( "UPDATE projects SET status = %s, terminal_reason = %s, " "runtime_phase = %s, runtime_error = %s, updated_at = now() " - "WHERE id = %s AND status NOT IN ('flag_found', 'solved')", + "WHERE id = %s AND status NOT IN ('flag_found', 'solved', 'completed')", (target, detail, phase, detail, project_id), ) connection.execute( @@ -491,7 +497,7 @@ def _mark_project_startup_failure( error_kind=kind, error=detail, ) - elif preserved_status in ("flag_found", "solved"): + elif preserved_status in ("flag_found", "solved", "completed"): self.state.logger.project( f"{event}_flag_preserved", project_id, @@ -548,6 +554,11 @@ def _startup_in_progress(self, project_id: str) -> bool: with self._lock: return project_id in self._startup_project_ids + def _acquire_task(self, project_id: str) -> bool: + if self._competition_project(project_id): + return self.resources.acquire_coordinated_task(project_id) + return self.resources.acquire_task(project_id) + def startup_in_progress(self, project_id: str) -> bool: """Public read-only view for the competition coordinator.""" return self._startup_in_progress(project_id) @@ -572,6 +583,7 @@ def _redispatch_project(self, project_id: str) -> None: return self._dispatch_project(project_id) except Exception as exc: + _recovery_logger.debug('Error handled in _redispatch_project', exc_info=True) self.state.logger.project( "project_redispatch_failed", project_id, @@ -595,11 +607,11 @@ def start_project(self, project_id: str) -> None: return if status == "running": return - if status in ("flag_found", "solved"): + if status in ("flag_found", "solved", "completed"): self._release_project_lease(project_id) return self._clear_project_member_retries(project_id) - if not self.resources.acquire_task(project_id): + if not self._acquire_task(project_id): self._queue_project(project_id) self._set_runtime_phase(project_id, "queued_capacity") self.state.logger.project( @@ -704,7 +716,7 @@ def resume_project(self, project_id: str) -> None: self._release_project_lease(project_id) return self._clear_project_member_retries(project_id) - if not self.resources.acquire_task(project_id): + if not self._acquire_task(project_id): self._queue_project(project_id) self._set_runtime_phase(project_id, "queued_capacity") self.state.logger.project( @@ -788,6 +800,11 @@ def handle_report(self, project_id: str, report) -> None: if hasattr(report, "model_dump") else str(report), }, ) + if self._competition_project(project_id): + # Competition seats are allocated by CompetitionService. A report + # must not mint unseated solvers: they would be recorded as active + # agents and then blocked by the seat check at launch. + return available_slots = max( 0, self.state.config.runtime.max_members_per_report - self._project_running_future_count(project_id), @@ -849,11 +866,12 @@ def _publish_competition_recon( *, event_id: str, payload: dict[str, Any], - ) -> None: + receiver_member: str | None = None, + ) -> list[dict]: """Bridge durable Member reports into the same-challenge PUB/SUB path.""" competition = getattr(self.state, "competition", None) if competition is None or not callable(getattr(competition, "publish_recon", None)): - return + return [] now = datetime.now(timezone.utc) try: with self.state.db.connect() as connection: @@ -873,12 +891,15 @@ def _publish_competition_recon( if source_member is None or item["member"] == source_member ] if not senders: - return + return [] sender = senders[0] actual_source = sender["member"] + published = [] for receiver in assignments: if receiver["member"] == actual_source: continue + if receiver_member and receiver["member"] != receiver_member: + continue message = Envelope( run_id=sender["run_id"], challenge_id=sender["challenge_id"], @@ -890,18 +911,31 @@ def _publish_competition_recon( correlation_id=event_id, lease_epoch=sender["epoch"], created_at=now, - deadline_at=now + timedelta(seconds=30), + deadline_at=now + timedelta(seconds=300), message_type="recon", payload=payload, ) - competition.publish_recon(message) + receipt = competition.publish_recon(message) + published.append({"request_id": message.request_id, "receiver": receiver["member"], + "sequence": receipt.get("sequence") if isinstance(receipt, dict) else None}) + return published except Exception as exc: + _recovery_logger.debug('Error handled in _publish_competition_recon', exc_info=True) self.state.logger.project( "competition_recon_publish_failed", project_id, member=source_member, error=f"{type(exc).__name__}: {exc}", ) + return [] + + def _send_member_message(self, project_id: str, source: str, receiver: str, payload: dict) -> dict: + event_id = "member:" + hashlib.sha256(json.dumps([project_id, source, receiver, payload], sort_keys=True).encode()).hexdigest()[:32] + receipts = self._publish_competition_recon(project_id, source, event_id=event_id, + payload=payload, receiver_member=receiver) + if not receipts: + raise ValueError("recipient is not an active teammate on this challenge or message delivery is unavailable") + return {"published": True, "receipts": receipts} def _broadcast_fact(self, project_id: str, fact_id: str, source_member: str | None) -> None: with self.state.db.connect() as conn: @@ -963,6 +997,7 @@ def _competition_project(self, project_id: str) -> bool: (project_id,), ).fetchone()) except Exception: + _recovery_logger.debug('Error handled in _competition_project', exc_info=True) return False def _competition_running(self, project_id: str) -> bool: @@ -991,6 +1026,7 @@ def _competition_members(self, project_id: str) -> set[str] | None: ).fetchall() return {row["member"] for row in rows} except Exception: + _recovery_logger.debug('Error handled in _competition_members', exc_info=True) return None def _competition_preferred_member(self, project_id: str) -> str | None: @@ -1009,6 +1045,7 @@ def _competition_preferred_member(self, project_id: str) -> str | None: ).fetchone() return row["member"] if row else None except Exception: + _recovery_logger.debug('Error handled in _competition_preferred_member', exc_info=True) return None def _competition_assignment(self, project_id: str, member_name: str) -> dict | None: @@ -1016,7 +1053,7 @@ def _competition_assignment(self, project_id: str, member_name: str) -> dict | N try: with self.state.db.connect() as connection: return connection.execute( - """SELECT a.* FROM competition_assignments a + """SELECT a.*,c.instance_generation FROM competition_assignments a JOIN competition_challenges c ON c.id=a.challenge_id JOIN competition_runs r ON r.id=a.run_id WHERE c.project_id=%s AND a.member=%s AND a.role<>'wp' @@ -1025,6 +1062,7 @@ def _competition_assignment(self, project_id: str, member_name: str) -> dict | N (project_id, member_name), ).fetchone() except Exception: + _recovery_logger.debug('Error handled in _competition_assignment', exc_info=True) return None def _global_active_members_locked(self) -> set[str]: @@ -1173,6 +1211,9 @@ def _launch_member( max_actions_per_task=self.state.config.runtime.max_member_actions_per_task, continuous=self._competition_project(project_id), on_report=self.handle_report, + on_fact=self._broadcast_fact, + evidence_dir=self.state.projects_dir / project_id / "evidence", + on_message=self._send_member_message, on_flag=self.on_flag_found, lease_owner=lease_owner, lease_token=lease_token, @@ -1350,12 +1391,28 @@ def _release_member_assignment( def _run_member(self, project_id, member, intent_id, category, is_initial): try: + seat = getattr(member.deps, "competition_assignment", None) + if seat is not None: + member.deps.competition_store.append_run_event( + seat["run_id"], "member.started", + {"assignment_id": seat["id"], "challenge_id": seat["challenge_id"], + "session_id": seat["session_id"], "member": member.name, + "intent_id": intent_id, "epoch": seat["epoch"]}, + ) return asyncio.run( member.solve_async(project_id, intent_id, category, is_initial=is_initial) ) except Exception as exc: + _recovery_logger.debug('Error handled in _run_member', exc_info=True) self.state.logger.project("member_crash", project_id, member=member.name, error=str(exc)) finally: + # Release the member's resources on the thread that created them. + # In particular the competition recon subscriber owns a ZeroMQ + # context plus socket; dropping it to the cyclic GC instead lets a + # later collection block forever in zmq_ctx_term while the socket + # is still open, freezing the whole interpreter. + with suppress(Exception): + member.stop() with self._lock: self._members.get(project_id, {}).pop(member.name, None) with self.state.db.connect() as conn: @@ -1401,7 +1458,7 @@ def stop_project(self, project_id: str) -> None: if status in ("created", "running"): with suppress(LifecycleError): self.lifecycle.transition(project_id, "stopped") - phase = "solved" if status == "solved" else "stopped" + phase = status if status in {"solved", "completed"} else "stopped" self._set_runtime_phase(project_id, phase) self._release_project_lease(project_id) @@ -1518,18 +1575,18 @@ def _run_platform_verdict(self, project_id: str) -> None: try: self._judge_pending_submissions(project_id) except Exception as exc: + _recovery_logger.debug('Error handled in _run_platform_verdict', exc_info=True) self.state.logger.project( "platform_verdict_crashed", project_id, error=f"{type(exc).__name__}: {exc}", ) - with suppress(Exception): - with self.state.db.connect() as conn: - conn.execute( - "UPDATE flag_submissions SET status = 'unknown' " - "WHERE project_id = %s AND status = 'judging'", - (project_id,), - ) + with suppress(Exception), self.state.db.connect() as conn: + conn.execute( + "UPDATE flag_submissions SET status = 'unknown' " + "WHERE project_id = %s AND status = 'judging'", + (project_id,), + ) finally: with self._lock: self._completing.discard(project_id) @@ -1628,6 +1685,7 @@ def _submit_to_platform(self, client, external_id: str, flag: str) -> tuple[str, except Ret2ShellError as exc: return "unknown", f"platform submit failed: {exc}" except Exception as exc: + _recovery_logger.debug('Error handled in _submit_to_platform', exc_info=True) return "unknown", f"{type(exc).__name__}: {exc}" solved = submission.get("solved") if isinstance(submission, dict) else None if solved is True: @@ -1664,6 +1722,7 @@ def _reject_submission(self, project_id: str, project_row, submission, detail: s source="platform", ) except Exception as exc: + _recovery_logger.debug('Error handled in _reject_submission', exc_info=True) self.state.logger.project( "rejected_flag_memory_failed", project_id, @@ -1709,6 +1768,7 @@ def _drain_postprocess_jobs(self) -> None: try: prepared = self._prepare_postprocess_job(job) except Exception as exc: + _recovery_logger.debug('Error handled in _drain_postprocess_jobs', exc_info=True) heartbeat_stop.set() heartbeat.join() error = f"{type(exc).__name__}: {exc}" @@ -1736,6 +1796,7 @@ def _drain_postprocess_jobs(self) -> None: except PostprocessLeaseLost: self._log_stale_postprocess_job(job, stage="commit") except Exception as exc: + _recovery_logger.debug('Error handled in _drain_postprocess_jobs', exc_info=True) error = f"{type(exc).__name__}: {exc}" with self.state.db.connect() as connection: failed = fail_job(connection, job, error) @@ -1766,6 +1827,7 @@ def _heartbeat_postprocess_job(self, job, stop: threading.Event, lost: threading lease_seconds=self._POSTPROCESS_LEASE_SECONDS, ) except Exception as exc: + _recovery_logger.debug('Error handled in _heartbeat_postprocess_job', exc_info=True) self.state.logger.project( "postprocess_heartbeat_error", job.project_id, @@ -1843,6 +1905,7 @@ def _commit_postprocess_job(self, job, prepared): try: self.state.memory._mirror_to_disk(memory) except Exception as exc: + _recovery_logger.debug('Error handled in _commit_postprocess_job', exc_info=True) self.state.logger.project( "memory_mirror_failed", job.project_id, @@ -1881,9 +1944,11 @@ def _run_loop(self) -> None: try: self._tick() except Exception as exc: + _recovery_logger.debug('Error handled in _run_loop', exc_info=True) try: self.state.logger.project("scheduler_error", "system", error=str(exc)) except Exception: + _recovery_logger.debug('Error handled in _run_loop', exc_info=True) print(f"[ipc-orchestrator] tick failed: {type(exc).__name__}: {exc}", file=sys.stderr, flush=True) self._stop.wait(interval) @@ -1917,6 +1982,7 @@ def _competition_project_ownership( else: terminal.add(row["project_id"]) except Exception: + _recovery_logger.debug('Error handled in _competition_project_ownership', exc_info=True) return set(), set() return active, terminal @@ -1938,11 +2004,18 @@ def _tick(self) -> None: ) self._initialize_reason_checkpoints(summaries) for summary in summaries: + if summary.status == "completed": + with self._lock: + needs_cleanup = summary.runtime_phase != "completed" or summary.id in self._project_leases or any(not member._stop.is_set() for member in self._members.get(summary.id, {}).values()) + if needs_cleanup: + self.stop_project(summary.id) + continue if summary.id in competition_projects: if summary.status == "running" and not self._startup_in_progress(summary.id): try: self._dispatch_project(summary.id) except Exception as exc: + _recovery_logger.debug('Error handled in _tick', exc_info=True) self.state.logger.project("scheduler_error", summary.id, error=str(exc)) continue if summary.id in competition_terminal_projects: @@ -1967,12 +2040,13 @@ def _tick(self) -> None: continue if self._startup_in_progress(summary.id): continue - if not self.resources.acquire_task(summary.id): + if not self._acquire_task(summary.id): self._queue_project(summary.id) continue try: self._dispatch_project(summary.id) except Exception as exc: + _recovery_logger.debug('Error handled in _tick', exc_info=True) self._mark_project_startup_failure( summary.id, exc, @@ -1987,6 +2061,7 @@ def _schedule_running_recovery(self, project_id: str) -> None: try: future = self.startup_executor.submit(self._recover_running_project, project_id) except Exception as exc: + _recovery_logger.debug('Error handled in _schedule_running_recovery', exc_info=True) with self._lock: self._startup_project_ids.discard(project_id) self._mark_project_startup_failure( @@ -2005,7 +2080,7 @@ def _recover_running_project(self, project_id: str) -> None: try: if not self._ensure_project_lease(project_id): return - if not self.resources.acquire_task(project_id): + if not self._acquire_task(project_id): self._queue_project(project_id) self._set_runtime_phase(project_id, "queued_capacity") self._release_project_lease(project_id) @@ -2033,6 +2108,7 @@ def _recover_running_project(self, project_id: str) -> None: "project_recovered", project_id, instance=self.state.instance_id ) except Exception as exc: + _recovery_logger.debug('Error handled in _recover_running_project', exc_info=True) self._mark_project_startup_failure( project_id, exc, @@ -2053,7 +2129,7 @@ def _dispatch_project(self, project_id: str) -> None: return with self.state.db.connect() as conn: detail = graph_store.project_detail(conn, project_id) - if detail is None: + if detail is None or detail.project.status in {"solved", "completed", "stopped", "timeout", "failed", "infra_error"}: return running = {intent_id for (pid, intent_id), fut in project_tasks.items() if pid == project_id and not fut.done()} open_intents = [i for i in detail.intents if i.to is None] @@ -2235,20 +2311,17 @@ def _drain_pending_projects(self) -> None: if not self._pending_projects: return project_id = self._pending_projects[0] - if not self.resources.can_admit_task(project_id): + if not self._competition_project(project_id) and not self.resources.can_admit_task(project_id): return self._remove_queued_project(project_id) status = self.lifecycle.status(project_id) - if status == "created": - self.start_project_async(project_id) - elif status == "stopped": + if status == "created" or status == "stopped": self.start_project_async(project_id) - elif status == "running": - if ( - not self._startup_in_progress(project_id) - and self._ensure_project_lease(project_id) - ): - self._schedule_running_recovery(project_id) + elif status == "running" and ( + not self._startup_in_progress(project_id) + and self._ensure_project_lease(project_id) + ): + self._schedule_running_recovery(project_id) def _select_member_for_intent(self, project_id: str, detail, intent, active: set[str]) -> str | None: preferred = self._intent_member_candidates(detail, intent) @@ -2372,6 +2445,8 @@ def _initialize_reason_checkpoints(self, summaries) -> None: ) def _reason_trigger(self, detail) -> str | None: + if detail.project.category == "pentest": + return "pentest_graph_changed" open_intent_count = self._project_open_intent_count(detail) checkpoint = self._reason_checkpoints.get(detail.project.id) if checkpoint is None: @@ -2419,6 +2494,7 @@ def _reap_finished_futures(self) -> None: try: result = future.result() except Exception as exc: + _recovery_logger.debug('Error handled in _reap_finished_futures', exc_info=True) crashed = True failure_error = f"{type(exc).__name__}: {exc}" self.state.logger.project( @@ -2490,10 +2566,76 @@ def _reap_finished_futures(self) -> None: self.state.logger.project("member_task_concluded", project_id, intent=intent_id, fact=result.fact_id) if result.fact_id is not None: self._broadcast_fact(project_id, result.fact_id, self._intent_worker(project_id, intent_id)) + self._flag_followup_from_fact(project_id, result.fact_id) elif result.status == "flag": self._clear_member_stalls(project_id, intent_id) self.state.logger.project("member_task_flag", project_id, intent=intent_id, flag=result.flag) + def _flag_followup_from_fact(self, project_id: str, fact_id: str) -> None: + """Turn a flag-shaped conclusion into an explicit submit tool task. + + A prose ``conclude`` is evidence only. The follow-up intent keeps the + submission contract enforceable while avoiding an automatic network + submission when a conclusion contains multiple candidates. + """ + with self.state.db.connect() as connection: + connection.execute( + "SELECT id FROM projects WHERE id=%s FOR UPDATE", (project_id,) + ) + row = connection.execute( + "SELECT description FROM facts WHERE project_id=%s AND id=%s", + (project_id, fact_id), + ).fetchone() + if row is None: + return + text = row["description"] + if re.search(r"\b(?:example|placeholder|sample flag)\b|示例|占位", text, re.IGNORECASE): + return + candidates = sorted(set(re.findall(r"(?:[A-Za-z0-9_]*ctf|flag)\{[^}\n]{1,400}\}", text, re.IGNORECASE))) + if len(candidates) != 1: + if len(candidates) > 1: + self.state.logger.project( + "flag.multiple_candidates_in_fact", project_id, fact_id=fact_id, + candidates=len(candidates), + ) + return + candidate = candidates[0] + existing = connection.execute( + "SELECT 1 FROM flag_submissions WHERE project_id=%s AND normalized_flag=%s LIMIT 1", + (project_id, candidate), + ).fetchone() + if existing: + return + description = "Submit the verified flag with the submit_flag tool; do not conclude prose" + open_intent = edge_store.find_similar_open_intent( + connection, project_id, [fact_id], description, + ) + if open_intent is None: + edge_store.create_intent( + connection, project_id, [fact_id], + description, + "orchestrator", + ) + self.state.logger.project( + "flag.found_in_fact_without_submission", project_id, + fact_id=fact_id, candidate=candidate, + ) + competition = connection.execute( + """SELECT c.run_id,c.id AS challenge_id FROM competition_challenges c + WHERE c.project_id=%s ORDER BY c.id LIMIT 1""", + (project_id,), + ).fetchone() + if competition: + connection.execute( + """INSERT INTO competition_run_events (run_id,kind,payload) + VALUES (%s,'flag.found_in_fact_without_submission',%s)""", + (competition["run_id"], Jsonb({ + "challenge_id": competition["challenge_id"], + "fact_id": fact_id, + "candidate": candidate, + })), + ) + def _intent_worker(self, project_id: str, intent_id: str) -> str | None: with self.state.db.connect() as conn: row = edge_store.get_intent(conn, project_id, intent_id) diff --git a/backend/core/postprocess_store.py b/backend/core/postprocess_store.py index 4e7d088..dca3db6 100644 --- a/backend/core/postprocess_store.py +++ b/backend/core/postprocess_store.py @@ -4,7 +4,6 @@ from dataclasses import dataclass from typing import Any - POSTPROCESS_KINDS = ("writeup", "memory", "archive") @@ -17,7 +16,14 @@ class PostprocessJob: lease_token: str +def _lock_project(connection: Any, project_id: str) -> None: + connection.execute( + "SELECT id FROM projects WHERE id=%s FOR NO KEY UPDATE", (project_id,), + ).fetchone() + + def enqueue_postprocess(connection: Any, project_id: str) -> None: + _lock_project(connection, project_id) for kind in POSTPROCESS_KINDS: connection.execute( """ @@ -39,21 +45,22 @@ def claim_next_job( ) -> PostprocessJob | None: row = connection.execute( """ - SELECT id - FROM postprocess_jobs - WHERE status IN ('pending', 'retry') - AND next_attempt_at <= now() - AND (lease_expires_at IS NULL OR lease_expires_at < now()) + SELECT job.id, job.project_id + FROM postprocess_jobs job + JOIN projects project ON project.id = job.project_id + WHERE job.status IN ('pending', 'retry') + AND job.next_attempt_at <= now() + AND (job.lease_expires_at IS NULL OR job.lease_expires_at < now()) AND ( - kind <> 'archive' + job.kind <> 'archive' OR EXISTS ( SELECT 1 FROM postprocess_jobs writeup - WHERE writeup.project_id = postprocess_jobs.project_id + WHERE writeup.project_id = job.project_id AND writeup.kind = 'writeup' AND writeup.status = 'completed' ) ) - ORDER BY created_at, id - FOR UPDATE SKIP LOCKED + ORDER BY job.created_at, job.id + FOR NO KEY UPDATE OF project SKIP LOCKED LIMIT 1 """ ).fetchone() @@ -67,11 +74,13 @@ def claim_next_job( lease_token = %s, lease_expires_at = now() + make_interval(secs => %s), updated_at = now() - WHERE id = %s + WHERE id = %s AND status IN ('pending', 'retry') RETURNING id, project_id, kind, attempts, lease_token """, (worker_id, token, lease_seconds, row["id"]), ).fetchone() + if claimed is None: + return None refresh_project_status(connection, claimed["project_id"]) return PostprocessJob(**claimed) @@ -109,6 +118,7 @@ def lock_job( database commit phase concurrently. """ + _lock_project(connection, job.project_id) row = connection.execute( """ SELECT id @@ -139,6 +149,7 @@ def complete_job( *, lease_locked: bool = False, ) -> bool: + _lock_project(connection, job.project_id) live_clause = "" if lease_locked else ( " AND lease_expires_at IS NOT NULL AND lease_expires_at > now()" ) @@ -172,6 +183,7 @@ def complete_existing_job( overwrite another worker's result. """ + _lock_project(connection, project_id) if lease_token is None: changed = connection.execute( """ @@ -215,6 +227,7 @@ def fail_job( *, max_attempts: int = 4, ) -> bool: + _lock_project(connection, job.project_id) delay = min(900, 15 * (2 ** max(0, job.attempts - 1))) changed = connection.execute( """ @@ -236,11 +249,21 @@ def fail_job( def recover_expired_jobs(connection: Any) -> int: cursor = connection.execute( """ - UPDATE postprocess_jobs + WITH expired_projects AS MATERIALIZED ( + SELECT project.id FROM projects project + WHERE EXISTS ( + SELECT 1 FROM postprocess_jobs job WHERE job.project_id=project.id + AND job.status='running' AND job.lease_expires_at bool: def acquire_task(self, project_id: str) -> bool: return self.limiter.acquire(project_id) + def acquire_coordinated_task(self, project_id: str) -> bool: + """Track cleanup for work already admitted by the competition scheduler.""" + return self.limiter.acquire(project_id, externally_scheduled=True) + def reclaim_orphaned_projects(self, active_project_ids: set[str]) -> list[str]: orphaned = sorted(pid for pid in self.pool.active_projects() if pid not in active_project_ids) for project_id in orphaned: diff --git a/backend/core/state.py b/backend/core/state.py index f39ebac..dcf824c 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os import re import shutil @@ -9,9 +10,9 @@ from pathlib import Path from backend.blackboard import graph_store +from backend.blackboard.db import Database from backend.core.config import AppConfig, load_config, save_config from backend.core.logging_util import IPCLogger -from backend.blackboard.db import Database from backend.mcp.mcp_client import MCPRegistry from backend.memory.memory_mcp import build_memory_mcp from backend.memory.memory_store import MemoryStore @@ -20,10 +21,11 @@ from backend.sandbox.container_pool import ContainerPool from backend.sandbox.network_manager import NetworkManager from backend.sandbox.resource_limiter import TaskSlotLimiter -from backend.tools.tool_mcp import build_tool_search_mcp from backend.tools.catalog import ToolCatalog +from backend.tools.tool_mcp import build_tool_search_mcp from backend.tools.tool_registry import ToolRegistry +_recovery_logger = logging.getLogger(__name__) def _env_flag(name: str) -> bool: value = os.environ.get(name, "").strip().lower() @@ -90,8 +92,8 @@ def __init__(self, root: str | Path = ".", config_dir: Path | None = None): self.network = NetworkManager(backend=self.config.runtime.sandbox_backend) self.mcps = MCPRegistry() - from backend.skills.store import SkillStore from backend.skills.mcp import build_skills_mcp + from backend.skills.store import SkillStore self.skills = SkillStore(self.artifact_root / "skills", Path("/opt/ipc-skills")) self.skills.bootstrap() self.mcps.register(build_skills_mcp(self.skills)) @@ -140,12 +142,20 @@ def _clean_runtime_state(self) -> None: def reload_config(self) -> None: self.config = load_config(self.config_dir) self.logger.set_enabled(self.config.log_enabled) + self.limiter.update_limits( + max_concurrent_tasks=self.config.limits.max_concurrent_tasks, + total_cpu=self.config.limits.total_cpu, + ) if self.orchestrator is not None: self.orchestrator.reload_config() def save_config(self) -> None: save_config(self.config, self.config_dir) self.logger.set_enabled(self.config.log_enabled) + self.limiter.update_limits( + max_concurrent_tasks=self.config.limits.max_concurrent_tasks, + total_cpu=self.config.limits.total_cpu, + ) if self.orchestrator is not None: self.orchestrator.reload_config() @@ -164,6 +174,7 @@ def project_log_filename(self, project_id: str) -> str | None: with self.db.connect() as conn: return graph_store.project_log_filename(conn, project_id) except Exception: + _recovery_logger.debug('Error handled in project_log_filename', exc_info=True) return project_id def attachments_dir(self, project_id: str) -> Path: diff --git a/backend/core/telemetry.py b/backend/core/telemetry.py new file mode 100644 index 0000000..ae6d97b --- /dev/null +++ b/backend/core/telemetry.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import asyncio +import faulthandler +import os +import threading +import time +from contextlib import suppress +from pathlib import Path + + +class RuntimeTelemetry: + def __init__(self, root: Path, *, stall_seconds: float = 10): + self.root = root + self.stall_seconds = stall_seconds + self.last_heartbeat = time.monotonic() + self.loop_lag_seconds = 0.0 + self.max_loop_lag_seconds = 0.0 + self._done = threading.Event() + self._task = None + self._thread = None + + def start(self): + self.last_heartbeat = time.monotonic() + self._task = asyncio.create_task(self._heartbeat()) + self._thread = threading.Thread(target=self._watch, name="ipc-loop-diagnostics", daemon=True) + self._thread.start() + + async def _heartbeat(self): + while not self._done.is_set(): + before = time.monotonic() + await asyncio.sleep(.5) + now = time.monotonic() + self.loop_lag_seconds = max(0, now - before - .5) + self.max_loop_lag_seconds = max(self.max_loop_lag_seconds, self.loop_lag_seconds) + self.last_heartbeat = now + + def _watch(self): + last_dump = 0.0 + while not self._done.wait(.5): + now = time.monotonic() + if now-self.last_heartbeat >= self.stall_seconds and now-last_dump >= 60: + last_dump = now + directory = self.root / "logs" / "runtime" + directory.mkdir(parents=True, exist_ok=True) + with (directory / "event-loop-stalls.log").open("ab") as stream: + stream.write(f"pid={os.getpid()} monotonic={now:.6f} stalled_seconds={now-self.last_heartbeat:.3f}\n".encode()) + stream.flush() + faulthandler.dump_traceback(file=stream, all_threads=True) + + def snapshot(self, state=None) -> dict: + result = {"pid": os.getpid(), "threads": threading.active_count(), + "cpu_seconds": round(time.process_time(), 6), + "event_loop_lag_seconds": round(self.loop_lag_seconds, 6), + "max_event_loop_lag_seconds": round(self.max_loop_lag_seconds, 6), + "heartbeat_age_seconds": round(time.monotonic()-self.last_heartbeat, 6)} + try: + import resource + result["peak_rss_kib"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + except ImportError: + result["peak_rss_kib"] = None + orchestrator = getattr(state, "orchestrator", None) + for name in ("executor", "startup_executor"): + queue = getattr(getattr(orchestrator, name, None), "_work_queue", None) + if queue is not None: + result[f"{name}_queued"] = queue.qsize() + competition = getattr(state, "competition", None) + result["competition_phases"] = dict(getattr(competition, "phase_metrics", {})) + return result + + async def close(self): + self._done.set() + if self._task: + self._task.cancel() + with suppress(asyncio.CancelledError): + await self._task + if self._thread: + self._thread.join(timeout=1) diff --git a/backend/core/validation.py b/backend/core/validation.py new file mode 100644 index 0000000..e60401e --- /dev/null +++ b/backend/core/validation.py @@ -0,0 +1,4 @@ + + +class InputValidationError(ValueError): + """Input violates its schema; preserve the ValueError API/recovery contract.""" diff --git a/backend/core/wp_writer.py b/backend/core/wp_writer.py index f149f7b..78bcfc6 100644 --- a/backend/core/wp_writer.py +++ b/backend/core/wp_writer.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import logging import os import re import tempfile @@ -10,6 +11,7 @@ from backend.blackboard import graph_store +_recovery_logger = logging.getLogger(__name__) WRITEUP_REQUEST = ( "CTF 题目已解出。请撰写一份 Markdown writeup,并在其中包含一个完整的 Python 利用脚本。" @@ -261,7 +263,12 @@ def persist_validated_writeup( raise ValueError("writeup content must be non-empty") if len(content) > 1_000_000: raise ValueError("writeup content is limited to 1,000,000 characters") - errors = validate_writeup(content, expected_flag=expected_flag, require_complete=True) + project = graph_store.get_project_row(connection, project_id) + if project and project.get("completion_policy") == "goals": + from backend.pentest.report import validate_report + errors = validate_report(content) + else: + errors = validate_writeup(content, expected_flag=expected_flag, require_complete=True) if errors: raise ValueError("invalid writeup: " + "; ".join(errors)) @@ -320,6 +327,7 @@ def _generate_writeup(detail, generator: Callable[[str], str], evidence_logs: Ma try: content = _unwrap_markdown(generator(prompt)) except Exception as exc: + _recovery_logger.debug('Error handled in _generate_writeup', exc_info=True) failures.append(f"generator call {attempt + 1}: {type(exc).__name__}: {exc}") continue errors = validate_writeup(content, expected_flag=detail.project.flag, require_complete=True) diff --git a/backend/mcp/mcp_client.py b/backend/mcp/mcp_client.py index 4193fa7..1c6de38 100644 --- a/backend/mcp/mcp_client.py +++ b/backend/mcp/mcp_client.py @@ -8,12 +8,14 @@ from contextlib import AsyncExitStack, asynccontextmanager from datetime import timedelta from pathlib import Path +from types import TracebackType from typing import Any from mcp import ClientSession, StdioServerParameters, types from mcp.client.stdio import stdio_client from mcp.server.fastmcp import FastMCP from mcp.shared.memory import create_connected_server_and_client_session +from typing_extensions import Self MCPClientTarget = FastMCP | StdioServerParameters @@ -84,7 +86,7 @@ def session(self) -> ClientSession: raise RuntimeError("MCPClient must be used as an async context manager") return self._session - async def __aenter__(self) -> MCPClient: + async def __aenter__(self) -> Self: if self._stack is not None: raise RuntimeError("MCPClient session is already open") stack = AsyncExitStack() @@ -111,7 +113,10 @@ async def __aenter__(self) -> MCPClient: self._session = session return self - async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + async def __aexit__( + self, exc_type: type[BaseException] | None, + exc: BaseException | None, tb: TracebackType | None, + ) -> None: stack, self._stack = self._stack, None self._session = None if stack is not None: @@ -156,12 +161,15 @@ def __init__(self, targets: Mapping[str, MCPRegistryTarget]) -> None: self._clients: dict[str, MCPClient] = {} self._open = False - async def __aenter__(self) -> MCPRegistrySession: + async def __aenter__(self) -> Self: await self._stack.__aenter__() self._open = True return self - async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + async def __aexit__( + self, exc_type: type[BaseException] | None, + exc: BaseException | None, tb: TracebackType | None, + ) -> None: self._open = False self._clients.clear() await self._stack.__aexit__(exc_type, exc, tb) diff --git a/backend/mcp/mcp_server.py b/backend/mcp/mcp_server.py index b0fef87..29577b6 100644 --- a/backend/mcp/mcp_server.py +++ b/backend/mcp/mcp_server.py @@ -1,9 +1,10 @@ from __future__ import annotations import argparse +from collections.abc import Callable from contextlib import asynccontextmanager from pathlib import Path -from typing import Any, Callable, Literal +from typing import Any, Literal from mcp.server.fastmcp import FastMCP @@ -66,7 +67,10 @@ def build_mcp_server(name: str, root: str | Path = ".", category: str = "misc") lifespan=close_lifespan(store.close), ) if name in {"tool_search", "tools"}: - from backend.tools.tool_mcp import build_category_tools_mcp, build_tool_search_mcp + from backend.tools.tool_mcp import ( + build_category_tools_mcp, + build_tool_search_mcp, + ) from backend.tools.tool_registry import ToolRegistry registry = ToolRegistry().load() diff --git a/backend/mcp/reverse_mcp.py b/backend/mcp/reverse_mcp.py index ff9cce1..f0efd2d 100644 --- a/backend/mcp/reverse_mcp.py +++ b/backend/mcp/reverse_mcp.py @@ -2,16 +2,20 @@ import asyncio import json +import logging import os import subprocess import sys import tempfile +from collections.abc import Iterator from pathlib import Path -from typing import Any, Iterator +from typing import Any from backend.mcp.mcp_server import MCPServer, create_mcp_server from backend.mcp.shared import _tool_unavailable +_recovery_logger = logging.getLogger(__name__) + _JVM_STARTED = False _DEFAULT_GHIDRA_INSTALL_DIR = Path("/opt/ghidra") _DEFAULT_GHIDRA_JAVA_HOME = Path("/opt/java21") @@ -115,6 +119,7 @@ def _r2_cmd_sync(binary: str, cmd: str) -> dict[str, Any]: handle.quit() return {"available": True, "binary": str(path), "command": cmd, "output": output} except Exception as exc: + _recovery_logger.debug('Error handled in _r2_cmd_sync', exc_info=True) return _tool_unavailable("reverse.r2_cmd", str(exc), binary=binary, command=cmd) @@ -172,6 +177,7 @@ def _r2_decompile_sync(binary: str, function: str) -> dict[str, Any]: "output": output, } except Exception as exc: + _recovery_logger.debug('Error handled in _r2_decompile_sync', exc_info=True) return _tool_unavailable( "reverse.decompile.r2", str(exc), @@ -247,6 +253,7 @@ def _decompile_sync(binary: str, function: str, timeout: int) -> dict[str, Any]: function=function, ) except Exception as exc: + _recovery_logger.debug('Error handled in _decompile_sync', exc_info=True) fallback = _r2_decompile_sync(str(path), function) if fallback.get("available"): return { @@ -276,6 +283,7 @@ def _decompile_all_sync(binary: str, limit: int, timeout: int) -> dict[str, Any] limit=limit, ) except Exception as exc: + _recovery_logger.debug('Error handled in _decompile_all_sync', exc_info=True) return _tool_unavailable("reverse.decompile_all", str(exc), binary=binary) @@ -284,6 +292,7 @@ def _list_functions_sync(binary: str, timeout: int = 90) -> dict[str, Any]: path = _ensure_binary(binary) return _run_ghidra_worker("list_functions", str(path), timeout) except Exception as exc: + _recovery_logger.debug('Error handled in _list_functions_sync', exc_info=True) return _tool_unavailable("reverse.list_functions", str(exc), binary=binary) @@ -294,6 +303,7 @@ def _strings_sync(binary: str, min_len: int, timeout: int = 90) -> dict[str, Any "strings", str(path), timeout, min_len=min_len ) except Exception as exc: + _recovery_logger.debug('Error handled in _strings_sync', exc_info=True) return _tool_unavailable("reverse.strings", str(exc), binary=binary) @@ -310,12 +320,13 @@ def _disassemble_sync( count=count, ) except Exception as exc: + _recovery_logger.debug('Error handled in _disassemble_sync', exc_info=True) return _tool_unavailable("reverse.disassemble", str(exc), binary=binary, address=addr) def _run_command(command: list[str], timeout: int = 60) -> tuple[bool, str]: try: - result = subprocess.run(command, capture_output=True, text=True, timeout=timeout) + result = subprocess.run(command, capture_output=True, text=True, timeout=timeout, check=False) except (OSError, subprocess.TimeoutExpired) as exc: return False, str(exc) output = "\n".join(part for part in (result.stdout, result.stderr) if part).strip() diff --git a/backend/mcp/reverse_worker.py b/backend/mcp/reverse_worker.py index fe940ec..868d458 100644 --- a/backend/mcp/reverse_worker.py +++ b/backend/mcp/reverse_worker.py @@ -9,10 +9,12 @@ import argparse import json +import logging import time from pathlib import Path from typing import Any +_recovery_logger = logging.getLogger(__name__) def _remaining(deadline: float) -> int: return max(1, int(deadline - time.monotonic())) @@ -71,6 +73,7 @@ def _run( program, selected, _remaining(deadline) ) except Exception as exc: + _recovery_logger.debug('Error handled in _run', exc_info=True) results.append( { "name": str(selected.getName()), @@ -175,6 +178,7 @@ def main(argv: list[str] | None = None) -> int: project_dir=args.project_dir, ) except Exception as exc: + _recovery_logger.debug('Error handled in main', exc_info=True) result = { "available": False, "error": f"{type(exc).__name__}: {exc}", diff --git a/backend/mcp/shared.py b/backend/mcp/shared.py index 7d029bc..44873dc 100644 --- a/backend/mcp/shared.py +++ b/backend/mcp/shared.py @@ -2,22 +2,26 @@ import asyncio import atexit -from collections import deque -from contextlib import asynccontextmanager, suppress -from datetime import datetime, timezone import hashlib import json +import logging import os import re import time import uuid +from collections import deque +from contextlib import asynccontextmanager, suppress +from datetime import datetime, timezone from html.parser import HTMLParser from pathlib import Path from typing import Any, Literal from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +import anyio + from backend.mcp.mcp_server import MCPServer, create_mcp_server +_recovery_logger = logging.getLogger(__name__) _SENSITIVE_KEY_RE = re.compile( r"(?:^|[-_])(?:authorization|cookie|password|passwd|proxy-authorization|set-cookie|" @@ -338,7 +342,10 @@ def page_id(self, page: Any | None = None) -> str: async def close(self) -> None: async with self._lock: if self._pending_tasks: - await asyncio.gather(*tuple(self._pending_tasks), return_exceptions=True) + pending = tuple(self._pending_tasks) + for task in pending: + task.cancel() + await asyncio.gather(*pending, return_exceptions=True) if self._context is not None: await self._context.close() if self._browser is not None: @@ -450,6 +457,7 @@ def _request_page_id(self, request: Any) -> str: try: return self.page_id(request.frame.page) except Exception: + _recovery_logger.debug('Error handled in _request_page_id', exc_info=True) return self._active_page_id or "main" def _network_request_allowed(self, request: Any) -> bool: @@ -484,6 +492,7 @@ async def _record_response(self, response: Any) -> None: try: headers = await response.all_headers() except Exception: + _recovery_logger.debug('Error handled in _record_response', exc_info=True) headers = {} content_type = str(headers.get("content-type", "")).split(";", 1)[0].strip().lower() preview = None @@ -499,6 +508,7 @@ async def _record_response(self, response: Any) -> None: self._network_preview_limit, ) except Exception: + _recovery_logger.debug('Error handled in _record_response', exc_info=True) preview = None started = self._request_started.pop(id(request), None) redirected_from = getattr(request, "redirected_from", None) @@ -721,7 +731,7 @@ def _close_browser_at_exit(browser: _BrowserSession) -> None: try: asyncio.run(browser.close()) except Exception: - pass + _recovery_logger.debug('Error handled in _close_browser_at_exit', exc_info=True) def build_browser_mcp(browser: _BrowserSession | None = None) -> MCPServer: @@ -734,7 +744,8 @@ async def lifespan(_server): finally: close = getattr(browser, "close", None) if callable(close): - await close() + with anyio.CancelScope(shield=True): + await close() server = create_mcp_server( "browser", "Stateful Playwright browser tools", lifespan=lifespan @@ -780,6 +791,7 @@ async def navigate( **snapshot, } except Exception as exc: + _recovery_logger.debug('Error handled in navigate', exc_info=True) return _tool_unavailable("browser.navigate", str(exc), url=_redact_url(url), page_id=page_id) @server.tool(name="click", description="Click an element and return the resulting page state.") @@ -790,7 +802,7 @@ async def click(selector: str, page_id: str | None = None) -> dict[str, Any]: try: await page.wait_for_load_state("domcontentloaded", timeout=5000) except Exception: - pass + _recovery_logger.debug('Error handled in click', exc_info=True) return { "available": True, "page_id": _browser_page_id(browser, page), @@ -798,6 +810,7 @@ async def click(selector: str, page_id: str | None = None) -> dict[str, Any]: **await _page_snapshot(page), } except Exception as exc: + _recovery_logger.debug('Error handled in click', exc_info=True) return _tool_unavailable("browser.click", str(exc), selector=selector) @server.tool(name="fill", description="Fill an input or textarea without submitting it.") @@ -813,6 +826,7 @@ async def fill(selector: str, value: str, page_id: str | None = None) -> dict[st "url": page.url, } except Exception as exc: + _recovery_logger.debug('Error handled in fill', exc_info=True) return _tool_unavailable("browser.fill", str(exc), selector=selector) @server.tool(name="press", description="Press a keyboard key on the current page.") @@ -827,6 +841,7 @@ async def press(key: str, page_id: str | None = None) -> dict[str, Any]: **await _page_snapshot(page), } except Exception as exc: + _recovery_logger.debug('Error handled in press', exc_info=True) return _tool_unavailable("browser.press", str(exc), key=key) @server.tool(name="eval_js", description="Evaluate JavaScript in the current page context.") @@ -840,6 +855,7 @@ async def eval_js(script: str, page_id: str | None = None) -> dict[str, Any]: "url": page.url, } except Exception as exc: + _recovery_logger.debug('Error handled in eval_js', exc_info=True) return _tool_unavailable("browser.eval_js", str(exc)) @server.tool(name="get_content", description="Return rendered HTML and visible text for the current page.") @@ -854,6 +870,7 @@ async def get_content(page_id: str | None = None) -> dict[str, Any]: "text": await page.locator("body").inner_text(), } except Exception as exc: + _recovery_logger.debug('Error handled in get_content', exc_info=True) return _tool_unavailable("browser.get_content", str(exc)) @server.tool( @@ -896,6 +913,7 @@ async def screenshot( ) return result except Exception as exc: + _recovery_logger.debug('Error handled in screenshot', exc_info=True) return _tool_unavailable("browser.screenshot", str(exc), path=str(out)) @server.tool( @@ -927,6 +945,7 @@ async def cookies( "cookies": items, } except Exception as exc: + _recovery_logger.debug('Error handled in cookies', exc_info=True) return _tool_unavailable("browser.cookies", str(exc)) @server.tool(name="set_cookie", description="Add or replace a cookie in the persistent browser context.") @@ -968,6 +987,7 @@ async def set_cookie( "cookie": public_cookie, } except Exception as exc: + _recovery_logger.debug('Error handled in set_cookie', exc_info=True) return _tool_unavailable("browser.set_cookie", str(exc), name=name) @server.tool( @@ -1008,6 +1028,7 @@ async def wait_for( **await _page_snapshot(page, max_chars=6000), } except Exception as exc: + _recovery_logger.debug('Error handled in wait_for', exc_info=True) return _tool_unavailable("browser.wait_for", str(exc), page_id=page_id) @server.tool(name="network_log_start", description="Start bounded browser network recording.") @@ -1028,6 +1049,7 @@ async def network_log_start( ) return {"available": True, **result} except Exception as exc: + _recovery_logger.debug('Error handled in network_log_start', exc_info=True) return _tool_unavailable("browser.network_log_start", str(exc), page_id=page_id) @server.tool(name="network_log_list", description="List bounded, redacted network events incrementally.") @@ -1050,6 +1072,7 @@ async def network_log_list( ), } except Exception as exc: + _recovery_logger.debug('Error handled in network_log_list', exc_info=True) return _tool_unavailable("browser.network_log_list", str(exc)) @server.tool(name="network_log_stop", description="Stop browser network recording.") @@ -1057,6 +1080,7 @@ async def network_log_stop() -> dict[str, Any]: try: return {"available": True, **await browser.network_log_stop()} except Exception as exc: + _recovery_logger.debug('Error handled in network_log_stop', exc_info=True) return _tool_unavailable("browser.network_log_stop", str(exc)) @server.tool(name="console_logs", description="List bounded, redacted browser console events.") @@ -1073,6 +1097,7 @@ async def console_logs( **browser.console_logs(after_id=after_id, limit=limit, levels=levels), } except Exception as exc: + _recovery_logger.debug('Error handled in console_logs', exc_info=True) return _tool_unavailable("browser.console_logs", str(exc)) @server.tool(name="page_errors", description="List bounded, redacted uncaught page errors.") @@ -1088,6 +1113,7 @@ async def page_errors( **browser.page_errors(after_id=after_id, limit=limit), } except Exception as exc: + _recovery_logger.debug('Error handled in page_errors', exc_info=True) return _tool_unavailable("browser.page_errors", str(exc)) @server.tool( @@ -1113,6 +1139,7 @@ async def upload_file( ], } except Exception as exc: + _recovery_logger.debug('Error handled in upload_file', exc_info=True) return _tool_unavailable("browser.upload_file", str(exc), selector=selector) @server.tool( @@ -1142,7 +1169,7 @@ async def download( try: await page.goto(url, wait_until="commit", timeout=timeout_ms) except Exception: - pass + _recovery_logger.debug('Error handled in download', exc_info=True) item = await download_info.value suggested_name = Path(str(getattr(item, "suggested_filename", "download"))).name suffix = Path(suggested_name).suffix @@ -1166,6 +1193,7 @@ async def download( **artifact, } except Exception as exc: + _recovery_logger.debug('Error handled in download', exc_info=True) return _tool_unavailable("browser.download", str(exc), page_id=page_id) return server diff --git a/backend/members/adapters.py b/backend/members/adapters.py index e58d0cb..c5d0f45 100644 --- a/backend/members/adapters.py +++ b/backend/members/adapters.py @@ -2,6 +2,7 @@ import ast import json +import logging import random import re import time @@ -12,8 +13,17 @@ from urllib.parse import urlsplit, urlunsplit from backend.core.config import LLMConfig +from backend.core.validation import InputValidationError + +_recovery_logger = logging.getLogger(__name__) ACTION_KINDS = ( + "pentest_board", "pentest_add_goal", "pentest_register_asset", "pentest_record_evidence", + "pentest_report_finding", "pentest_verify_finding", "pentest_prove_goal", "pentest_plan_work", "pentest_record_coverage", + "fact", + "message", + "artifact_read", + "job_wait", "tool", "bash", "install", @@ -40,12 +50,13 @@ class MemberAction: @classmethod def from_obj(cls, obj: dict[str, Any]) -> MemberAction: if not isinstance(obj, dict): - raise ValueError(f"model action must be a JSON object, got {type(obj).__name__}") + raise InputValidationError(f"model action must be a JSON object, got {type(obj).__name__}") raw_kind = obj.get("action") or obj.get("kind") kind = raw_kind.strip() if isinstance(raw_kind, str) else raw_kind if kind not in ACTION_KINDS: raise ValueError(f"invalid action kind: {kind!r}") - args = {k: v for k, v in obj.items() if k not in ("action", "kind", "thought")} + excluded = {"action", "thought"} | ({"kind"} if not str(kind).startswith("pentest_") else set()) + args = {k: v for k, v in obj.items() if k not in excluded} if kind == "bash": for alias in ("cmd", "shell", "script"): if "command" not in args and alias in args: @@ -323,6 +334,7 @@ def health(self) -> dict: "retryable": exc.retryable, } except Exception as exc: + _recovery_logger.debug('Error handled in health', exc_info=True) return {"ok": False, "status": 0, "error": str(exc), "format": self.config.api_format} def decide(self, context: dict) -> MemberAction: @@ -599,7 +611,7 @@ def _openai_request_body( def _openai_response_text(payload: Any, surface: str) -> tuple[str, dict[str, Any]]: if not isinstance(payload, dict): - raise ValueError(f"LLM response must be a JSON object, got {type(payload).__name__}") + raise InputValidationError(f"LLM response must be a JSON object, got {type(payload).__name__}") if isinstance(payload.get("choices"), list): return _chat_response_text(payload) if surface == "responses" or isinstance(payload.get("output"), list): @@ -774,6 +786,7 @@ def _response_error_text(response: Any) -> str: value = response.json() return json.dumps(value, ensure_ascii=False)[:4000].lower() except Exception: + _recovery_logger.debug('Error handled in _response_error_text', exc_info=True) return str(getattr(response, "text", ""))[:4000].lower() @@ -1046,6 +1059,7 @@ def health(self) -> dict: "retryable": exc.retryable, } except Exception as exc: + _recovery_logger.debug('Error handled in health', exc_info=True) return {"ok": False, "status": 0, "error": str(exc), "format": self.config.api_format} def decide(self, context: dict) -> MemberAction: diff --git a/backend/members/base_member.py b/backend/members/base_member.py index 90ddee4..862ce58 100644 --- a/backend/members/base_member.py +++ b/backend/members/base_member.py @@ -3,15 +3,27 @@ import asyncio import hashlib import json +import logging import re import shlex import threading from collections import deque +from collections.abc import Callable from dataclasses import dataclass from typing import Any -from collections.abc import Callable +from backend.agent.jobs import INLINE_LIMIT +from backend.agent.runtime import AgentRuntime, AgentSessionWriter +from backend.agent.session import ( + AgentSessionStore, + SessionConflict, + SessionNeedsRebuild, +) from backend.blackboard import edge_store, graph_store, node_store +from backend.competition.conversation import ConversationAdapter, ToolCall +from backend.competition.runtime import SessionRunner +from backend.competition.store import CompetitionConflict +from backend.competition.transport import ReconSubscriber from backend.core.difficulty import ( DIFFICULTY_RANK, detect_attack_surfaces, @@ -19,13 +31,11 @@ max_difficulty, normalize_difficulty, ) -from backend.core.logging_util import IPCLogger from backend.core.ipc import FlagConflictError, submit_flag_candidate +from backend.core.logging_util import IPCLogger from backend.core.postprocess_store import enqueue_postprocess +from backend.core.validation import InputValidationError from backend.mcp.mcp_client import MCPRegistry, MCPRegistrySession, MCPRegistryTarget -from backend.agent.jobs import INLINE_LIMIT -from backend.agent.runtime import AgentRuntime, AgentSessionWriter -from backend.agent.session import AgentSessionStore, SessionConflict, SessionNeedsRebuild from backend.members.adapters import ( ACTION_KINDS, BaseAdapter, @@ -33,24 +43,25 @@ MemberAction, ProviderError, ) -from backend.competition.runtime import SessionRunner -from backend.competition.store import CompetitionConflict -from backend.competition.conversation import ConversationAdapter, ToolCall -from backend.competition.transport import ReconSubscriber from backend.members.member_tools import ( - MEMBER_TOOLS, action_payload, is_parallel_safe, is_terminal, + tools_for_category, ) from backend.memory.memory_search import search as mem_search from backend.memory.memory_store import MemoryStore from backend.sandbox.command_policy import host_path_violation from backend.sandbox.sandbox import Sandbox +from backend.tools.tool_inventory import ( + member_tool_inventory, + member_tool_inventory_path, +) from backend.tools.tool_mcp import build_category_tools_mcp, build_tool_search_mcp -from backend.tools.tool_inventory import member_tool_inventory, member_tool_inventory_path from backend.tools.tool_registry import LANGUAGES, PUBLIC_MCPS, ToolRegistry +_recovery_logger = logging.getLogger(__name__) + _LOCAL_WEBUI_URL_RE = re.compile(r"https?://(?:127\.0\.0\.1|localhost|0\.0\.0\.0):(\d{2,5})\b") _PORT_FLAG_RE = re.compile(r"(?:^|\s)(?:--port|-p)\s+(\d{2,5})(?:\s|$)") _WEBUI_HINT_RE = re.compile(r"\b(webui|gradio|streamlit|jupyter|flask|uvicorn)\b", re.IGNORECASE) @@ -105,6 +116,9 @@ class MemberDeps: competition_recon_advance: Callable[[str, int], int] | None = None runtime: Any | None = None job_store: Any | None = None + on_fact: Callable[[str, str, str], None] | None = None + on_message: Callable[[str, str, str, dict], dict] | None = None + evidence_dir: Any | None = None @dataclass @@ -125,6 +139,7 @@ class DispatchResult: invalid_action: bool = False invalid_knowledge: list[str] | None = None observation: str | None = None + process: dict[str, Any] | None = None class _IntentLeaseLost(RuntimeError): @@ -153,6 +168,9 @@ def __init__(self, name: str, adapter: BaseAdapter, deps: MemberDeps): self._recon_subscriber: ReconSubscriber | None = None self._session_recording_failed = False self._job_service: Any | None = None + configure_output = getattr(getattr(deps, "sandbox", None), "configure_output", None) + if callable(configure_output): + configure_output(getattr(getattr(deps, "runtime", None), "tool_output_bytes", 16384)) def _jobs(self): """Return the detached-job service, or None when it cannot be backed. @@ -210,6 +228,7 @@ def _renew_session_writer(self, runner=None) -> bool: try: return bool(renew()) except Exception: + _recovery_logger.debug('Error handled in _renew_session_writer', exc_info=True) return False def _record_competition_event(self, callback) -> bool: @@ -220,6 +239,7 @@ def _record_competition_event(self, callback) -> bool: callback(runner) return True except Exception as exc: + _recovery_logger.debug('Error handled in _record_competition_event', exc_info=True) self._session_recording_failed = True self._stop.set() self.deps.logger.project( @@ -284,6 +304,16 @@ def _drain_competition_recon(self, runner: SessionRunner | None = None) -> list[ ) messages.append(message) subscriber.ack(sequence) + if runner is not None: + pending = getattr(runner, "recon_sequences", []) + runner.recon_sequences = [*pending, sequence][-100:] + if (self.deps.competition_store is not None and envelope.get("run_id") + and callable(getattr(self.deps.competition_store, "append_run_event", None))): + self.deps.competition_store.append_run_event( + envelope["run_id"], "competition.message_read", + {"sequence": sequence, "request_id": envelope.get("request_id"), + "session_id": envelope.get("receiver_session_id"), "challenge_id": envelope.get("challenge_id")}, + ) return messages def solve(self, project_id: str, intent_id: str, category: str, is_initial: bool = False) -> SolveResult: @@ -371,6 +401,7 @@ async def _solve_with_mcp( f"Begin the persistent {category} challenge session for intent {intent_id}." ) except Exception as exc: + _recovery_logger.debug('Error handled in _solve_with_mcp', exc_info=True) self._session_recording_failed = True d.logger.project( "competition_session_restore_failed", @@ -451,6 +482,7 @@ async def _solve_with_mcp( error_kind="provider", ) except Exception as exc: + _recovery_logger.debug('Error handled in _solve_with_mcp', exc_info=True) error = f"{type(exc).__name__}: {exc}" d.logger.project( "member_error", @@ -539,7 +571,7 @@ async def _solve_with_mcp( self._observe("[previous competition tool call is unknown; do not replay it]") continue if not self._record_competition_event( - lambda runner: runner.record_provider_message( + lambda runner, action=action, call_id=call_id: runner.record_provider_message( {"role": "assistant", "content": json.dumps( {"action": action.kind, "arguments": action.args}, ensure_ascii=False, @@ -551,7 +583,7 @@ async def _solve_with_mcp( self._release(project_id, intent_id) return SolveResult(status="stalled", steps=step, error="session lease lost") if not self._record_competition_event( - lambda runner: runner.record_tool_call(session_call) + lambda runner, session_call=session_call: runner.record_tool_call(session_call) ): self._release(project_id, intent_id) return SolveResult(status="stalled", steps=step, error="session lease lost") @@ -608,7 +640,7 @@ async def _solve_with_mcp( "terminal": dispatched.result.status if dispatched.result else None, } if not self._record_competition_event( - lambda runner: runner.record_tool_result( + lambda runner, result_payload=result_payload, session_call=session_call: runner.record_tool_result( session_call, { "role": "tool", @@ -741,17 +773,28 @@ def _semantic_compressor(self): """ from backend.agent.compaction import summarize + runtime = getattr(self.deps, "runtime", None) + timeout = getattr(runtime, "compaction_timeout", 30) + def request(prompt: str, dropped: list[dict[str, Any]]) -> str: - rendered = json.dumps(dropped, ensure_ascii=False)[:400_000] - return self.adapter.chat( - [{"role": "user", "content": rendered}], - system_prompt=prompt, - temperature=0.0, + from backend.agent.compaction import fit_history + + rendered = json.dumps(fit_history(dropped, max_bytes=100_000), ensure_ascii=False) + config = self.adapter.config.model_copy(update={"reasoning_effort": "low"}) + adapter = ConversationAdapter( + config, max_output_tokens=getattr(runtime, "compaction_max_output_tokens", 2048), + read_timeout=timeout, turn_timeout=timeout, progress_timeout=timeout, ) + result = adapter.turn([{"role": "user", "content": rendered}], prompt, [], lambda _text: None, self._stop) + compress.last_metadata["usage"] = result.usage + return result.text def compress(messages: list[dict[str, Any]]): - return summarize(messages, request=request) + compress.last_metadata = {"budget_seconds": timeout, "attempts": 1} + return summarize(messages, request=request, + on_degraded=lambda reason: compress.last_metadata.update(degraded_reason=reason)) + compress.last_metadata = {} return compress def _make_native_runner( @@ -770,31 +813,39 @@ def _make_native_runner( self.adapter.config, max_output_tokens=getattr(runtime_config, "max_output_tokens", 32768), read_timeout=getattr(runtime_config, "provider_read_timeout", 300), + turn_timeout=getattr(runtime_config, "provider_turn_timeout", 900), + progress_timeout=getattr(runtime_config, "provider_progress_timeout", 300), ) - kwargs = dict( - system=( + kwargs = { + "system": ( f"You are Member {self.name}, a persistent {self.role_blurb}. " "Each action is its own tool; call the one that matches what " "you want to do. Read-only tools may be called together in one " "turn. Continue until the challenge is solved, a real flag is " - "verified, or evidence is genuinely exhausted." + "verified, or evidence is genuinely exhausted. Publish verified intermediate facts with publish_fact " + "without concluding your intent; put artifacts under /workspace/shared/// and include " + "their hash and run command. Use send_message to hand off work to an active teammate, not at every step. " + "Read received facts before redoing analysis. After a sustained stretch without new evidence, report " + "the failed approaches and switch hypothesis. Use job_start/job_wait for long work and artifact_read " + "to page large outputs. A verified flag must be submitted with submit_flag." ), - tools=tools, - cancel=self._stop, - emit=lambda text: self.deps.logger.llm( + "tools": tools, + "cancel": self._stop, + "emit": lambda text: self.deps.logger.llm( "stream", self._progress_project_id or "unknown", member=self.name, text=text[:2000], ), - before_turn=self._drain_competition_recon, - compressor=self._semantic_compressor(), - context_token_limit=getattr(runtime_config, "context_token_limit", 0), - compaction_trigger_ratio=getattr( + "before_turn": self._drain_competition_recon, + "compressor": self._semantic_compressor(), + "context_bytes": getattr(runtime_config, "context_bytes", 1_000_000), + "context_token_limit": getattr(runtime_config, "context_token_limit", 0), + "compaction_trigger_ratio": getattr( runtime_config, "compaction_trigger_ratio", 0.75 ), - max_turns=getattr(runtime_config, "max_turns", 0), - parallel_safe=is_parallel_safe, - is_terminal=is_terminal, - ) + "max_turns": getattr(runtime_config, "max_turns", 0), + "parallel_safe": is_parallel_safe, + "is_terminal": is_terminal, + } assignment = self.deps.competition_assignment if assignment is not None and self.deps.competition_store is not None: return SessionRunner( @@ -874,8 +925,9 @@ async def _solve_with_native_session( ), ) try: - runner = self._make_native_runner(list(MEMBER_TOOLS), project_id, intent_id) + runner = self._make_native_runner(tools_for_category(category), project_id, intent_id) except Exception as exc: + _recovery_logger.debug('Error handled in _solve_with_native_session', exc_info=True) self._release(project_id, intent_id) return SolveResult( status="failed", steps=0, error=str(exc), @@ -1017,6 +1069,8 @@ async def execute(_self, name, arguments, *, idempotency_key): } if dispatched.observation: payload["observation"] = dispatched.observation + if dispatched.process is not None: + payload["process"] = dispatched.process return payload runner.executor = Executor() @@ -1073,6 +1127,7 @@ async def execute(_self, name, arguments, *, idempotency_key): retryable=True, error_kind="session_rebuild", ) except Exception as exc: + _recovery_logger.debug('Error handled in _solve_with_native_session', exc_info=True) self._release(project_id, intent_id) self.deps.logger.project( "member_error", project_id, member=self.name, intent=intent_id, @@ -1139,6 +1194,36 @@ async def _dispatch( ) -> DispatchResult: d = self.deps kind = action.kind + if kind.startswith("pentest_"): + from backend.pentest import store as pentest_store + if category != "pentest": + return DispatchResult(invalid_action=True, observation="pentest tools require a pentest project") + try: + prepared_evidence = None + if kind == "pentest_record_evidence" and d.evidence_dir is not None: + from backend.pentest.evidence import archive_pages + prepared_evidence = await asyncio.to_thread(archive_pages, d.sandbox.read_page, + action.args.get("path", ""), d.evidence_dir) + with d.db.connect() as conn: + if kind == "pentest_board": + board = pentest_store.snapshot(conn, project_id) + result = {**board, "records": board["records"][-100:], "events": board["events"][-20:]} + else: + assignment = d.competition_assignment or {} + actor = {"member": self.name, "intent_id": intent_id, "lease_owner": d.lease_owner, + "lease_token": d.lease_token, "assignment": assignment, + "session_id": assignment.get("session_id"), "epoch": assignment.get("epoch"), + "instance_generation": assignment.get("instance_generation")} + result = pentest_store.apply(conn, project_id, kind.removeprefix("pentest_"), {k: v for k, v in action.args.items() if k != "thought"}, + actor=actor, read_artifact=(lambda _: prepared_evidence) if prepared_evidence is not None else d.sandbox.read_file) + status = graph_store.get_project_row(conn, project_id)["status"] + text = json.dumps(result, ensure_ascii=False, default=str) + d.logger.project("pentest.action", project_id, member=self.name, operation=kind, record_id=result.get("id")) + if status == "completed": + return DispatchResult(result=SolveResult(status="completed", steps=step), graph_action="pentest", observation=text) + return DispatchResult(graph_action="pentest", observation=text) + except (ValueError, KeyError, OSError) as exc: + return DispatchResult(invalid_action=True, observation=str(exc)) if kind == "bash": cmd = self._string_arg(action.args.get("command", "")) if not cmd.strip(): @@ -1170,7 +1255,18 @@ async def _dispatch( observation=text, ) res = d.sandbox.exec(cmd, timeout=60) - text = f"$ {cmd}\n{res.stdout}\n{res.stderr}".strip() + from backend.sandbox.output import preview_text + + budget = getattr(d.runtime, "tool_output_bytes", 16384) + text = preview_text(f"$ {preview_text(cmd, 2048)}\n{res.stdout}\n{res.stderr}".strip(), budget) + artifacts = getattr(res, "artifacts", {}) + process = {"exit_code": res.exit_code, "timed_out": res.timed_out, + "duration_seconds": getattr(res, "duration_seconds", None), + "output_truncated": bool(artifacts), "artifacts": artifacts} + if artifacts: + text += "\n[output artifacts] " + json.dumps(artifacts, ensure_ascii=False) + if res.timed_out: + text += "\nCommand timed out. Use job_start for work that requires more than 60 seconds." self._observe(text) self._observe_webui_links(project_id, cmd, res.stdout, res.stderr) d.logger.tool( @@ -1181,6 +1277,7 @@ async def _dispatch( exit_code=res.exit_code, stdout=res.stdout[:4000], stderr=res.stderr[:4000], + process=process, ) missing_command = self._missing_command_from_result(cmd, res.stderr) if missing_command: @@ -1188,7 +1285,23 @@ async def _dispatch( return DispatchResult( invalid_action=True, invalid_knowledge=knowledge, observation=text ) - return DispatchResult(observation=text) + return DispatchResult(observation=text, process=process) + if kind == "job_wait": + service = self._jobs() + if service is None: + return DispatchResult(invalid_action=True, observation="job service unavailable") + job_id = self._string_arg(action.args.get("job_id", "")) + timeout = max(1, min(15, int(action.args.get("timeout", 5)))) + result = await asyncio.to_thread(service.wait, job_id, timeout=timeout, cancel=self._stop) + return DispatchResult(observation="[job_wait] " + json.dumps(result, ensure_ascii=False, default=str)) + if kind == "artifact_read": + reader = getattr(d.sandbox, "read_page", None) + if not callable(reader): + return DispatchResult(invalid_action=True, observation="artifact paging unavailable") + result = await asyncio.to_thread(reader, self._string_arg(action.args.get("path", "")), + offset=max(0, int(action.args.get("offset", 0))), + limit=max(1, min(8192, int(action.args.get("limit", 8192))))) + return DispatchResult(observation=json.dumps(result, ensure_ascii=False)) if kind in ("job_start", "job_status", "job_logs", "job_cancel"): return self._dispatch_job(project_id, intent_id, action) if kind == "install": @@ -1213,6 +1326,7 @@ async def _dispatch( try: out = await mcp_session.call_tool(server, tool, args) except Exception as exc: + _recovery_logger.debug('Error handled in _dispatch', exc_info=True) out = {"error": str(exc)} text = f"[mcp:{server}.{tool}] {out}"[:5000] self._observe(text) @@ -1256,6 +1370,50 @@ async def _dispatch( if kind == "report": report = self._submit_report(project_id, intent_id, action) return DispatchResult(graph_action="report" if report is not None else None) + if kind == "fact": + description = self._string_arg(action.args.get("description", "")).strip() + evidence = self._string_arg(action.args.get("evidence", "")).strip() + artifacts = action.args.get("artifacts", []) + if not description or not evidence or len(description) > 1800 or len(evidence) > 1800: + return DispatchResult(invalid_action=True, observation="publish_fact requires a bounded claim and concrete evidence") + if not isinstance(artifacts, list) or len(artifacts) > 8 or len(json.dumps(artifacts)) > 4096: + return DispatchResult(invalid_action=True, observation="invalid artifact handoff") + for artifact in artifacts: + if (not isinstance(artifact, dict) or not str(artifact.get("path", "")).startswith("/workspace/shared/") + or not re.fullmatch(r"[a-fA-F0-9]{64}", str(artifact.get("sha256", "")))): + return DispatchResult(invalid_action=True, observation="shared artifacts need a /workspace/shared/ path and SHA-256") + text = description + "\nEvidence: " + evidence + if artifacts: + text += "\nArtifacts: " + json.dumps(artifacts, ensure_ascii=False) + with d.db.connect() as connection: + connection.execute("SELECT id FROM projects WHERE id=%s FOR UPDATE", (project_id,)) + owned = connection.execute( + """SELECT id FROM intents WHERE project_id=%s AND id=%s AND worker=%s AND lease_owner=%s + AND lease_token=%s AND lease_expires_at>now() AND to_fact_id IS NULL FOR UPDATE""", + (project_id, intent_id, self.name, d.lease_owner, d.lease_token), + ).fetchone() + if owned is None: + raise _IntentLeaseLost("intent lease lost before fact publication") + old = connection.execute("SELECT id FROM facts WHERE project_id=%s AND description=%s LIMIT 1", (project_id, text)).fetchone() + if old: + fact_id = old["id"] + else: + fact = node_store.reserve_fact(connection, project_id, text) + node_store.insert_fact(connection, project_id, fact.id, fact.description) + fact_id = fact.id + graph_store.add_link(connection, project_id, self.name, f"fact:{fact_id}", "explore") + if not old and d.on_fact: + d.on_fact(project_id, fact_id, self.name) + d.logger.project("member_fact_published", project_id, member=self.name, fact_id=fact_id, artifacts=artifacts) + return DispatchResult(graph_action="fact", observation=f"Published fact {fact_id}; your intent remains active.") + if kind == "message": + receiver = self._string_arg(action.args.get("receiver", "")).strip() + content = self._string_arg(action.args.get("content", "")).strip() + if not receiver or receiver == self.name or not content or len(content) > 1800 or d.on_message is None: + return DispatchResult(invalid_action=True, observation="message needs one active teammate and bounded content") + result = d.on_message(project_id, self.name, receiver, + {"kind": "member_message", "content": content, "reply_to": action.args.get("reply_to", "")}) + return DispatchResult(graph_action="message", observation=json.dumps(result, ensure_ascii=False)) if kind == "intent": if not allow_intent: d.logger.project( @@ -1329,6 +1487,7 @@ def _heartbeat(self, project_id, intent_id) -> bool: self.deps.lease_token = token return True except Exception as exc: + _recovery_logger.debug('Error handled in _heartbeat', exc_info=True) self.deps.logger.project( "intent_heartbeat_failed", project_id, @@ -1362,9 +1521,10 @@ def _restore_progress(self, project_id: str, intent_id: str) -> None: raw = self.deps.sandbox.read_file(self._progress_path) parsed = json.loads(raw) if raw else [] if not isinstance(parsed, list): - raise ValueError("progress checkpoint must contain a JSON array") + raise InputValidationError("progress checkpoint must contain a JSON array") self.observations = [str(item)[:5000] for item in parsed if str(item).strip()][-8:] except Exception as exc: + _recovery_logger.debug('Error handled in _restore_progress', exc_info=True) self.deps.logger.project( "member_progress_restore_failed", project_id, @@ -1392,6 +1552,7 @@ def _persist_progress(self) -> None: json.dumps(self.observations[-8:], ensure_ascii=False), ) except Exception as exc: + _recovery_logger.debug('Error handled in _persist_progress', exc_info=True) if self._container_gone(exc): self._progress_save_error_reported = True self.deps.logger.project( @@ -1440,6 +1601,7 @@ def _seed_tool_inventory(self, project_id: str) -> None: member_tool_inventory().rstrip() + "\n\n" + self._runtime_tool_inventory_note(), ) except Exception as exc: + _recovery_logger.debug('Error handled in _seed_tool_inventory', exc_info=True) self.deps.logger.project( "member_tool_inventory_seed_failed", project_id, @@ -1463,6 +1625,7 @@ def _prime_tool_context(self, project_id: str, intent_id: str, category: str) -> query, available_mcps=self._available_mcp_names() )[:5] except Exception as exc: + _recovery_logger.debug('Error handled in _prime_tool_context', exc_info=True) self.deps.logger.project( "member_tool_context_prime_failed", project_id, @@ -1506,6 +1669,7 @@ def _probe_cli_tools(self) -> dict[str, bool]: availability = {str(k): bool(v) for k, v in parsed.items()} break except Exception: + _recovery_logger.debug('Error handled in _probe_cli_tools', exc_info=True) availability = {} self._tool_availability = availability return availability @@ -2121,9 +2285,13 @@ def _raise_flag(self, project_id, intent_id, action: MemberAction, step) -> Solv from_fact = fact.id else: from_fact = row["to_fact_id"] - edge_store.complete_goal_intent( - conn, project_id, [from_fact], desc, self.name, self.name - ) + # Competition challenges finish through the platform verdict. + # A single accepted flag must not close the challenge's goal + # edge while the platform still expects more flags. + if getattr(d, "competition_assignment", None) is None: + edge_store.complete_goal_intent( + conn, project_id, [from_fact], desc, self.name, self.name + ) candidate = submit_flag_candidate( conn, project_id, @@ -2197,11 +2365,11 @@ def _build_context(self, project_id, intent_id, category, step, is_initial, eval attachment["path"] = attachment_path(attachment["filename"], attachment["path"]) runtime_notes = [ "All work happens inside this challenge's Linux task container. Only container paths exist.", - "Never touch host paths: no Windows drive paths, no /host, no host interpreter, and no path " + ("Never touch host paths: no Windows drive paths, no /host, no host interpreter, and no path " "inside the IPC deployment itself (source tree, data/, artifacts/, exports/, writeups, .qa-artifacts). " - "Reading a solution or writeup file instead of solving the challenge invalidates the run.", - "If a CLI tool or Python library is missing, use the install action to add it to this container " - "(manager: apt|pip|gem|npm|go), then retry. Do not substitute a host tool.", + "Reading a solution or writeup file instead of solving the challenge invalidates the run."), + ("If a CLI tool or Python library is missing, use the install action to add it to this container " + "(manager: apt|pip|gem|npm|go), then retry. Do not substitute a host tool."), "Flag search priority for this round: try /flag first, then environment variables, then other methods.", "If attachment_true is true, inspect the listed attachments before blind target probing.", ( @@ -2225,16 +2393,28 @@ def _build_context(self, project_id, intent_id, category, step, is_initial, eval "fragments) and only report a flag that reads as a coherent whole." ) except Exception: - pass + _recovery_logger.debug('Error handled in _build_context', exc_info=True) + if category == "pentest": + runtime_notes = [note for note in runtime_notes if not note.startswith("Flag search priority")] return { "role": self.name, "role_blurb": self.role_blurb, "category": category, + "pentest_contract": ( + "Use pentest_board first. Respect scope/constraints. Decompose explicit objectives with pentest_add_goal. " + "Record discovered assets and actual artifact evidence, register candidate findings, plan a separate validation " + "intent, then prove objectives with verified evidence. Use plan_work for dependent subtasks and publish negative " + "results; do not repeat failed routes. Existing jobs, bash, MCP and model configuration remain the execution harness. " + "Platform flags must still be submitted through submit_flag; a goal proof cannot substitute for platform completion." + if category == "pentest" else None + ), "step": step, "max_steps": None if d.continuous else min(d.max_steps, d.max_actions_per_task), "short_task": not d.continuous, "task_contract": ( - "Continue this challenge session until there is a verified result, an operator cancellation, " + "A flag is not submitted by writing it in conclude text. Whenever you have a flag, you MUST call " + "submit_flag with the exact candidate and evidence before concluding; conclude text alone is never " + "a submission. Continue this challenge session until there is a verified result, an operator cancellation, " "a lost lease, or concrete evidence that a different direction or helper is required. " "There is no artificial step budget. Do not repeat previous attempts." if d.continuous else @@ -2306,6 +2486,7 @@ def _observe_webui_links(self, project_id: str, command: str, stdout: str, stder try: url = expose(project_id, self.name, port) except Exception: + _recovery_logger.debug('Error handled in _observe_webui_links', exc_info=True) continue self._observe( f"[webui:{port}] Shared browser URL: {url}/ " diff --git a/backend/members/member_tools.py b/backend/members/member_tools.py index d5f3edc..33ce7a2 100644 --- a/backend/members/member_tools.py +++ b/backend/members/member_tools.py @@ -26,11 +26,17 @@ from typing import Any +from backend.core.validation import InputValidationError + TERMINAL_TOOLS = frozenset({"submit_flag", "conclude", "give_up"}) -PARALLEL_SAFE_TOOLS = frozenset({"memory_search", "tool_search", "job_status", "job_logs"}) +PARALLEL_SAFE_TOOLS = frozenset({"memory_search", "tool_search", "job_status", "job_logs", "artifact_read"}) TOOL_ACTIONS: dict[str, str] = { + "publish_fact": "fact", + "send_message": "message", + "artifact_read": "artifact_read", + "job_wait": "job_wait", "bash": "bash", "install": "install", "mcp_tool": "tool", @@ -69,6 +75,19 @@ def _tool( MEMBER_TOOLS: tuple[dict[str, Any], ...] = ( + _tool("publish_fact", "Publish a verified intermediate fact and notify active teammates without finishing your intent. Include concrete evidence and versioned shared artifact paths. Never use this instead of submit_flag.", + {"description": {**_STRING, "maxLength": 1800}, "evidence": {**_STRING, "maxLength": 1800}, + "artifacts": {"type": "array", "maxItems": 8, "items": {"type": "object", "properties": { + "path": _STRING, "sha256": _STRING, "version": _STRING, "purpose": _STRING}, + "required": ["path", "sha256"], "additionalProperties": False}}, "thought": _STRING}, + ["description", "evidence"]), + _tool("send_message", "Send a bounded business message to one active teammate on this challenge. State the requested subtask or handoff, shared artifact version, and expected response. Set reply_to to the incoming request_id when responding.", + {"receiver": _STRING, "content": {**_STRING, "maxLength": 1800}, "reply_to": _STRING, "thought": _STRING}, + ["receiver", "content"]), + _tool("artifact_read", "Read a bounded page of a workspace artifact or a full-output file returned by bash. Use next_offset to continue; do not cat large files into context.", + {"path": _STRING, "offset": {**_INTEGER, "minimum": 0}, "limit": {**_INTEGER, "minimum": 1, "maximum": 8192}, "thought": _STRING}, ["path"]), + _tool("job_wait", "Wait up to 15 seconds for a job to exit or produce more progress. Use this instead of shell sleep loops. A still-running or lost job is not success.", + {"job_id": _STRING, "timeout": {**_INTEGER, "minimum": 1, "maximum": 15}, "thought": _STRING}, ["job_id"]), _tool( "bash", "Run one shell command in this challenge's container. Killed after 60 " @@ -189,6 +208,30 @@ def _tool( ), ) +from backend.pentest.models import ACTION_MODELS + +PENTEST_TOOLS = ( + _tool("pentest_board", "Read this project's persistent objectives, scope, constraints, assets, findings, evidence and dependent work.", {}), + *(_tool("pentest_" + operation, { + "add_goal": "Register an explicit, verifiable objective from the user task.", + "register_asset": "Register one discovered asset and its parent. Do not expand beyond the explicit task scope.", + "record_evidence": "Register an existing nonempty artifact from this task container; the harness computes its SHA-256. Include the actual command and an evidence summary.", + "report_finding": "Register a candidate vulnerability with reproducible steps and existing evidence IDs. It still requires a separate validation intent.", + "verify_finding": "Record a separate intent's actual validation of one candidate. Preserve inconclusive and negative results.", + "prove_goal": "Prove one objective using existing evidence and confirmed findings. Platform tasks still require all platform flags; standalone tasks complete only when all required objectives are proven.", + "plan_work": "Persist a focused subtask with prerequisite work IDs, input evidence, priority and a concrete deliverable. Diamond dispatches it after prerequisites finish.", + "record_coverage": "Record a specific asset test. Mark tested only with execution evidence; failures do not count as coverage.", + }[operation], model.model_json_schema()["properties"], model.model_json_schema().get("required", [])) + for operation, model in ACTION_MODELS.items()), +) +TOOL_ACTIONS.update({tool["name"]: tool["name"] for tool in PENTEST_TOOLS}) +PARALLEL_SAFE_TOOLS = PARALLEL_SAFE_TOOLS | {"pentest_board"} + + +def tools_for_category(category): + return list((*MEMBER_TOOLS, *PENTEST_TOOLS) if category == "pentest" else MEMBER_TOOLS) + + MEMBER_TOOL_NAMES = tuple(tool["name"] for tool in MEMBER_TOOLS) @@ -234,23 +277,24 @@ def tool_call_for_action(action: dict[str, Any]) -> tuple[str, dict[str, Any]]: rides along because every tool schema accepts it. """ if not isinstance(action, dict): - raise ValueError(f"action must be a dict, got {type(action).__name__}") + raise InputValidationError(f"action must be a dict, got {type(action).__name__}") kind = action.get("action") or action.get("kind") if kind not in ACTION_TOOLS: raise ValueError(f"no native tool maps to action kind: {kind!r}") - arguments = {k: v for k, v in action.items() if k not in ("action", "kind")} + excluded = {"action"} | ({"kind"} if not str(kind).startswith("pentest_") else set()) + arguments = {k: v for k, v in action.items() if k not in excluded} return ACTION_TOOLS[kind], arguments __all__ = [ + "ACTION_TOOLS", "MEMBER_TOOLS", "MEMBER_TOOL_NAMES", "PARALLEL_SAFE_TOOLS", "TERMINAL_TOOLS", "TOOL_ACTIONS", - "ACTION_TOOLS", "action_payload", - "tool_call_for_action", "is_parallel_safe", "is_terminal", + "tool_call_for_action", ] diff --git a/backend/memory/memory_mcp.py b/backend/memory/memory_mcp.py index 1ab50f6..356dcec 100644 --- a/backend/memory/memory_mcp.py +++ b/backend/memory/memory_mcp.py @@ -1,7 +1,8 @@ from __future__ import annotations import asyncio -from typing import Any, Callable, Literal +from collections.abc import Callable +from typing import Any, Literal from backend.mcp.mcp_server import MCPServer, create_mcp_server from backend.memory.memory_search import search diff --git a/backend/memory/memory_store.py b/backend/memory/memory_store.py index ba46d95..fbb4a29 100644 --- a/backend/memory/memory_store.py +++ b/backend/memory/memory_store.py @@ -8,7 +8,6 @@ from backend.persistence.database import Database, PostgresDatabase - CATEGORIES = ("knowledge", "tool_usage", "exploit", "lessons") diff --git a/backend/ops/agent_bridge.py b/backend/ops/agent_bridge.py index da51d72..7b454c6 100644 --- a/backend/ops/agent_bridge.py +++ b/backend/ops/agent_bridge.py @@ -1,14 +1,3 @@ -"""Ops chat on the unified agent runtime, using native provider tool calls. - -Before this module, Ops asked the model to emit ``{"reply":...,"tool_call":...}`` -as JSON inside prose, then re-encoded each round as alternating user/assistant -text. That is what lost the tool transcript: the durable conversation kept only -the final reply, so the next turn could not see what the model had already run. - -Here tool calls go over the provider's own tool-call channel and every -provider message, call and result becomes an ``agent_events`` row, so the next -turn's context is rebuilt from the real transcript. -""" from __future__ import annotations import json diff --git a/backend/ops/attachments.py b/backend/ops/attachments.py index 2f031e6..5606396 100644 --- a/backend/ops/attachments.py +++ b/backend/ops/attachments.py @@ -1,4 +1,3 @@ -"""Durable, non-executable files attached to an IPC conversation.""" from __future__ import annotations import json @@ -7,7 +6,6 @@ from pathlib import Path from typing import BinaryIO - UPLOAD_ID_RE = re.compile(r"^upload_[a-f0-9]{24}$") MAX_UPLOAD_BYTES = 20 * 1024 * 1024 MAX_CONTEXT_BYTES = 120_000 diff --git a/backend/ops/ipc_mcp.py b/backend/ops/ipc_mcp.py index 157ced5..84b7086 100644 --- a/backend/ops/ipc_mcp.py +++ b/backend/ops/ipc_mcp.py @@ -1,13 +1,13 @@ from __future__ import annotations +import logging import re import threading from collections.abc import Callable from typing import Any -from mcp.server.transport_security import TransportSecuritySettings - from mcp.server.fastmcp import Context +from mcp.server.transport_security import TransportSecuritySettings from backend.blackboard import edge_store, graph_store from backend.core.config import CATEGORIES @@ -26,6 +26,8 @@ from backend.ops.store import OpsStore from backend.ops.tools import OpsToolError, OpsToolExecutor +_recovery_logger = logging.getLogger(__name__) + _SESSION_ID_RE = re.compile(r"^ops_[a-f0-9]{16}$") _SENSITIVE_FIELD_RE = re.compile( r"(?i)\b(?:authorization|api[ _-]?key|token|cookie|password|secret)\b\s*[:=]\s*[^\s,;]+" @@ -58,7 +60,40 @@ def build_ipc_mcp(state_provider: Callable[[], AppState]) -> MCPServer: ) tools = _Tools(state_provider) - @server.tool(name="ipc_question", description="Ask for missing platform information. Return to the operator while pending; call again with the same operation key to read the answer. Credentials go into the workflow secret store; for native GZCTF use secret_name=gzctf_token (or gzctf_username/gzctf_password).") + @server.tool(name="ipc_gzctf_workflow", description=( + "Create or update the built-in default GZCTF workflow. Supply only platform base_url, " + "game_id and optional name/team_id/ends_at/remote_concurrency/attachment_base_url. " + "No HTTP mappings or platform probing are needed. Credentials must already be in the " + "workflow secret store or deployment environment; ask ipc_question for gzctf_token " + "after creating the draft when needed. Return the draft for normal confirmation." + )) + def gzctf_workflow(settings: dict[str, Any], workflow_id: str | None = None) -> dict[str, Any]: + from backend.ops.service import OpsAgentService + + if "secrets" in settings: + raise ValueError("use the structured workflow secret store for credentials") + state = state_provider() + service = getattr(state, "ops_agent_service", None) or OpsAgentService(state) + return service.configure_gzctf_workflow(settings, workflow_id=workflow_id) + + @server.tool(name="ipc_tsecbench_workflow", description=( + "Create or update the built-in default TSecBench benchmark workflow. Supply only " + "the platform base_url and optional name/team_id/remote_concurrency. No HTTP " + "mappings or platform probing are needed. The benchmark token goes into the " + "workflow secret store as benchmark_token; ask ipc_question with " + "secret_name=benchmark_token after creating the draft when it is missing. " + "Return the draft for normal confirmation." + )) + def tsecbench_workflow(settings: dict[str, Any], workflow_id: str | None = None) -> dict[str, Any]: + from backend.ops.service import OpsAgentService + + if "secrets" in settings: + raise ValueError("use the structured workflow secret store for credentials") + state = state_provider() + service = getattr(state, "ops_agent_service", None) or OpsAgentService(state) + return service.configure_tsecbench_workflow(settings, workflow_id=workflow_id) + + @server.tool(name="ipc_question", description="Ask for missing platform information. Return to the operator while pending; call again with the same operation key to read the answer. Credentials go into the workflow secret store; use secret_name=benchmark_token for a native TSecBench workflow, or secret_name=gzctf_token (or gzctf_username/gzctf_password) for native GZCTF.") def question(operation_key: str, title: str, options: list[str] | None = None, workflow_id: str | None = None, secret_name: str | None = None, ctx: Context | None = None) -> dict: @@ -598,5 +633,6 @@ def _session_id_from_context(ctx: Context | None) -> str | None: headers = getattr(request, "headers", None) value = headers.get("x-ipc-ops-session", "") if headers is not None else "" except Exception: + _recovery_logger.debug('Error handled in _session_id_from_context', exc_info=True) return None return value if isinstance(value, str) and _SESSION_ID_RE.fullmatch(value) else None diff --git a/backend/ops/models.py b/backend/ops/models.py index d8b8073..9ce3d41 100644 --- a/backend/ops/models.py +++ b/backend/ops/models.py @@ -83,7 +83,7 @@ def validate_prefix(cls, value: str) -> str: class ChallengeMappingSpec(BaseModel): model_config = ConfigDict(extra="forbid") - platform: Literal["http_json", "ret2shell", "gzctf"] = "http_json" + platform: Literal["http_json", "ret2shell", "gzctf", "tsecbench"] = "http_json" game_id: int | None = None list_url: str = "" list_path: str = "data" @@ -94,6 +94,7 @@ class ChallengeMappingSpec(BaseModel): attachments_field: str = "files" remote_field: str = "" solved_field: str = "" + score_field: str = "score" hints_field: str = "" level: int = Field(default=1, ge=1, le=1000) track_id: str = "" @@ -102,6 +103,7 @@ class ChallengeMappingSpec(BaseModel): pagination_path: str = "" max_pages: int = Field(default=100, ge=1, le=1000) max_challenges: int = Field(default=0, ge=0, le=1000) + allow_hints: bool = False category_map: dict[str, str] = Field(default_factory=dict) attachment_base_url: str = "" headers: list[SecretHeader] = Field(default_factory=list) @@ -117,6 +119,8 @@ def require_list_url_for_http_json(self) -> ChallengeMappingSpec: raise ValueError("list_url is required for the http_json platform") if self.platform == "gzctf" and self.game_id is None: raise ValueError("game_id is required for the gzctf platform") + if self.platform == "tsecbench" and not self.list_url.strip(): + raise ValueError("list_url is required for the tsecbench platform") return self @field_validator("attachment_base_url") @@ -133,6 +137,7 @@ def validate_attachment_base_url(cls, value: str) -> str: "attachments_field", "remote_field", "solved_field", + "score_field", "hints_field", "level_field", "track_id_field", @@ -165,6 +170,7 @@ def to_field_mapping(self, headers: dict[str, str]) -> FieldMapping: attachments_field=self.attachments_field, remote_field=self.remote_field, solved_field=self.solved_field, + score_field=self.score_field, hints_field=self.hints_field, level=self.level, track_id=self.track_id, @@ -174,6 +180,7 @@ def to_field_mapping(self, headers: dict[str, str]) -> FieldMapping: max_pages=self.max_pages, max_challenges=self.max_challenges, category_map=self.category_map, + allow_hints=self.allow_hints, headers=headers, attachment_base_url=self.attachment_base_url, ) @@ -240,9 +247,12 @@ class PlatformWorkflowSpec(BaseModel): model_config = ConfigDict(extra="forbid") name: str = Field(min_length=1, max_length=120) + template_id: Literal["", "gzctf", "tsecbench"] = "" + template_version: int = Field(default=1, ge=1, le=1) competition_id: str = Field(default="", max_length=256) team_id: str = Field(default="", max_length=256) ends_at: str = "" + remote_concurrency: int = Field(default=0, ge=0, le=100) remote_instance_limit: int = Field(default=0, ge=0, le=100) challenges: ChallengeMappingSpec submit: FlagSubmitSpec | None = None @@ -252,7 +262,12 @@ class PlatformWorkflowSpec(BaseModel): @field_validator("name") @classmethod def validate_name(cls, value: str) -> str: - return value.strip() + value = value.strip() + if not value: + raise ValueError("workflow name must be the actual competition name") + if value.casefold() in {"proposed", "probe", "workflow", "new workflow"}: + raise ValueError("workflow name must be the actual competition name, not a placeholder") + return value @field_validator("ends_at") @classmethod @@ -293,6 +308,8 @@ def allowed_secret_names(self) -> set[str]: "platform_token", } ) + if self.challenges.platform == "tsecbench": + names.update({"benchmark_token", "platform_token"}) return names diff --git a/backend/ops/network.py b/backend/ops/network.py index 98c648f..2990c58 100644 --- a/backend/ops/network.py +++ b/backend/ops/network.py @@ -90,7 +90,7 @@ def _resolved_address(self, parsed) -> str: raise NetworkPolicyError( f"workflow hostname resolves to a non-public address: {parsed.hostname}" ) - return sorted(addresses)[0] + return min(addresses) def _resolve_addresses(self, hostname: str, port: int | None, scheme: str) -> set[str]: effective_port = port or (443 if scheme == "https" else 80) @@ -164,5 +164,5 @@ def _pinned_request( response.reason = raw.reason response.encoding = requests.utils.get_encoding_from_headers(response.headers) if not stream: - response.content + _ = response.content return response diff --git a/backend/ops/presets.py b/backend/ops/presets.py new file mode 100644 index 0000000..7308986 --- /dev/null +++ b/backend/ops/presets.py @@ -0,0 +1,26 @@ +"""Built-in platform workflow registry; legacy imports remain supported.""" +from copy import deepcopy + +from backend.platform.gzctf.workflow import GZCTF_TEMPLATE_VERSION, GZCTFWorkflowInput +from backend.platform.gzctf.workflow import TEMPLATE as GZCTF_TEMPLATE +from backend.platform.tsecbench.workflow import TEMPLATE as TSECBENCH_TEMPLATE +from backend.platform.tsecbench.workflow import ( + TSECBENCH_TEMPLATE_VERSION, + TSECBenchWorkflowInput, +) + +DEFAULT_WORKFLOW_TEMPLATE = "gzctf" + + +def workflow_templates() -> list[dict]: + return deepcopy([GZCTF_TEMPLATE, TSECBENCH_TEMPLATE]) + + +__all__ = [ + "DEFAULT_WORKFLOW_TEMPLATE", + "GZCTF_TEMPLATE_VERSION", + "TSECBENCH_TEMPLATE_VERSION", + "GZCTFWorkflowInput", + "TSECBenchWorkflowInput", + "workflow_templates", +] diff --git a/backend/ops/questions.py b/backend/ops/questions.py index b2b1150..3b26d05 100644 --- a/backend/ops/questions.py +++ b/backend/ops/questions.py @@ -4,7 +4,7 @@ from backend.competition.store import CompetitionConflict, new_id from backend.ops.models import validate_secret_name -from backend.ops.store import OpsStore +from backend.ops.store import OpsStore, WorkflowSessionConflict class QuestionStore: @@ -23,10 +23,21 @@ def create(self, *, session_id: str, operation_key: str, title: str, secret_name = validate_secret_name(secret_name) if not workflow_id: raise ValueError("sensitive questions require a target workflow") + workflow = OpsStore(self.state.root, database=self.state.db).get_workflow(workflow_id) + if secret_name not in workflow["spec"].allowed_secret_names(): + raise ValueError("question secret_name is not allowed by the workflow") scoped_key = f"{session_id}:{operation_key}" with self.state.db.connect() as connection: if not connection.execute("SELECT id FROM sessions WHERE id=%s", (session_id,)).fetchone(): raise ValueError("question session not found") + if workflow_id: + bound = connection.execute( + "SELECT agent_workflow_id FROM sessions WHERE id=%s", (session_id,) + ).fetchone() + if bound and bound["agent_workflow_id"] and bound["agent_workflow_id"] != workflow_id: + raise WorkflowSessionConflict( + "question targets another workflow; create a new Ops session" + ) connection.execute("SELECT pg_advisory_xact_lock(hashtext(%s))", (scoped_key,)) old = connection.execute("SELECT * FROM competition_questions WHERE operation_key=%s", (scoped_key,)).fetchone() if old: @@ -79,3 +90,18 @@ def result(self, question_id: str, session_id: str) -> dict: if not row: raise KeyError(question_id) return row + + def dismiss(self, question_id: str, session_id: str) -> dict: + with self.state.db.connect() as connection: + row = connection.execute( + """UPDATE competition_questions SET state='cancelled',answered_at=now() + WHERE id=%s AND ops_session_id=%s AND state='pending' RETURNING id,state""", + (question_id, session_id), + ).fetchone() + if row: + return row + row = connection.execute("SELECT id,state FROM competition_questions WHERE id=%s AND ops_session_id=%s", + (question_id, session_id)).fetchone() + if row is None: + raise KeyError(question_id) + return row diff --git a/backend/ops/service.py b/backend/ops/service.py index 5d32ba4..d7871b5 100644 --- a/backend/ops/service.py +++ b/backend/ops/service.py @@ -1,6 +1,6 @@ from __future__ import annotations -import json +import logging import re import secrets import shutil @@ -13,17 +13,10 @@ import requests +from backend.agent.session import AgentSessionStore from backend.blackboard import graph_store from backend.core.state import AppState -from backend.members.adapters import health_check, make_adapter -from backend.ops.models import ( - PlatformWorkflowSpec, - SecretHeader, - render_template, - validate_secret_name, -) -from backend.ops.network import WorkflowHttpClient -from backend.agent.session import AgentSessionStore +from backend.members.adapters import health_check from backend.ops.agent_bridge import ( PROPOSE_WORKFLOW_TOOL, OpsToolBridge, @@ -32,11 +25,20 @@ redact, ) from backend.ops.attachments import OpsAttachmentStore -from backend.ops.store import OpsStore +from backend.ops.models import ( + PlatformWorkflowSpec, + SecretHeader, + render_template, + validate_secret_name, +) +from backend.ops.network import WorkflowHttpClient +from backend.ops.store import OpsStore, WorkflowSessionConflict from backend.ops.tools import OpsToolError, OpsToolExecutor from backend.platform.adapter import PlatformAdapter from backend.platform.factory import build_adapter +_recovery_logger = logging.getLogger(__name__) + _RUN_ID_RE = re.compile(r"^run_[A-Za-z0-9_-]{8,128}$") _SYSTEM_PROMPT = """\ @@ -84,6 +86,10 @@ class WorkflowConfirmationError(OpsAgentError): pass +class WorkflowSessionError(OpsAgentError): + pass + + class _OpsRunInterrupted(OpsAgentError): """Internal control flow used by API-backed IPC runs.""" @@ -139,7 +145,7 @@ def write(self, kind: str, text: str) -> None: or time.monotonic() >= self._deadline ) if ready: - keep = self._hold if self._hold < len(self._buffer) else len(self._buffer) + keep = min(len(self._buffer), self._hold) chunk = self._buffer[: len(self._buffer) - keep] if chunk: pending.append((kind, chunk)) @@ -167,6 +173,7 @@ def __init__(self, state: AppState) -> None: self._run_condition = threading.Condition() self._run_threads: dict[str, threading.Thread] = {} self._run_threads_lock = threading.Lock() + self.store.cleanup_duplicate_workflows() self._recover_stale_runs() def _recover_stale_runs(self) -> None: @@ -299,6 +306,8 @@ def _workflow_context(self, workflow_id: str | None) -> str: except KeyError: return "" spec = workflow.get("spec") or {} + if hasattr(spec, "model_dump"): + spec = spec.model_dump(mode="json") challenges = spec.get("challenges") or {} if isinstance(spec, dict) else {} submit = spec.get("submit") or {} if isinstance(spec, dict) else {} lines = [ @@ -342,6 +351,11 @@ def chat( session_id = self.store.create_session(_session_title(safe_message))["id"] else: self.store.get_session(session_id) + if workflow_id: + try: + self.store.bind_agent_workflow(session_id, workflow_id) + except WorkflowSessionConflict as exc: + raise WorkflowSessionError(str(exc)) from exc if normalized_secrets: self.store.save_session_secrets(session_id, normalized_secrets) @@ -400,7 +414,10 @@ def _save_workflow_proposal( self, spec_data: dict[str, Any], session_id: str ) -> dict[str, Any]: spec = PlatformWorkflowSpec.model_validate(spec_data) - workflow = self.store.create_workflow(spec, session_id=session_id, source="agent") + try: + workflow = self.store.update_session_workflow(session_id, spec) + except WorkflowSessionConflict as exc: + raise WorkflowSessionError(str(exc)) from exc return self.workflow_view(workflow) def _chat_with_api( @@ -609,6 +626,11 @@ def _start_api_run( session_id = self.store.create_session(_session_title(safe_message))["id"] else: self.store.get_session(session_id) + if workflow_id: + try: + self.store.bind_agent_workflow(session_id, workflow_id) + except WorkflowSessionConflict as exc: + raise WorkflowSessionError(str(exc)) from exc run_id = f"run_{secrets.token_hex(16)}" self.store.create_run(session_id, run_id) @@ -693,6 +715,7 @@ def _run_api_background( response = self._finish_interrupted_api_chat(session_id=session_id) self.store.finish_run(run_id, status="interrupted", response=response) except Exception as exc: + _recovery_logger.debug('Error handled in _run_api_background', exc_info=True) safe_error = _replace_secret_values(str(exc), redaction_values).strip() if not safe_error: safe_error = "IPC API run failed" @@ -804,6 +827,30 @@ def create_workflow( ) return self.workflow_view(workflow) + def configure_gzctf_workflow(self, data: dict[str, Any], *, workflow_id: str | None = None) -> dict[str, Any]: + from backend.ops.presets import GZCTFWorkflowInput + + settings = GZCTFWorkflowInput.model_validate(data) + spec = settings.workflow_spec() + if workflow_id: + existing = self.store.get_workflow(workflow_id) + if existing["spec"].challenges.platform != "gzctf": + raise ValueError("only a GZCTF workflow can be updated with the GZCTF template") + return self.update_workflow(workflow_id, spec.model_dump(), secrets_values=settings.secrets) + return self.create_workflow(spec.model_dump(), secrets_values=settings.secrets) + + def configure_tsecbench_workflow(self, data: dict[str, Any], *, workflow_id: str | None = None) -> dict[str, Any]: + from backend.ops.presets import TSECBenchWorkflowInput + + settings = TSECBenchWorkflowInput.model_validate(data) + spec = settings.workflow_spec() + if workflow_id: + existing = self.store.get_workflow(workflow_id) + if existing["spec"].challenges.platform != "tsecbench": + raise ValueError("only a TSecBench workflow can be updated with the TSecBench template") + return self.update_workflow(workflow_id, spec.model_dump(), secrets_values=settings.secrets) + return self.create_workflow(spec.model_dump(), secrets_values=settings.secrets) + def update_workflow( self, workflow_id: str, @@ -958,9 +1005,12 @@ def workflow_view(self, workflow: dict[str, Any]) -> dict[str, Any]: "updated_at": workflow["updated_at"], "spec": { "name": spec.name, + "template_id": spec.template_id, + "template_version": spec.template_version, "competition_id": spec.competition_id, "team_id": spec.team_id, "ends_at": spec.ends_at, + "remote_concurrency": spec.remote_concurrency, "remote_instance_limit": spec.remote_instance_limit, "challenges": challenge_view, "submit": submit_view, @@ -1036,6 +1086,7 @@ def _import( raise ValueError(f"unknown external_id values: {missing}") selected = [by_id[external_id] for external_id in select] from tempfile import TemporaryDirectory + from backend.platform.downloads import stage_challenges imported: list[dict[str, Any]] = [] diff --git a/backend/ops/store.py b/backend/ops/store.py index 41c0a14..0580ba1 100644 --- a/backend/ops/store.py +++ b/backend/ops/store.py @@ -11,10 +11,11 @@ from pathlib import Path from typing import Any +from psycopg.errors import UniqueViolation + from backend.core.config import LLMConfig from backend.ops.models import PlatformWorkflowSpec, validate_secret_name from backend.persistence.database import Database, PostgresDatabase -from psycopg.errors import UniqueViolation _RUN_ID_RE = re.compile(r"^run_[A-Za-z0-9_-]{8,128}$") _TERMINAL_RUN_STATUSES = {"completed", "interrupted", "error", "abandoned"} @@ -24,6 +25,10 @@ class WorkflowInUse(RuntimeError): """A workflow still referenced by a competition run cannot be deleted.""" +class WorkflowSessionConflict(RuntimeError): + """An Ops conversation attempted to switch its bound workflow.""" + + class OpsStore: def __init__( self, @@ -120,7 +125,7 @@ def create_session(self, title: str) -> dict[str, str]: session_id = f"ops_{secrets.token_hex(8)}" with self._lock, self._connect() as connection: connection.execute( - "INSERT INTO sessions (id, title, created_at, updated_at) VALUES (%s, %s, %s, %s)", + "INSERT INTO sessions (id, title, created_at, updated_at, agent_workflow_id) VALUES (%s, %s, %s, %s, NULL)", (session_id, title[:120] or "New conversation", now, now), ) return self.get_session(session_id) @@ -128,7 +133,7 @@ def create_session(self, title: str) -> dict[str, str]: def get_session(self, session_id: str) -> dict[str, str]: with self._lock, self._connect() as connection: row = connection.execute( - "SELECT id, title, created_at, updated_at FROM sessions WHERE id = %s", + "SELECT id, title, created_at, updated_at, agent_workflow_id FROM sessions WHERE id = %s", (session_id,), ).fetchone() if row is None: @@ -138,7 +143,7 @@ def get_session(self, session_id: str) -> dict[str, str]: def list_sessions(self) -> list[dict[str, str]]: with self._lock, self._connect() as connection: rows = connection.execute( - "SELECT id, title, created_at, updated_at FROM sessions ORDER BY updated_at DESC" + "SELECT id, title, created_at, updated_at, agent_workflow_id FROM sessions ORDER BY updated_at DESC" ).fetchall() return [dict(row) for row in rows] @@ -160,6 +165,40 @@ def agent_session_id(self, session_id: str) -> str | None: value = row["agent_session_id"] if row is not None else None return str(value) if value else None + def agent_workflow_id(self, session_id: str) -> str | None: + """Return the workflow permanently associated with an Ops session.""" + self.get_session(session_id) + with self._lock, self._connect() as connection: + row = connection.execute( + "SELECT agent_workflow_id FROM sessions WHERE id = %s", (session_id,) + ).fetchone() + value = row["agent_workflow_id"] if row is not None else None + return str(value) if value else None + + def bind_agent_workflow(self, session_id: str, workflow_id: str) -> str: + """Bind a session once; switching workflows requires a new session.""" + self.get_session(session_id) + workflow_id = str(workflow_id or "").strip() + if not workflow_id: + raise ValueError("workflow_id is required") + with self._lock, self._connect() as connection: + if not connection.execute("SELECT 1 FROM workflows WHERE id=%s", (workflow_id,)).fetchone(): + raise KeyError(workflow_id) + row = connection.execute( + "SELECT agent_workflow_id FROM sessions WHERE id=%s FOR UPDATE", + (session_id,), + ).fetchone() + current = row["agent_workflow_id"] if row else None + if current and str(current) != workflow_id: + raise WorkflowSessionConflict( + "this Ops session is already bound to another workflow; create a new session" + ) + connection.execute( + "UPDATE sessions SET agent_workflow_id=%s,updated_at=%s WHERE id=%s", + (workflow_id, _now(), session_id), + ) + return workflow_id + def set_agent_session_id(self, session_id: str, agent_session_id: str | None) -> None: self.get_session(session_id) value = str(agent_session_id).strip() if agent_session_id else None @@ -462,6 +501,15 @@ def create_workflow( workflow_id = f"wf_{secrets.token_hex(8)}" spec_json, digest = _serialize_spec(spec) with self._lock, self._connect() as connection: + if session_id is not None: + bound = connection.execute( + "SELECT agent_workflow_id FROM sessions WHERE id=%s FOR UPDATE", + (session_id,), + ).fetchone() + if bound and bound["agent_workflow_id"]: + raise WorkflowSessionConflict( + "this Ops session is already bound to another workflow; create a new session" + ) connection.execute( """ INSERT INTO workflows ( @@ -471,6 +519,11 @@ def create_workflow( """, (workflow_id, session_id, source, spec.name, spec_json, digest, now, now), ) + if session_id is not None: + connection.execute( + "UPDATE sessions SET agent_workflow_id=%s,updated_at=%s WHERE id=%s", + (workflow_id, now, session_id), + ) if session_id is not None: session_values = self._secret_namespace("sessions", session_id) selected = { @@ -501,6 +554,13 @@ def update_workflow(self, workflow_id: str, spec: PlatformWorkflowSpec) -> dict[ raise KeyError(workflow_id) return self.get_workflow(workflow_id) + def update_session_workflow(self, session_id: str, spec: PlatformWorkflowSpec) -> dict[str, Any]: + """Create the first proposal or update the workflow already bound to a session.""" + bound = self.agent_workflow_id(session_id) + if bound: + return self.update_workflow(bound, spec) + return self.create_workflow(spec, session_id=session_id, source="agent") + def get_workflow(self, workflow_id: str) -> dict[str, Any]: with self._lock, self._connect() as connection: row = connection.execute("SELECT * FROM workflows WHERE id = %s", (workflow_id,)).fetchone() @@ -521,6 +581,39 @@ def list_workflows(self) -> list[dict[str, Any]]: rows = connection.execute("SELECT id FROM workflows ORDER BY updated_at DESC").fetchall() return [self.get_workflow(row["id"]) for row in rows] + def cleanup_duplicate_workflows(self) -> int: + """Remove unreferenced historical drafts while preserving provenance.""" + with self._lock, self._connect() as connection: + rows = connection.execute( + """SELECT w.id,w.session_id,w.status,w.updated_at, + EXISTS (SELECT 1 FROM competition_runs r WHERE r.workflow_id=w.id) AS referenced + FROM workflows w ORDER BY w.session_id NULLS LAST,w.updated_at DESC,w.id DESC""" + ).fetchall() + keep: set[str] = set() + keep.update( + row["agent_workflow_id"] for row in connection.execute( + "SELECT agent_workflow_id FROM sessions WHERE agent_workflow_id IS NOT NULL" + ).fetchall() + ) + latest: set[str] = set() + unbound_kept = False + for row in rows: + sid = row.get("session_id") + if sid and sid not in latest: + latest.add(sid) + keep.add(row["id"]) + elif sid is None and not unbound_kept: + keep.add(row["id"]) + unbound_kept = True + if row.get("referenced"): + keep.add(row["id"]) + doomed = [row["id"] for row in rows if row["id"] not in keep and row.get("status") == "draft"] + if doomed: + connection.execute("DELETE FROM workflows WHERE id = ANY(%s)", (doomed,)) + for workflow_id in doomed: + self._delete_secret_namespace("workflows", workflow_id) + return len(doomed) + def delete_workflow(self, workflow_id: str) -> bool: with self._lock, self._connect() as connection: runs = connection.execute( diff --git a/backend/ops/tools.py b/backend/ops/tools.py index 31a8de6..42965d8 100644 --- a/backend/ops/tools.py +++ b/backend/ops/tools.py @@ -2,6 +2,7 @@ import copy import json +import logging import os import re import shlex @@ -11,6 +12,7 @@ from backend.sandbox.command_policy import host_path_violation from backend.sandbox.docker_manager import _load_docker_sdk +_recovery_logger = logging.getLogger(__name__) class OpsToolError(RuntimeError): """A tool call was invalid or could not be completed.""" @@ -126,7 +128,7 @@ def execute(self, name: str, arguments: dict[str, Any] | None = None, *, session raise OpsToolError("question requires a conversation session") try: return QuestionStore(self.state).create(session_id=session_id, **args) - except (ValueError, TypeError) as exc: + except (ValueError, TypeError, RuntimeError) as exc: raise OpsToolError(str(exc)) from exc handlers = { "list_task_sandboxes": self.list_task_sandboxes, @@ -222,7 +224,7 @@ def host_exec(self, command: str, timeout: int = 60) -> dict[str, Any]: try: container.remove(force=True) except Exception: - pass + _recovery_logger.debug('Error handled in host_exec', exc_info=True) stdout, stderr = _split_output(output) return { @@ -292,6 +294,7 @@ def _host_helper_image(client) -> str: client.images.get(preferred) return preferred except Exception: + _recovery_logger.debug('Error handled in _host_helper_image', exc_info=True) return "ipc-ctfagent:latest" diff --git a/backend/pentest/__init__.py b/backend/pentest/__init__.py new file mode 100644 index 0000000..6c52d3b --- /dev/null +++ b/backend/pentest/__init__.py @@ -0,0 +1 @@ +"""Persistent pentest domain, executed by IPC's existing agent harness.""" diff --git a/backend/pentest/evidence.py b/backend/pentest/evidence.py new file mode 100644 index 0000000..42b3590 --- /dev/null +++ b/backend/pentest/evidence.py @@ -0,0 +1,51 @@ +"""Archive exact artifact bytes with bounded memory before opening a write transaction.""" +from __future__ import annotations + +import base64 +import hashlib +import os +import tempfile +import time +from pathlib import Path + +MAX_EVIDENCE_BYTES = 64 * 1024 * 1024 + + +def archive_pages(read_page, path, directory: Path): + directory.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + offset = 0 + total = None + deadline = time.monotonic() + 30 + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=directory, prefix=".evidence-", delete=False) as output: + temporary = Path(output.name) + while total is None or offset < total: + if time.monotonic() > deadline: + raise ValueError("evidence transfer exceeded its 30 second budget") + page = read_page(path, offset=offset, limit=8192) + size = int(page["total_bytes"]) + if not 0 < size <= MAX_EVIDENCE_BYTES: + raise ValueError("evidence must contain 1 byte to 64 MiB") + if total is not None and total != size: + raise ValueError("evidence artifact changed during transfer") + total = size + raw = base64.b64decode(page["data"], validate=True) if page["encoding"] == "base64" else page["data"].encode("utf-8") + if not raw or int(page["next_offset"]) != offset + len(raw): + raise ValueError("invalid evidence artifact page") + digest.update(raw) + output.write(raw) + offset += len(raw) + output.flush() + os.fsync(output.fileno()) + if offset != total: + raise ValueError("evidence transfer length mismatch") + sha = digest.hexdigest() + target = directory / f"{sha}.bin" + os.replace(temporary, target) + temporary = None + return {"sha256": sha, "size": total, "archive_path": f"evidence/{sha}.bin"} + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) diff --git a/backend/pentest/models.py b/backend/pentest/models.py new file mode 100644 index 0000000..e77f7bc --- /dev/null +++ b/backend/pentest/models.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class Input(BaseModel): + model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) + + +class Setup(Input): + scope: str = Field(default="", max_length=8000) + constraints: list[str] = Field(default_factory=list, max_length=40) + goals: list[str] = Field(default_factory=list, max_length=40) + + @field_validator("constraints", "goals") + @classmethod + def bounded_text(cls, value): + if any(not item.strip() or len(item) > 2000 for item in value): + raise ValueError("entries must contain 1 to 2000 characters") + return value + + +class Goal(Input): + description: str = Field(min_length=1, max_length=2000) + required: bool = True + + +class Asset(Input): + kind: Literal["domain", "ip", "service", "app", "endpoint"] + value: str = Field(min_length=1, max_length=2000) + parent_id: str | None = None + in_scope: bool = True + source: str = Field(default="", max_length=2000) + + +class Evidence(Input): + summary: str = Field(min_length=1, max_length=2000) + path: str = Field(min_length=1, max_length=1000) + command: str = Field(default="", max_length=4000) + + +class Finding(Input): + title: str = Field(min_length=1, max_length=500) + asset_id: str + vulnerability: str = Field(min_length=1, max_length=200) + severity: Literal["info", "low", "medium", "high", "critical"] = "medium" + description: str = Field(min_length=1, max_length=8000) + reproduction: str = Field(min_length=1, max_length=8000) + remediation: str = Field(default="", max_length=8000) + evidence_ids: list[str] = Field(min_length=1, max_length=30) + + +class VerifyFinding(Input): + finding_id: str + outcome: Literal["confirmed", "rejected", "inconclusive"] + evidence_ids: list[str] = Field(min_length=1, max_length=30) + note: str = Field(min_length=1, max_length=4000) + + +class Proof(Input): + goal_id: str + evidence_ids: list[str] = Field(min_length=1, max_length=30) + finding_ids: list[str] = Field(default_factory=list, max_length=30) + note: str = Field(min_length=1, max_length=4000) + + +class Work(Input): + description: str = Field(min_length=1, max_length=4000) + goal_id: str + asset_id: str | None = None + depends_on: list[str] = Field(default_factory=list, max_length=30) + evidence_ids: list[str] = Field(default_factory=list, max_length=30) + priority: int = Field(default=5, ge=0, le=10) + deliverable: str = Field(min_length=1, max_length=2000) + + +class Coverage(Input): + asset_id: str + test: str = Field(min_length=1, max_length=500) + state: Literal["in_progress", "tested", "not_applicable", "failed"] + evidence_ids: list[str] = Field(default_factory=list, max_length=30) + + +ACTION_MODELS = { + "add_goal": Goal, "register_asset": Asset, "record_evidence": Evidence, + "report_finding": Finding, "verify_finding": VerifyFinding, + "prove_goal": Proof, "plan_work": Work, "record_coverage": Coverage, +} + + +class Action(Input): + operation: Literal[ + "add_goal", "register_asset", "record_evidence", "report_finding", + "verify_finding", "prove_goal", "plan_work", "record_coverage", + ] + data: dict diff --git a/backend/pentest/projection.py b/backend/pentest/projection.py new file mode 100644 index 0000000..e2a7ee0 --- /dev/null +++ b/backend/pentest/projection.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import hashlib +import json + +from backend.blackboard import graph_store +from backend.pentest.store import records + + +def graph(conn, pid, *, view="exploration", offset=0, limit=250, query=""): + detail = graph_store.project_detail(conn, pid) + if detail is None: + raise KeyError(pid) + rows = records(conn, pid) + nodes, edges = [], [] + + def node(rid, kind, label, state="", **data): + nodes.append({"data": {"id": rid, "kind": kind, "label": label, "state": state, **data}}) + + def edge(source, target, rel): + edges.append({"data": {"id": f"{rel}:{source}:{target}", "source": source, "target": target, "rel": rel, "label": rel}}) + + if view == "assets": + coverage = [row for row in rows if row["kind"] == "coverage"] + for row in rows: + if row["kind"] != "asset": + continue + data = row["data"] + tests = [item for item in coverage if item["data"]["asset_id"] == row["id"]] + tested = any(item["state"] == "tested" for item in tests) + node(row["id"], data["kind"], data["value"], "tested" if tested else "untested", + in_scope=data["in_scope"], tested=tested, record=row, tests=tests) + if data["parent_id"]: + edge(data["parent_id"], row["id"], "contains") + else: + for fact in detail.facts: + if fact.id != "goal": + node(f"fact:{fact.id}", "origin" if fact.id == "origin" else "fact", fact.description, "confirmed") + for intent in detail.intents: + iid = f"intent:{intent.id}" + node(iid, "intent", intent.description, "done" if intent.to else ("running" if intent.worker else "pending"), record=intent.model_dump()) + for source in intent.from_: + if source != "goal": + edge(f"fact:{source}", iid, "derived_from") + if intent.to and intent.to != "goal": + edge(iid, f"fact:{intent.to}", "yields") + for hint in detail.hints: + node(f"hint:{hint.id}", "hint", hint.content) + edge(f"hint:{hint.id}", "fact:origin", "derived_from") + for row in rows: + data = row["data"] + if row["kind"] == "goal": + node(row["id"], "goal", data["description"], row["state"], record=row) + edge("fact:origin", row["id"], "spawns") + for finding in data.get("proof", {}).get("finding_ids", []): + edge(finding, row["id"], "proves") + for evidence in data.get("proof", {}).get("evidence_ids", []): + edge(evidence, row["id"], "proves") + elif row["kind"] in {"finding", "evidence"}: + node(row["id"], row["kind"], data.get("title") or data["summary"], row["state"], record=row) + producer = data.get("producer", {}).get("intent_id") + edge(f"intent:{producer}" if producer else "fact:origin", row["id"], "yields") + for evidence in data.get("evidence_ids", []): + edge(evidence, row["id"], "proves") + for row in rows: + if row["kind"] == "work" and row["data"].get("intent_id"): + edge(row["data"]["goal_id"], f"intent:{row['data']['intent_id']}", "spawns") + ids = {item["data"]["id"] for item in nodes} + edges = list({item["data"]["id"]: item for item in edges if item["data"]["source"] in ids and item["data"]["target"] in ids}.values()) + coverage = {"in_scope": sum(n["data"].get("in_scope", False) for n in nodes), + "tested": sum(n["data"].get("in_scope", False) and n["data"].get("tested", False) for n in nodes)} + version = hashlib.sha256(json.dumps({"nodes": nodes, "edges": edges}, sort_keys=True).encode()).hexdigest() + query = query.strip().casefold() + matching = [n for n in nodes if not query or query in f"{n['data']['kind']} {n['data']['label']} {n['data']['state']}".casefold()] + page = matching[offset:offset + limit] + selected = {n["data"]["id"] for n in page} + internal = [e for e in edges if e["data"]["source"] in selected and e["data"]["target"] in selected] + boundary = [e for e in edges if e not in internal and (e["data"]["source"] in selected or e["data"]["target"] in selected)] + return {"nodes": page, "edges": internal, "boundary_edges": boundary, "view": view, "coverage": coverage, + "total_nodes": len(matching), "next_offset": offset + len(page), "has_more": offset + len(page) < len(matching), + "version": version} diff --git a/backend/pentest/report.py b/backend/pentest/report.py new file mode 100644 index 0000000..eadb612 --- /dev/null +++ b/backend/pentest/report.py @@ -0,0 +1,38 @@ +from backend.blackboard import graph_store +from backend.core.redaction import redact_object +from backend.core.wp_writer import _redact_text +from backend.pentest.store import records + + +def render_report(conn, project_id): + project = graph_store.get_project_row(conn, project_id) + rows = records(conn, project_id) + sections = [f"# {project['title']}\n", "## Pentest assessment", f"Status: {project['status']}\n"] + for config in (item for item in rows if item["kind"] == "config"): + sections += ["## Scope and constraints", config["data"]["scope"], *[f"- {rule}" for rule in config["data"]["constraints"]]] + sections += ["\n## Objectives"] + for goal in (item for item in rows if item["kind"] == "goal"): + sections += [f"- [{goal['state']}] {goal['data']['description']}", str(goal["data"].get("proof", {}).get("note", ""))] + sections += ["\n## Findings and verification"] + for finding in (item for item in rows if item["kind"] == "finding"): + data = finding["data"] + sections += [f"### {data['title']} ({finding['state']}, {data['severity']})", data["description"], + "Reproduction: " + data["reproduction"], "Remediation: " + data["remediation"], + "Evidence: " + ", ".join(data["evidence_ids"])] + for review in data.get("reviews", []): + sections.append(f"Review: {review['outcome']} - {review['note']}") + sections += ["\n## Evidence manifest"] + for evidence in (item for item in rows if item["kind"] == "evidence"): + data = evidence["data"] + sections += [f"- {evidence['id']}: {data['summary']}", f" Artifact: {data['path']}\n SHA-256: {data['sha256']}", + f" Command: {data['command']}"] + sections += ["\n## Coverage and remaining work"] + for item in rows: + if item["kind"] in {"work", "coverage"}: + sections.append(f"- [{item['state']}] {item['data'].get('description') or item['data'].get('test')}") + return _redact_text(redact_object("\n\n".join(sections) + "\n", project.get("flag"))) + + +def validate_report(content): + required = ("# ", "## Pentest assessment", "## Objectives", "## Evidence manifest") + return [] if isinstance(content, str) and all(text in content for text in required) else ["incomplete pentest assessment report"] diff --git a/backend/pentest/schema.py b/backend/pentest/schema.py new file mode 100644 index 0000000..7c3b005 --- /dev/null +++ b/backend/pentest/schema.py @@ -0,0 +1,22 @@ +PENTEST_SCHEMA = ( + "ALTER TABLE projects ADD COLUMN IF NOT EXISTS completion_policy TEXT NOT NULL DEFAULT 'verified_flag'", + """CREATE TABLE IF NOT EXISTS pentest_records ( + project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('config','goal','asset','evidence','finding','work','coverage')), + dedupe_key TEXT NOT NULL, + state TEXT NOT NULL, + data JSONB NOT NULL DEFAULT '{}'::jsonb, + revision BIGINT NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (project_id,id), UNIQUE (project_id,kind,dedupe_key) + )""", + """CREATE TABLE IF NOT EXISTS pentest_events ( + sequence BIGSERIAL PRIMARY KEY, + project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + kind TEXT NOT NULL, record_id TEXT NOT NULL, revision BIGINT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + )""", + "CREATE INDEX IF NOT EXISTS idx_pentest_events_project ON pentest_events(project_id,sequence)", +) diff --git a/backend/pentest/store.py b/backend/pentest/store.py new file mode 100644 index 0000000..21fc83d --- /dev/null +++ b/backend/pentest/store.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +import hashlib +import ipaddress +import json +import re +from urllib.parse import urlsplit, urlunsplit + +from psycopg.types.json import Jsonb + +from backend.blackboard import edge_store, graph_store, node_store +from backend.core.postprocess_store import enqueue_postprocess +from backend.pentest.models import ACTION_MODELS, Setup + +TERMINAL = {"solved", "completed", "stopped", "failed", "timeout", "infra_error"} + + +def _key(*values) -> str: + return hashlib.sha256(json.dumps(values, sort_keys=True, ensure_ascii=False).encode()).hexdigest() + + +def project(conn, project_id: str, *, writable=False, allow_completed=False): + if writable: + conn.execute("SELECT pg_advisory_xact_lock(hashtext(%s))", (project_id,)) + row = conn.execute("SELECT * FROM projects WHERE id=%s FOR UPDATE", (project_id,)).fetchone() + if row is None: + raise KeyError(project_id) + if row["category"] != "pentest": + raise ValueError("project is not a pentest task") + if writable and row["status"] in TERMINAL and not (allow_completed and row["status"] == "completed"): + raise ValueError("pentest task is not accepting new work") + return row + + +def record(conn, project_id, record_id, kind=None): + row = conn.execute("SELECT * FROM pentest_records WHERE project_id=%s AND id=%s", (project_id, record_id)).fetchone() + if row is None or (kind and row["kind"] != kind): + raise ValueError(f"unknown {kind or 'record'} in this project: {record_id}") + return row + + +def records(conn, project_id, kind=None): + return conn.execute( + "SELECT * FROM pentest_records WHERE project_id=%s AND (%s::text IS NULL OR kind=%s) ORDER BY created_at,id", + (project_id, kind, kind), + ).fetchall() + + +def _save(conn, project_id, kind, key, data, state="pending", *, update=False): + old = conn.execute( + "SELECT * FROM pentest_records WHERE project_id=%s AND kind=%s AND dedupe_key=%s", + (project_id, kind, key), + ).fetchone() + if old and (not update or (old["data"] == data and old["state"] == state)): + return old + rid = old["id"] if old else f"{kind}:{_key(kind, key)[:20]}" + row = conn.execute( + """INSERT INTO pentest_records(project_id,id,kind,dedupe_key,state,data) + VALUES (%s,%s,%s,%s,%s,%s) + ON CONFLICT (project_id,id) DO UPDATE SET data=EXCLUDED.data,state=EXCLUDED.state, + revision=pentest_records.revision+1,updated_at=now() RETURNING *""", + (project_id, rid, kind, key, state, Jsonb(data)), + ).fetchone() + conn.execute( + "INSERT INTO pentest_events(project_id,kind,record_id,revision) VALUES (%s,%s,%s,%s)", + (project_id, kind, rid, row["revision"]), + ) + graph_store.touch_project(conn, project_id) + conn.execute("SELECT pg_notify('ipc_scheduler',%s)", (f"pentest:{project_id}",)) + return row + + +def initialize(conn, project_id, goal, scope, setup: Setup | None = None): + setup = setup or Setup() + _save(conn, project_id, "config", "config", {"scope": setup.scope or scope, "constraints": setup.constraints}, "active") + for description in setup.goals or [goal]: + _save(conn, project_id, "goal", _key(description.strip().casefold()), {"description": description, "required": True}) + + +def snapshot(conn, project_id, *, after=0, limit=500, record_offset=0, record_limit=None): + row = project(conn, project_id) + all_records = records(conn, project_id) + events = conn.execute( + "SELECT * FROM pentest_events WHERE project_id=%s AND sequence>%s ORDER BY sequence LIMIT %s", + (project_id, after, limit), + ).fetchall() + version = conn.execute("SELECT COALESCE(max(sequence),0) AS version FROM pentest_events WHERE project_id=%s", (project_id,)).fetchone()["version"] + counts = {kind: sum(item["kind"] == kind for item in all_records) for kind in ("goal", "asset", "finding", "evidence", "work", "coverage")} + selected = all_records[record_offset:record_offset + record_limit] if record_limit is not None else all_records + progress = conn.execute("SELECT metadata FROM competition_challenges WHERE project_id=%s ORDER BY id LIMIT 1", (project_id,)).fetchone() + return {"platform_progress": (progress["metadata"].get("flag_progress") or {}) if progress else {}, + "project_id": project_id, "completion_policy": row["completion_policy"], "status": row["status"], + "records": selected, "record_counts": counts, + "goals": [item for item in all_records if item["kind"] == "goal"], + "scope": next((item["data"] for item in all_records if item["kind"] == "config"), {}), + "records_next_offset": record_offset + len(selected), "records_more": record_offset + len(selected) < len(all_records), + "version": version, "events": events, + "next_cursor": events[-1]["sequence"] if events else after, + "has_more": bool(events and events[-1]["sequence"] < version)} + + +def _refs(conn, pid, ids, kind): + return [record(conn, pid, rid, kind) for rid in dict.fromkeys(ids)] + + +def _actor_fence(conn, pid, actor): + if not actor or not actor.get("intent_id"): + return + owned = conn.execute( + """SELECT id FROM intents WHERE project_id=%s AND id=%s AND worker=%s + AND lease_owner=%s AND lease_token=%s AND lease_expires_at>now() + AND to_fact_id IS NULL FOR UPDATE""", + (pid, actor["intent_id"], actor["member"], actor.get("lease_owner"), actor.get("lease_token")), + ).fetchone() + if not owned: + raise ValueError("intent lease lost before pentest write") + assignment = actor.get("assignment") + if assignment: + owned = conn.execute( + """SELECT id FROM competition_assignments WHERE id=%s AND epoch=%s + AND lease_owner=%s AND released_at IS NULL AND lease_expires_at>now() FOR UPDATE""", + (assignment["id"], assignment["epoch"], assignment["lease_owner"]), + ).fetchone() + if not owned: + raise ValueError("competition assignment expired before pentest write") + + +def normalize_asset(kind, value): + value = value.strip() + if kind == "domain": + return value.rstrip('.').lower().encode('idna').decode('ascii') + if kind == "ip": + return str(ipaddress.ip_address(value)) + if kind in {"app", "endpoint"} and "://" in value: + parsed = urlsplit(value) + if parsed.username or parsed.password or parsed.scheme not in {"http", "https"}: + raise ValueError("asset URL must use http(s) without credentials") + return urlunsplit((parsed.scheme.lower(), parsed.netloc.lower(), parsed.path or '/', parsed.query, '')) + return value + + +def apply(conn, pid, operation, data, *, actor=None, read_artifact=None): + payload = ACTION_MODELS[operation].model_validate(data).model_dump() + p = project(conn, pid, writable=True, allow_completed=operation == "prove_goal") + _actor_fence(conn, pid, actor) + provenance = {key: value for key, value in (actor or {}).items() if key in {"member", "intent_id", "session_id", "epoch", "instance_generation"}} + payload["producer"] = provenance or {"member": "operator"} + if p["status"] == "completed": + existing = record(conn, pid, payload["goal_id"], "goal") + if existing["data"].get("proof") == payload: + return existing + raise ValueError("completed objective proof cannot be changed") + if operation == "add_goal": + return _save(conn, pid, "goal", _key(payload["description"].casefold()), payload) + if operation == "register_asset": + if payload["parent_id"]: + record(conn, pid, payload["parent_id"], "asset") + payload["normalized"] = normalize_asset(payload["kind"], payload["value"]) + return _save(conn, pid, "asset", _key(payload["kind"], payload["normalized"]), payload, "discovered") + if operation == "record_evidence": + if read_artifact is None: + raise ValueError("artifact reader unavailable") + raw = read_artifact(payload["path"]) + if raw is None: + raise ValueError("evidence artifact does not exist") + if isinstance(raw, dict): + if not re.fullmatch(r"[a-f0-9]{64}", raw.get("sha256", "")) or not 0 < raw.get("size", 0) <= 64 * 1024 * 1024: + raise ValueError("invalid evidence archive metadata") + payload.update(raw) + return _save(conn, pid, "evidence", _key(payload["path"], payload["sha256"]), payload, "recorded") + content = raw.encode() if isinstance(raw, str) else bytes(raw) + if len(content) > 64 * 1024 * 1024: + raise ValueError("evidence artifact exceeds 64 MiB") + if not content: + raise ValueError("evidence artifact is empty") + payload.update(sha256=hashlib.sha256(content).hexdigest(), size=len(content)) + return _save(conn, pid, "evidence", _key(payload["path"], payload["sha256"]), payload, "recorded") + if operation in {"report_finding", "verify_finding", "prove_goal", "record_coverage", "plan_work"}: + evidence = _refs(conn, pid, payload.get("evidence_ids", []), "evidence") + generation = provenance.get("instance_generation") + if generation is not None and any(item["data"].get("producer", {}).get("instance_generation", generation) != generation for item in evidence): + raise ValueError("evidence belongs to an old instance generation") + if operation == "report_finding": + record(conn, pid, payload["asset_id"], "asset") + return _save(conn, pid, "finding", _key(payload["asset_id"], payload["vulnerability"], payload["title"].casefold()), payload, "candidate") + if operation == "verify_finding": + finding = record(conn, pid, payload["finding_id"], "finding") + previous = finding["data"] + if provenance.get("intent_id") and provenance["intent_id"] == previous.get("producer", {}).get("intent_id"): + raise ValueError("use a separate validation intent to review a finding") + if payload in previous.get("reviews", []): + return finding + history = [*previous.get("reviews", []), payload] + return _save(conn, pid, "finding", finding["dedupe_key"], {**previous, "reviews": history}, payload["outcome"], update=True) + if operation == "prove_goal": + goal = record(conn, pid, payload["goal_id"], "goal") + findings = _refs(conn, pid, payload["finding_ids"], "finding") + if any(item["state"] != "confirmed" for item in findings): + raise ValueError("goal proof requires confirmed findings") + result = _save(conn, pid, "goal", goal["dedupe_key"], {**goal["data"], "proof": payload}, "proven", update=True) + if p["completion_policy"] == "goals": + goals = records(conn, pid, "goal") + required = [item for item in goals if item["data"].get("required", True)] + if required and all(item["state"] == "proven" for item in required): + fact = node_store.create_fact(conn, pid, "All required pentest goals have evidence-backed proofs.") + edge_store.complete_goal_intent(conn, pid, [fact.id], "Pentest objectives verified", "diamond") + conn.execute("UPDATE projects SET status='completed',runtime_phase='completion_pending',updated_at=now() WHERE id=%s", (pid,)) + enqueue_postprocess(conn, pid) + return result + if operation == "plan_work": + record(conn, pid, payload["goal_id"], "goal") + if payload["asset_id"]: + record(conn, pid, payload["asset_id"], "asset") + _refs(conn, pid, payload["depends_on"], "work") + return _save(conn, pid, "work", _key(payload["goal_id"], payload["asset_id"], payload["description"].casefold()), payload) + if operation == "record_coverage": + record(conn, pid, payload["asset_id"], "asset") + if payload["state"] in {"tested", "not_applicable"} and not payload["evidence_ids"]: + raise ValueError("completed coverage requires execution evidence") + return _save(conn, pid, "coverage", _key(payload["asset_id"], payload["test"]), payload, payload["state"], update=True) + raise ValueError("unsupported pentest operation") + + +def plan_next(conn, pid, detail, trigger, *, max_open=2): + project(conn, pid, writable=True) + detail = graph_store.project_detail(conn, pid) + # Project locking serializes wakeups. Persistent work->intent bindings survive restarts. + for work in records(conn, pid, "work"): + iid = work["data"].get("intent_id") + if work["state"] == "running" and iid: + intent = edge_store.get_intent(conn, pid, iid) + if intent and intent["to_fact_id"] is not None: + _save(conn, pid, "work", work["dedupe_key"], {**work["data"], "result_fact_id": intent["to_fact_id"]}, "done", update=True) + open_count = sum(item.to is None for item in detail.intents) + capacity = max(0, max_open - open_count) + created = [] + work_items = sorted(records(conn, pid, "work"), key=lambda item: -item["data"].get("priority", 0)) + for work in work_items: + if len(created) >= capacity: + break + if work["state"] != "pending": + continue + deps = _refs(conn, pid, work["data"]["depends_on"], "work") + if any(item["state"] != "done" for item in deps): + continue + sources = [item["data"]["result_fact_id"] for item in deps] or ["origin"] + description = f"{work['data']['description']}\nDeliverable: {work['data']['deliverable']}\nPentest work: {work['id']}" + intent = edge_store.create_intent(conn, pid, sources, description, "diamond") + _save(conn, pid, "work", work["dedupe_key"], {**work["data"], "intent_id": intent.id}, "running", update=True) + created.append(intent) + if created: + return created[0] + if open_count or not capacity: + return None + # One checkpoint per durable graph revision; repeated ticks cannot manufacture work. + platform = conn.execute("SELECT metadata FROM competition_challenges WHERE project_id=%s LIMIT 1", (pid,)).fetchone() + digest = _key(sorted({item.description for item in detail.facts if item.id not in {"origin", "goal"}}), [item.content for item in detail.hints], + [(item["id"], item["revision"]) for item in records(conn, pid) if item["kind"] != "config"], + platform["metadata"].get("flag_progress") if platform else None) + config = records(conn, pid, "config")[0] + if config["data"].get("planning_checkpoint") == digest: + return None + _save(conn, pid, "config", "config", {**config["data"], "planning_checkpoint": digest}, "active", update=True) + source = next((item.id for item in reversed(detail.facts) if item.id != "goal"), "origin") + return edge_store.create_intent(conn, pid, [source], + "Review the persistent pentest board and new evidence. Validate candidate findings in this separate intent; " + "plan only prerequisite-ready work, retain failed routes, and prove goals only with actual evidence. " + "If no useful work remains, conclude with a concrete blocker; do not repeat the same probes.", "diamond") diff --git a/backend/persistence/database.py b/backend/persistence/database.py index c4a631d..0f5da6d 100644 --- a/backend/persistence/database.py +++ b/backend/persistence/database.py @@ -154,9 +154,8 @@ def configure(self) -> PostgresDatabase: @contextmanager def connect(self) -> Generator[DatabaseConnection, None, None]: self.open() - with self._pool.connection() as raw: - with raw.transaction(): - yield DatabaseConnection(raw) + with self._pool.connection() as raw, raw.transaction(): + yield DatabaseConnection(raw) def close(self) -> None: if self._opened: diff --git a/backend/persistence/migrations/versions/20260807_0001_postgres_core.py b/backend/persistence/migrations/versions/20260807_0001_postgres_core.py index b31339b..3d6129c 100644 --- a/backend/persistence/migrations/versions/20260807_0001_postgres_core.py +++ b/backend/persistence/migrations/versions/20260807_0001_postgres_core.py @@ -1,4 +1,3 @@ -"""create PostgreSQL-only IPC schema""" from alembic import op diff --git a/backend/persistence/migrations/versions/20260807_0002_goal_intent_uniqueness.py b/backend/persistence/migrations/versions/20260807_0002_goal_intent_uniqueness.py index b27debc..16b1887 100644 --- a/backend/persistence/migrations/versions/20260807_0002_goal_intent_uniqueness.py +++ b/backend/persistence/migrations/versions/20260807_0002_goal_intent_uniqueness.py @@ -1,8 +1,6 @@ -"""deduplicate terminal goal intents and enforce one per project""" from alembic import op - revision = "20260807_0002" down_revision = "20260807_0001" branch_labels = None diff --git a/backend/persistence/migrations/versions/20260920_0003_competition.py b/backend/persistence/migrations/versions/20260920_0003_competition.py index 31664f9..3409ede 100644 --- a/backend/persistence/migrations/versions/20260920_0003_competition.py +++ b/backend/persistence/migrations/versions/20260920_0003_competition.py @@ -1,4 +1,3 @@ -"""Competition identity, immutable deadlines, global slots and session events.""" from alembic import op revision = "20260920_0003" diff --git a/backend/persistence/migrations/versions/20260920_0004_competition_runtime.py b/backend/persistence/migrations/versions/20260920_0004_competition_runtime.py index b32df5d..205e011 100644 --- a/backend/persistence/migrations/versions/20260920_0004_competition_runtime.py +++ b/backend/persistence/migrations/versions/20260920_0004_competition_runtime.py @@ -1,8 +1,6 @@ -"""Complete the competition runtime, recovery, verdict and replay schema.""" from alembic import op - revision = "20260920_0004" down_revision = "20260920_0003" branch_labels = None diff --git a/backend/persistence/migrations/versions/20260920_0005_competition_artifacts.py b/backend/persistence/migrations/versions/20260920_0005_competition_artifacts.py index eed5a80..60a4d68 100644 --- a/backend/persistence/migrations/versions/20260920_0005_competition_artifacts.py +++ b/backend/persistence/migrations/versions/20260920_0005_competition_artifacts.py @@ -1,8 +1,6 @@ -"""Version shared competition artifacts to prevent silent lost updates.""" from alembic import op - revision = "20260920_0005" down_revision = "20260920_0004" branch_labels = None diff --git a/backend/persistence/migrations/versions/20260921_0006_redact_competition_credentials.py b/backend/persistence/migrations/versions/20260921_0006_redact_competition_credentials.py index 4da01de..a2cbfac 100644 --- a/backend/persistence/migrations/versions/20260921_0006_redact_competition_credentials.py +++ b/backend/persistence/migrations/versions/20260921_0006_redact_competition_credentials.py @@ -1,8 +1,6 @@ -"""Remove provider credentials accidentally persisted in early run snapshots.""" from alembic import op - revision = "20260921_0006" down_revision = "20260920_0005" branch_labels = None diff --git a/backend/persistence/migrations/versions/20260923_0007_engine_observations.py b/backend/persistence/migrations/versions/20260923_0007_engine_observations.py index 2bc8b27..c4698b7 100644 --- a/backend/persistence/migrations/versions/20260923_0007_engine_observations.py +++ b/backend/persistence/migrations/versions/20260923_0007_engine_observations.py @@ -1,8 +1,6 @@ -"""Fence legacy/competition schedulers and persist run observations.""" from alembic import op - revision = "20260923_0007" down_revision = "20260921_0006" branch_labels = None diff --git a/backend/persistence/migrations/versions/20260925_0008_run_scoped_challenges.py b/backend/persistence/migrations/versions/20260925_0008_run_scoped_challenges.py index 6aecc88..e52f2b2 100644 --- a/backend/persistence/migrations/versions/20260925_0008_run_scoped_challenges.py +++ b/backend/persistence/migrations/versions/20260925_0008_run_scoped_challenges.py @@ -1,8 +1,6 @@ -"""Fence competition challenge execution state to a single run.""" from alembic import op - revision = "20260925_0008" down_revision = "20260923_0007" branch_labels = None diff --git a/backend/persistence/migrations/versions/20260929_0009_agent_sessions.py b/backend/persistence/migrations/versions/20260929_0009_agent_sessions.py index db208dc..acf73af 100644 --- a/backend/persistence/migrations/versions/20260929_0009_agent_sessions.py +++ b/backend/persistence/migrations/versions/20260929_0009_agent_sessions.py @@ -1,17 +1,7 @@ -"""Unified agent sessions: append-only transcript shared by every runtime. - -The competition tables are the only existing producer of durable agent -history, so they are backfilled here. Session ids are preserved verbatim: -five tables reference ``competition_sessions(id)``, and reusing the id lets -those foreign keys be re-pointed at ``agent_sessions`` without rewriting any -child row. -""" - from alembic import op from backend.agent.schema import AGENT_SCHEMA - revision = "20260929_0009" down_revision = "20260925_0008" branch_labels = None diff --git a/backend/persistence/migrations/versions/20261008_0010_agent_jobs.py b/backend/persistence/migrations/versions/20261008_0010_agent_jobs.py index 2bd6f26..b5a6fbd 100644 --- a/backend/persistence/migrations/versions/20261008_0010_agent_jobs.py +++ b/backend/persistence/migrations/versions/20261008_0010_agent_jobs.py @@ -1,19 +1,5 @@ -"""Detached sandbox jobs, and a health projection over agent sessions. - -Everything a Member ran was previously wrapped in ``timeout -k 5s 60s``, so -brute force, large file analysis and slow scans were killed rather than -finished. A job is started detached and polled instead, and its durable row is -what lets a poll survive a worker restart. - -The health view answers separately what the UI used to show as a single green -dot: API up, coordinator ticking, worker holding the seat, task progressing. - -Additive only: one new table, its indexes, and a view over existing columns. -""" - from alembic import op - revision = "20261008_0010" down_revision = "20260929_0009" branch_labels = None diff --git a/backend/persistence/migrations/versions/20261013_0011_ops_workflow_limits.py b/backend/persistence/migrations/versions/20261013_0011_ops_workflow_limits.py new file mode 100644 index 0000000..c48f2ff --- /dev/null +++ b/backend/persistence/migrations/versions/20261013_0011_ops_workflow_limits.py @@ -0,0 +1,34 @@ + +from alembic import op + +revision = "20261013_0011" +down_revision = "20261008_0010" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("ALTER TABLE sessions ADD COLUMN IF NOT EXISTS agent_workflow_id TEXT") + op.execute( + "CREATE INDEX IF NOT EXISTS ix_ops_sessions_workflow ON sessions(agent_workflow_id)" + ) + op.execute( + "ALTER TABLE competition_runs ADD COLUMN IF NOT EXISTS limits JSONB NOT NULL DEFAULT '{}'::jsonb" + ) + op.execute( + """UPDATE workflows w SET name='Imported competition ' || w.id + WHERE lower(trim(w.name)) IN ('proposed','probe','workflow','new workflow') + AND EXISTS (SELECT 1 FROM competition_runs r WHERE r.workflow_id=w.id)""" + ) + op.execute( + """DELETE FROM workflows w + WHERE lower(trim(w.name)) IN ('proposed','probe','workflow','new workflow') + AND NOT EXISTS (SELECT 1 FROM competition_runs r WHERE r.workflow_id=w.id) + AND NOT EXISTS (SELECT 1 FROM sessions s WHERE s.agent_workflow_id=w.id)""" + ) + + +def downgrade(): + op.execute("DROP INDEX IF EXISTS ix_ops_sessions_workflow") + op.execute("ALTER TABLE sessions DROP COLUMN IF EXISTS agent_workflow_id") + op.execute("ALTER TABLE competition_runs DROP COLUMN IF EXISTS limits") diff --git a/backend/persistence/migrations/versions/20261016_0012_control_operations.py b/backend/persistence/migrations/versions/20261016_0012_control_operations.py new file mode 100644 index 0000000..fb43a67 --- /dev/null +++ b/backend/persistence/migrations/versions/20261016_0012_control_operations.py @@ -0,0 +1,21 @@ +from alembic import op + +from backend.agent.schema import AGENT_SCHEMA +from backend.competition.control_schema import CONTROL_SCHEMA + +revision = "20261016_0012" +down_revision = "20261013_0011" +branch_labels = None +depends_on = None + + +def upgrade(): + for statement in CONTROL_SCHEMA: + op.execute(statement) + for statement in AGENT_SCHEMA: + if statement.startswith("CREATE OR REPLACE VIEW agent_session_health"): + op.execute(statement) + + +def downgrade(): + op.execute("DROP TABLE IF EXISTS competition_control_operations") diff --git a/backend/persistence/migrations/versions/20261017_0013_pentest.py b/backend/persistence/migrations/versions/20261017_0013_pentest.py new file mode 100644 index 0000000..d4db95a --- /dev/null +++ b/backend/persistence/migrations/versions/20261017_0013_pentest.py @@ -0,0 +1,19 @@ +from alembic import op + +from backend.pentest.schema import PENTEST_SCHEMA + +revision = "20261017_0013" +down_revision = "20261016_0012" +branch_labels = None +depends_on = None + + +def upgrade(): + for statement in PENTEST_SCHEMA: + op.execute(statement) + + +def downgrade(): + op.execute("DROP TABLE IF EXISTS pentest_events") + op.execute("DROP TABLE IF EXISTS pentest_records") + op.execute("ALTER TABLE projects DROP COLUMN IF EXISTS completion_policy") diff --git a/backend/persistence/schema.py b/backend/persistence/schema.py index e7635aa..2c194bc 100644 --- a/backend/persistence/schema.py +++ b/backend/persistence/schema.py @@ -1,6 +1,5 @@ from __future__ import annotations - SCHEMA_COMPATIBILITY_STATEMENTS = ( "ALTER TABLE IF EXISTS projects ADD COLUMN IF NOT EXISTS external_id TEXT", "ALTER TABLE IF EXISTS projects ADD COLUMN IF NOT EXISTS category TEXT NOT NULL DEFAULT 'misc'", @@ -230,7 +229,8 @@ title TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL, updated_at TIMESTAMPTZ NOT NULL, - agent_session_id TEXT + agent_session_id TEXT, + agent_workflow_id TEXT ) """, """ @@ -293,6 +293,8 @@ updated_at TIMESTAMPTZ NOT NULL ) """, + "ALTER TABLE sessions ADD COLUMN IF NOT EXISTS agent_workflow_id TEXT", + "CREATE INDEX IF NOT EXISTS ix_ops_sessions_workflow ON sessions(agent_workflow_id)", """ CREATE TABLE IF NOT EXISTS active_sessions ( session_hash BYTEA PRIMARY KEY, @@ -359,8 +361,10 @@ from backend.agent.schema import AGENT_SCHEMA +from backend.competition.control_schema import CONTROL_SCHEMA from backend.competition.schema import COMPETITION_SCHEMA +from backend.pentest.schema import PENTEST_SCHEMA -SCHEMA_STATEMENTS = (*SCHEMA_STATEMENTS, *COMPETITION_SCHEMA, *AGENT_SCHEMA) +SCHEMA_STATEMENTS = (*SCHEMA_STATEMENTS, *COMPETITION_SCHEMA, *AGENT_SCHEMA, *CONTROL_SCHEMA, *PENTEST_SCHEMA) SCHEMA_CONTRACT_STATEMENTS = (*SCHEMA_COMPATIBILITY_STATEMENTS, *SCHEMA_STATEMENTS) diff --git a/backend/platform/README.md b/backend/platform/README.md new file mode 100644 index 0000000..ada14ec --- /dev/null +++ b/backend/platform/README.md @@ -0,0 +1,68 @@ +# Platform adapters + +Shared contracts and helpers stay at this level. Every platform owns a package +under this directory: + +```text +backend/platform/ + adapter.py + factory.py + mapping.py + gzctf/ + __init__.py + adapter.py + ret2shell/ + __init__.py + adapter.py + mcp.py + tsecbench/ + __init__.py + adapter.py +``` + +Platform packages expose their client and adapter through `__init__.py`. +`factory.py` imports those public exports. URLs, authentication, response +parsing, attachment handling, and instance lifecycle code stay inside the +matching platform package. + +To add a platform, create `backend/platform//`, expose its public types +through that package's `__init__.py`, and register it in `factory.py`. + +GZCTF participant sessions use the `GZCTF_Token` ASP.NET Identity cookie. +Set `IPC_GZ_BASE_URL` and `IPC_GZ_TOKEN` for IPC to use an existing browser +session. `IPC_GZ_USERNAME` and `IPC_GZ_PASSWORD` remain a fallback when no +cookie is supplied. Workflow credentials take precedence over deployment +defaults. Secrets are installed only in the private HTTP session. + +TSecBench tasks authenticate with the `BENCHMARK_TOKEN` header from the +workflow secret store (`benchmark_token`). Challenge entries are direct +`ip:port` addresses behind the benchmark VPN, which the deployment connects; +the adapter never dials the VPN itself. The platform caps active containers +at three, published through `instance_limit()`. + +## Built-in workflows + +The fixed workflow definition and personalization model live with their adapter: +`gzctf/workflow.py` and `tsecbench/workflow.py`. `backend.ops.presets` remains the +shared registry and re-exports its original symbols for compatibility. + +Use `POST /api/ops/workflows/gzctf` with the platform origin, game ID and token +or account credentials. Use `POST /api/ops/workflows/tsecbench` with the benchmark +origin and `secrets.benchmark_token`; no game ID is required. The Competition UI +provides both setup forms. Saved configurations can be edited and reused, while +each run retains its original template version and configuration snapshot. + +Optional `category_map` overrides classify specific external IDs, source +categories, or `*`; `{"*":"pentest"}` imports tasks into the Pentest workspace. +TSecBench concurrency is bounded to three, with zero inheriting the platform +limit. Hints reduce scores and require explicit `allow_hints: true`; preflight +never requests a hint or starts a container. + +TSecBench preflight reports application-side connectivity to any already +available target addresses, plus the separate requirement for VPN connectivity +from task containers. A successful platform API request alone does not establish +target reachability. Partial correct submissions and per-flag duplicate results +retain solver capacity until all platform flags are complete. + +`python scripts/check_workflow_package.py` verifies that both workflows can be +loaded from an installed wheel outside the source tree. diff --git a/backend/platform/__init__.py b/backend/platform/__init__.py index f70a5dc..2758470 100644 --- a/backend/platform/__init__.py +++ b/backend/platform/__init__.py @@ -3,4 +3,4 @@ from backend.platform.gzctf import GZCTFAdapter, GZCTFClient from backend.platform.mapping import FieldMapping, PlatformChallenge -__all__ = ["FieldMapping", "HttpJsonAdapter", "GZCTFAdapter", "GZCTFClient", "PlatformChallenge", "build_adapter"] +__all__ = ["FieldMapping", "GZCTFAdapter", "GZCTFClient", "HttpJsonAdapter", "PlatformChallenge", "build_adapter"] diff --git a/backend/platform/_common.py b/backend/platform/_common.py index 22bfbd5..ab79319 100644 --- a/backend/platform/_common.py +++ b/backend/platform/_common.py @@ -1,16 +1,9 @@ -"""Small, dependency-free helpers shared by platform adapters. - -Platform implementations are deliberately self contained. Keeping these -helpers here avoids importing application configuration or attachment -utilities into an adapter just to normalize a category or filename. -""" from __future__ import annotations from collections.abc import Iterable - PLATFORM_CATEGORIES: tuple[str, ...] = ( - "pwn", "reverse", "crypto", "web", "misc", "ai", "osint", + "pwn", "reverse", "crypto", "web", "misc", "ai", "osint", "pentest", ) _INVALID_FILENAME_CHARS = set('<>:"/\\|?*') diff --git a/backend/platform/adapter.py b/backend/platform/adapter.py index 0734674..3a8ff67 100644 --- a/backend/platform/adapter.py +++ b/backend/platform/adapter.py @@ -1,13 +1,14 @@ from __future__ import annotations from abc import ABC, abstractmethod -from pathlib import Path from collections.abc import Callable +from pathlib import Path from typing import Any from urllib.parse import unquote, urljoin, urlparse import requests +from backend.core.validation import InputValidationError from backend.platform._common import PLATFORM_CATEGORIES, numbered_filename, safe_stem from backend.platform.mapping import FieldMapping, PlatformChallenge @@ -108,7 +109,7 @@ def fetch_challenges(self) -> list[PlatformChallenge]: raise ValueError("challenge list endpoint did not return JSON") from exc items = _json_path(payload, self.mapping.list_path) if not isinstance(items, list): - raise ValueError(f"list_path '{self.mapping.list_path}' did not resolve to a list") + raise InputValidationError(f"list_path '{self.mapping.list_path}' did not resolve to a list") pages.extend(self._normalize(item) for item in items) if not self.mapping.pagination_path: break @@ -116,7 +117,7 @@ def fetch_challenges(self) -> list[PlatformChallenge]: if next_url in (None, ""): break if not isinstance(next_url, str): - raise ValueError("pagination_path must resolve to a URL or null") + raise InputValidationError("pagination_path must resolve to a URL or null") next_url = urljoin(url, next_url) parsed = urlparse(next_url) if parsed.scheme not in {"http", "https"} or not parsed.netloc: @@ -129,7 +130,7 @@ def fetch_challenges(self) -> list[PlatformChallenge]: def _normalize(self, item: Any) -> PlatformChallenge: if not isinstance(item, dict): - raise ValueError("challenge list entries must be JSON objects") + raise InputValidationError("challenge list entries must be JSON objects") external_id = _field(item, self.mapping.id_field) title = _field(item, self.mapping.title_field) if external_id is None or title is None: @@ -141,7 +142,7 @@ def _normalize(self, item: Any) -> PlatformChallenge: if isinstance(raw_attachments, str): raw_attachments = [raw_attachments] if not isinstance(raw_attachments, list): - raise ValueError("configured attachments field must contain a URL or list of URLs") + raise InputValidationError("configured attachments field must contain a URL or list of URLs") base_url = self.mapping.attachment_base_url or self.mapping.list_url attachment_urls = [urljoin(base_url, str(url)) for url in raw_attachments] return PlatformChallenge( @@ -152,6 +153,7 @@ def _normalize(self, item: Any) -> PlatformChallenge: attachment_urls=attachment_urls, remote=bool(self.mapping.remote_field and _field(item, self.mapping.remote_field, False) is True), solved=bool(self.mapping.solved_field and _field(item, self.mapping.solved_field, False) is True), + platform_data={"score": _field(item, self.mapping.score_field)} if self.mapping.score_field else {}, hints=([str(h) for h in _field(item, self.mapping.hints_field, [])] if self.mapping.hints_field and isinstance(_field(item, self.mapping.hints_field, []), list) else []), ) diff --git a/backend/platform/downloads.py b/backend/platform/downloads.py index cd286f8..1dcb352 100644 --- a/backend/platform/downloads.py +++ b/backend/platform/downloads.py @@ -1,11 +1,10 @@ -"""Bounded parallel staging with a fresh adapter for each challenge.""" from __future__ import annotations +from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from contextlib import ExitStack from pathlib import Path from threading import BoundedSemaphore -from typing import Callable from urllib.parse import urlsplit diff --git a/backend/platform/factory.py b/backend/platform/factory.py index 9a4f7db..558c6b3 100644 --- a/backend/platform/factory.py +++ b/backend/platform/factory.py @@ -1,4 +1,3 @@ -"""Construct platform adapters without leaking platform branches into IPC.""" from __future__ import annotations from collections.abc import Callable, Mapping @@ -11,6 +10,7 @@ from backend.platform.gzctf import GZCTFAdapter, GZCTFClient from backend.platform.mapping import FieldMapping from backend.platform.ret2shell import Ret2ShellAdapter, Ret2ShellClient +from backend.platform.tsecbench import TSECBenchAdapter, TSECBenchClient def build_adapter( @@ -34,14 +34,32 @@ def build_adapter( category_map=mapping.category_map, max_attachment_bytes=max_attachment_bytes, ) + if mapping.platform == "tsecbench": + resolved_base_url = base_url or mapping.list_url + if resolved_base_url: + parsed = urlsplit(resolved_base_url) + if parsed.scheme and parsed.netloc: + resolved_base_url = f"{parsed.scheme}://{parsed.netloc}" + return TSECBenchAdapter( + TSECBenchClient( + base_url=resolved_base_url, + token=(credentials.get("benchmark_token") or credentials.get("platform_token") or credentials.get("token") or ""), + ), + mapping, + max_attachment_bytes=max_attachment_bytes, + competition_id=competition_id, + team_id=team_id, + ) if mapping.platform == "gzctf": resolved_base_url = base_url or "" source_url = resolved_base_url or mapping.list_url if source_url: parsed = urlsplit(source_url) - if parsed.scheme and parsed.netloc: - if not resolved_base_url or parsed.path.lower().startswith("/api/"): - resolved_base_url = f"{parsed.scheme}://{parsed.netloc}" + if ( + parsed.scheme and parsed.netloc + and (not resolved_base_url or parsed.path.lower().startswith("/api/")) + ): + resolved_base_url = f"{parsed.scheme}://{parsed.netloc}" return GZCTFAdapter( GZCTFClient( base_url=resolved_base_url, diff --git a/backend/platform/gzctf/adapter.py b/backend/platform/gzctf/adapter.py index 075830b..41f6091 100644 --- a/backend/platform/gzctf/adapter.py +++ b/backend/platform/gzctf/adapter.py @@ -1,27 +1,39 @@ from __future__ import annotations -import os import base64 import hashlib +import logging +import os import secrets from datetime import UTC, datetime from pathlib import Path from typing import Any -from urllib.parse import urljoin, urlsplit, unquote +from urllib.parse import unquote, urljoin, urlsplit import requests +from typing_extensions import Self from backend.platform._common import PLATFORM_CATEGORIES, numbered_filename, safe_stem from backend.platform.adapter import PlatformAdapter from backend.platform.mapping import FieldMapping, PlatformChallenge +_recovery_logger = logging.getLogger(__name__) class GZCTFError(RuntimeError): pass class GZCTFLoginError(GZCTFError): - pass + reason = "auth_required" + + +class GZCTFInstanceError(GZCTFError): + """A classified lifecycle failure; a failed renewal never implies destruction.""" + + def __init__(self, message: str, *, reason: str, retry_after: int = 60): + super().__init__(message) + self.reason = reason + self.retry_after = max(5, min(3600, retry_after)) class GZCTFPreflightError(GZCTFError): @@ -81,11 +93,18 @@ def _find_level(track: dict[str, Any], level: int) -> dict[str, Any] | None: def _parse_time(value: Any) -> datetime | None: - if not value or not isinstance(value, str): + if value is None or isinstance(value, bool): return None + if isinstance(value, datetime): + return value if value.tzinfo else value.replace(tzinfo=UTC) try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError: + if isinstance(value, (int, float)) or str(value).strip().replace(".", "", 1).isdigit(): + seconds = float(value) + if seconds > 100_000_000_000: + seconds /= 1000 + return datetime.fromtimestamp(seconds, UTC) + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except (ValueError, TypeError, OverflowError, OSError): return None return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) @@ -454,11 +473,11 @@ def delete_container(self, game_id: int, challenge_id: int) -> None: timeout=self.timeout, ) if response.status_code == 404: - return None + return if response.status_code == 400 and "未创建容器" in response.text: - return None + return response.raise_for_status() - return None + return def get_container(self, game_id: int, challenge_id: int) -> dict[str, Any]: """Read the live container state from the challenge detail. @@ -681,7 +700,7 @@ def create_ticket(self, payload: dict[str, Any]) -> Any: def close(self) -> None: self.session.close() - def __enter__(self) -> GZCTFClient: + def __enter__(self) -> Self: return self def __exit__(self, *exc: object) -> None: @@ -795,8 +814,10 @@ def _gz_instance_view(payload: Any) -> dict[str, Any]: if entry: view["entry"] = str(entry) view["url"] = str(entry) - if expires: - view["expires_at"] = str(expires) + expiry = _parse_time(expires) + if expiry is not None: + view["expires_at"] = expiry.isoformat().replace("+00:00", "Z") + view["renew_before_seconds"] = 300 return view @@ -814,21 +835,38 @@ def _gz_container_error(exc: requests.HTTPError, external_id: str) -> Exception: return GZCTFLoginError( "GZCTF container request was not authorized; provide a new token" ) - if status == 429: - return GZCTFError( - f"GZCTF rate limited the container request for challenge {external_id}" - ) - if status == 404: - return GZCTFError(f"GZCTF container challenge {external_id} does not exist") detail = "" try: detail = (response.text or "").strip()[:300] if response is not None else "" except Exception: + _recovery_logger.debug('Error handled in _gz_container_error', exc_info=True) detail = "" + headers = getattr(response, "headers", {}) or {} + try: + retry_after = int(headers.get("Retry-After", "60")) + except (ValueError, TypeError): + retry_after = 60 + reason = "transient" + if status == 429: + reason = "rate_limited" + elif status == 404: + reason = "missing" + elif status == 400 and any(token in detail.casefold() for token in ( + "尚不可延长", "尚不能延长", "不可续期", "未到续期", "too early", "not yet", "cannot be extended yet", + )): + reason = "too_early" + elif any(token in detail.casefold() for token in ("expired", "not found", "not created", "已过期", "不存在")): + reason = "expired" suffix = f": {detail}" if detail else "" - return GZCTFError( - f"GZCTF container request for challenge {external_id} failed " - f"with status {status or 'unknown'}{suffix}" + if status == 429: + description = f"GZCTF rate limited the container request for challenge {external_id}" + elif status == 404: + description = f"GZCTF container challenge {external_id} does not exist" + else: + description = f"GZCTF container request for challenge {external_id} failed with status {status or 'unknown'}" + return GZCTFInstanceError( + f"{description}{suffix}", + reason=reason, retry_after=retry_after, ) @@ -1045,7 +1083,7 @@ def _normalise(self, raw: dict[str, Any]) -> PlatformChallenge | None: if external is None or title is None: return None category_raw = str(_first_value(raw, self.mapping.category_field, "category", "type", default="misc")) - category = self.mapping.category_map.get(category_raw, category_raw).strip().lower() + category = self.mapping.category_map.get(str(raw.get("id", "")), self.mapping.category_map.get("*", self.mapping.category_map.get(category_raw, category_raw))).strip().lower() if category not in PLATFORM_CATEGORIES: category = "misc" files, descriptors = self._normalise_files( @@ -1332,12 +1370,12 @@ def start_instance(self, external_id: str) -> dict[str, Any]: def stop_instance(self, external_id: str) -> None: if not self.supports_instances: - return None + return try: self.client.delete_container(self.game_id, int(external_id)) except requests.HTTPError as exc: raise _gz_container_error(exc, external_id) from exc - return None + return def renew_instance(self, external_id: str) -> dict[str, Any]: if not self.supports_instances: diff --git a/backend/platform/gzctf/workflow.py b/backend/platform/gzctf/workflow.py new file mode 100644 index 0000000..a685a56 --- /dev/null +++ b/backend/platform/gzctf/workflow.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from urllib.parse import urlsplit + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from backend.core.config import CATEGORIES +from backend.ops.models import PlatformWorkflowSpec, _validate_http_url + +GZCTF_TEMPLATE_VERSION = 1 +TEMPLATE = {'id': 'gzctf', 'name': 'GZCTF', 'version': 1, 'default': True, 'required': ['base_url', 'game_id'], 'optional': ['name', 'team_id', 'ends_at', 'remote_concurrency', 'attachment_base_url', 'allow_private_networks', 'category_map'], 'secret_names': ['gzctf_token', 'gzctf_username', 'gzctf_password'], 'capabilities': ['challenges', 'attachments', 'submit', 'query', 'instances']} + + +class GZCTFWorkflowInput(BaseModel): + model_config = ConfigDict(extra="forbid") + + base_url: str = Field(max_length=2048) + game_id: int = Field(gt=0) + name: str = Field(default="", max_length=120) + team_id: str = Field(default="", max_length=256) + ends_at: str = "" + remote_concurrency: int = Field(default=0, ge=0, le=100) + attachment_base_url: str = "" + category_map: dict[str, str] = Field(default_factory=dict) + allow_private_networks: bool = False + max_attachment_bytes: int = Field(default=100 * 1024 * 1024, ge=1024, le=1024 * 1024 * 1024) + secrets: dict[str, str] = Field(default_factory=dict, repr=False, exclude=True) + + @field_validator("category_map") + @classmethod + def validate_category_map(cls, value): + if any(not key or len(key) > 256 or category not in CATEGORIES for key, category in value.items()): + raise ValueError("category_map must map challenge IDs, category names or * to a supported category") + return value + + @field_validator("base_url") + @classmethod + def validate_base_url(cls, value: str) -> str: + value = _validate_http_url(value).rstrip("/") + parsed = urlsplit(value) + if parsed.query or parsed.path not in ("", "/"): + raise ValueError("base_url must be the GZCTF platform origin, for example https://ctf.example") + return value + + @field_validator("secrets") + @classmethod + def validate_secrets(cls, values: dict[str, str]) -> dict[str, str]: + allowed = {"gzctf_token", "gzctf_username", "gzctf_password", "platform_token"} + if set(values) - allowed: + raise ValueError("GZCTF credentials must use gzctf_token or gzctf_username/gzctf_password") + if any(not value.strip() or len(value) > 16384 for value in values.values()): + raise ValueError("credentials must contain 1 to 16384 characters") + return values + + def workflow_spec(self) -> PlatformWorkflowSpec: + return PlatformWorkflowSpec.model_validate({ + "name": self.name.strip() or f"GZCTF Game {self.game_id}", + "competition_id": str(self.game_id), "team_id": self.team_id, + "template_id": "gzctf", "template_version": GZCTF_TEMPLATE_VERSION, + "ends_at": self.ends_at, "remote_concurrency": self.remote_concurrency, + "allow_private_networks": self.allow_private_networks, + "max_attachment_bytes": self.max_attachment_bytes, + "challenges": {"platform": "gzctf", "game_id": self.game_id, + "list_url": self.base_url, "attachment_base_url": self.attachment_base_url, "category_map": self.category_map}, + }) + + diff --git a/backend/platform/mapping.py b/backend/platform/mapping.py index 45717d5..01b6933 100644 --- a/backend/platform/mapping.py +++ b/backend/platform/mapping.py @@ -8,7 +8,7 @@ class FieldMapping(BaseModel): model_config = ConfigDict(extra="forbid") - platform: Literal["http_json", "ret2shell", "gzctf"] = "http_json" + platform: Literal["http_json", "ret2shell", "gzctf", "tsecbench"] = "http_json" game_id: int | None = None list_url: str = "" list_path: str = "data" @@ -19,6 +19,7 @@ class FieldMapping(BaseModel): attachments_field: str = "files" remote_field: str = "" solved_field: str = "" + score_field: str = "score" hints_field: str = "" level: int = 1 track_id: str = "" @@ -27,6 +28,7 @@ class FieldMapping(BaseModel): pagination_path: str = "" max_pages: int = 100 max_challenges: int = 0 + allow_hints: bool = False category_map: dict[str, str] = Field(default_factory=dict) headers: dict[str, str] = Field(default_factory=dict) attachment_base_url: str = "" @@ -45,6 +47,8 @@ def require_list_url_for_http_json(self) -> FieldMapping: raise ValueError("list_url is required for the http_json platform") if self.platform == "gzctf" and self.game_id is None: raise ValueError("game_id is required for the gzctf platform") + if self.platform == "tsecbench" and not self.list_url: + raise ValueError("list_url is required for the tsecbench platform") return self @field_validator("level") @@ -68,7 +72,7 @@ def validate_max_challenges(cls, value: int) -> int: raise ValueError("max_challenges must be between 0 and 1000") return value - @field_validator("level_field", "track_id_field", "pagination_path") + @field_validator("level_field", "track_id_field", "pagination_path", "score_field") @classmethod def validate_optional_paths(cls, value: str) -> str: value = value.strip() diff --git a/backend/platform/ret2shell/adapter.py b/backend/platform/ret2shell/adapter.py index 1a4c31c..5882f98 100644 --- a/backend/platform/ret2shell/adapter.py +++ b/backend/platform/ret2shell/adapter.py @@ -4,10 +4,12 @@ import os import threading import time +from collections.abc import Callable from pathlib import Path -from typing import Any, Callable +from typing import Any import requests +from typing_extensions import Self from backend.platform._common import PLATFORM_CATEGORIES, numbered_filename, safe_stem from backend.platform.adapter import PlatformAdapter @@ -481,7 +483,7 @@ def submit_flag( def close(self) -> None: self.session.close() - def __enter__(self) -> Ret2ShellClient: + def __enter__(self) -> Self: return self def __exit__(self, *exc: object) -> None: diff --git a/backend/platform/ret2shell/mcp.py b/backend/platform/ret2shell/mcp.py index 19665b4..1a59454 100644 --- a/backend/platform/ret2shell/mcp.py +++ b/backend/platform/ret2shell/mcp.py @@ -1,18 +1,22 @@ from __future__ import annotations import asyncio +import logging import os import shutil import socket import subprocess import threading import time -from typing import Any, Callable +from collections.abc import Callable +from typing import Any from urllib.parse import urlparse from backend.mcp.mcp_server import MCPServer, close_lifespan, create_mcp_server from backend.platform.ret2shell.adapter import Ret2ShellClient +_recovery_logger = logging.getLogger(__name__) + WSRX_BINARY = os.getenv("IPC_R2S_WSRX_BINARY", "wsrx") WSRX_BIND_HOST = os.getenv("IPC_R2S_WSRX_BIND_HOST", "0.0.0.0") WSRX_ENDPOINT_HOST = os.getenv("IPC_R2S_WSRX_ENDPOINT_HOST", "ipc-app") @@ -140,9 +144,10 @@ def _terminate_process(process: Any) -> None: try: process.wait(timeout=5) except Exception: + _recovery_logger.debug('Error handled in _terminate_process', exc_info=True) process.kill() except Exception: - pass + _recovery_logger.debug('Error handled in _terminate_process', exc_info=True) def _alive(self, challenge_id: int) -> list[_Tunnel]: diff --git a/backend/platform/tsecbench/CHALLENGES_API.md b/backend/platform/tsecbench/CHALLENGES_API.md new file mode 100644 index 0000000..f70836e --- /dev/null +++ b/backend/platform/tsecbench/CHALLENGES_API.md @@ -0,0 +1,361 @@ +# Challenges API 接入文档 + +> 本文档面向通过 AI Agent 程序调用 TSec Benchmark Platform 评测接口的开发者或 Coding Agent。 +> +> 文档分为「总览」与「接口详情」两部分: +> - **总览**(§1~§4):认证方式、接口列表与调用流程、注意事项、统一异常处理。 +> - **接口详情**(§5):逐个介绍每个接口的请求/响应及其专属业务异常。 + +--- + +# 使用须知 +在使用本API之前,你必须完成以下步骤: + +1. 获取BENCHMARK_BASE_URL 和 BENCHMARK_TOKEN 配置信息。(在TSecBench平台创建跑分任务后下发) +2. 连接靶场VPN。所有题目入口地址,必须在连接靶场环境VPN后才可访问,否则将无法解题拿到答案。(靶场VPN和BENCHMARK_BASE_URL、BENCHMARK_TOKEN一起在TSecBench平台创建跑分任务后下发) + +# 一、总览 + +## 1. 认证方式 + +所有 `/openapi/v1/challenges/*` 接口通过 HTTP 请求头 `BENCHMARK_TOKEN` 进行鉴权。 + +| 请求头 | 类型 | 必填 | 说明 | +|--------|------|------|------| +| `BENCHMARK_TOKEN` | string (UUID) | 是 | 创建跑分任务时返回的 token,用于标识当前任务 | + +示例: + +```http +GET /openapi/v1/challenges HTTP/1.1 +Host: +BENCHMARK_TOKEN: a1b2c3d4-e5f6-7890-abcd-ef1234567890 +``` + +> 若 token 无效或缺失,返回 `404 task_not_found` 错误。 + +--- + +## 2. 接口列表与调用流程 + +下表列出全部接口;序号即推荐的标准调用顺序,调用流程一目了然。 + +| # | 方法 | 路径 | 作用 | 详情 | +|---|------|------|------|------| +| 1 | `GET` | `/openapi/v1/challenges` | 获取题目列表及作答进度 | [§5.1](#51-列出题目) | +| 2 | `POST` | `/openapi/v1/challenges/start` | 启动目标题目容器(获取题目入口地址) | [§5.2](#52-启动题目容器) | +| 3 | `GET` | `/openapi/v1/challenges/hint` | [可选] 获取提示(会扣分) | [§5.3](#53-获取提示) | +| 4 | `POST` | `/openapi/v1/challenges/submit` | 提交 flag | [§5.4](#54-提交-flag) | +| 5 | `POST` | `/openapi/v1/challenges/close` | 关闭题目容器,释放资源 | [§5.5](#55-关闭题目容器) | + +调用流程: + +``` +1. GET /openapi/v1/challenges — 获取题目列表 +2. POST /openapi/v1/challenges/start — 启动目标题目容器(获取题目入口地址) +3. (连接 VPN 后访问题目入口地址,进行渗透/解题) +4. GET /openapi/v1/challenges/hint — [可选] 获取提示(会扣分) +5. POST /openapi/v1/challenges/submit — 提交 flag +6. POST /openapi/v1/challenges/close — 关闭题目容器,释放资源 +``` + +- 步骤 2-6 可对每道题目重复执行。 +- 同一时间,最多只能启动 3 道题目。 +- 一道题可能有多个 flag(`flag_count`),需多次 submit。 +- 查看 hint 后提交会按 `hint_cost_radio` 比例扣减该题得分。 +- 已通关(所有 flag 全部正确提交)的题目不能再查看提示。 + +--- + +## 3. 注意事项 + +1. **VPN 连接**:启动题目后返回的 `container_addr` 需要通过 SSLVPN 直连,确保 VPN 已连接再访问靶场。 +2. **超时机制**:跑分任务有总时限,超时后所有接口将返回 `invalid_state` 错误。 +3. **提示扣分**:调用 hint 接口后,该题后续 flag 得分按比例扣减(具体比例由题目配置决定)。 +4. **幂等性**:同一 flag 重复提交不会重复加分,第二次会返回 `duplicate` 错误。 +5. **资源释放**:完成答题后务必调用 close 接口释放容器资源。 + +--- + +## 4. 异常处理 + +本节只介绍**统一的响应结构与通用错误码**。各接口的专属业务异常请见对应的接口详情小节(§5)。 + +### 4.1 统一错误响应格式 + +所有业务异常均返回以下 JSON 结构: + +```json +{ + "code": "error_code", + "message": "人类可读的错误描述", + "detail": {} +} +``` + +| 字段 | 类型 | 说明 | +|------|------|------| +| `code` | string | 机器可读的错误码(固定值,见下表) | +| `message` | string | 错误描述信息 | +| `detail` | object | 附加详情(通常为空对象) | + +### 4.2 通用错误码一览 + +下表为全部可能出现的错误码及其触发场景。每个接口实际可能返回其中的哪些,见各接口详情中的「业务异常」。 + +| HTTP 状态码 | code | 触发场景 | +|-------------|------|----------| +| **404** | `task_not_found` | `BENCHMARK_TOKEN` 无效、缺失或对应任务不存在 | +| **404** | `challenge_not_found` | `unique_code` 不存在于当前任务的用例集中 | +| **409** | `invalid_state` | 任务已结束(超时过期或手动停止),不允许继续操作 | +| **409** | `duplicate` | 同一个 flag(相同 flag_index)被重复正确提交,幂等保护 | +| **503** | `resource_unavailable` | 靶场资源不可用(实例未就绪或已耗尽) | +| **500** | `internal_error` | 未预期的系统内部错误 | + +### 4.3 框架级验证错误 + +当请求参数不满足基本约束时(如 `flag` 字段长度不在 1~4096 范围),FastAPI 返回 `422 Unprocessable Entity`: + +```json +{ + "detail": [ + { + "loc": ["body", "flag"], + "msg": "String should have at least 1 character", + "type": "string_too_short" + } + ] +} +``` + +> 此格式由框架生成,与业务异常格式不同,请注意区分。 + +> 说明:`task_not_found`(token 鉴权失败)与 `invalid_state`(任务已结束)属于所有接口的共性异常,下文各接口的「业务异常」仅额外列出该接口特有的异常,不再重复这两项。 + +--- + +# 二、接口详情 + +## 5. 接口详情 + +### 5.1 列出题目 + +获取当前任务关联的所有题目及作答进度。 + +``` +GET /openapi/v1/challenges +``` + +**请求参数**:无(仅需 header 认证) + +**成功响应** `200`: + +```json +[ + { + "unique_code": "web_sql_injection_01", + "description": "通过 SQL 注入获取管理员凭证并读取 flag", + "difficulty": "easy", + "level": 1, + "total_score": 100, + "flag_count": 2, + "correct_flag_count": 1, + "is_completed": false, + "container_status": "available", + "container_addr": ["10.0.1.5:8080"] + }, + { + "unique_code": "crypto_rsa_02", + "description": "RSA 参数不当导致私钥可恢复", + "difficulty": "hard", + "level": 3, + "total_score": 200, + "flag_count": 1, + "correct_flag_count": 0, + "is_completed": false, + "container_status": "stopped", + "container_addr": [] + } +] +``` + +| 字段 | 类型 | 说明 | +|------|------|------| +| `unique_code` | string | 题目唯一标识,后续接口均用此值 | +| `description` | string \| null | 题目描述(可能为空) | +| `difficulty` | string | 难度等级(如 easy / medium / hard) | +| `level` | int | 题目关卡 | +| `total_score` | int | 该题满分(所有 flag 分数之和) | +| `flag_count` | int | 该题 flag 总数 | +| `correct_flag_count` | int | 当前已正确提交的 flag 数 | +| `is_completed` | bool | 本题是否已全部通关 | +| `container_status` | string | 该题靶场容器的当前状态:`pending`(启动中)/ `available`(已就绪,可访问)/ `stop_pending`(停止中)/ `stopped`(已停止)。尚未启动过或已关闭的题目为 `stopped` | +| `container_addr` | array[string] | 靶场容器直连地址(IP:端口)数组,一个题目可能有多个地址;**仅当 `container_status == available` 时才有值**,其余状态一律为空数组 `[]`。选手需通过 VPN 直连访问 | + +**业务异常**:除共性异常(`task_not_found`、`invalid_state`)外,本接口无额外业务异常。 + +--- + +### 5.2 启动题目容器 + +为指定题目启动靶场容器,返回容器直连地址。 + +``` +POST /openapi/v1/challenges/start?unique_code={unique_code} +``` + +**请求参数**: + +| 参数 | 位置 | 类型 | 必填 | 说明 | +|------|------|------|------|------| +| `unique_code` | query | string | 是 | 题目唯一标识 | + +**成功响应** `200`: + +```json +{ + "unique_code": "web_sql_injection_01", + "container_addr": ["10.0.1.5:8080"] +} +``` + +| 字段 | 类型 | 说明 | +|------|------|------| +| `unique_code` | string | 题目唯一标识 | +| `container_addr` | array[string] | 靶场容器地址(IP:端口),一个题目可能有多个地址,选手通过 VPN 直连访问 | + +**业务异常**: + +| HTTP 状态码 | code | 触发场景 | +|-------------|------|----------| +| **404** | `challenge_not_found` | `unique_code` 不存在于当前任务的用例集中 | +| **409** | `invalid_state` | 当前活跃的题目实例数已达到上限,需先关闭已有题目再启动新题目 | +| **503** | `resource_unavailable` | 靶场资源不可用(实例未就绪或已耗尽) | + +--- + +### 5.3 获取提示 + +获取指定题目的提示信息。**注意**:查看提示后提交 flag 将按比例扣分。已通关的题目(所有 flag 已正确提交)不能再查看提示。 + +``` +GET /openapi/v1/challenges/hint?unique_code={unique_code} +``` + +**请求参数**: + +| 参数 | 位置 | 类型 | 必填 | 说明 | +|------|------|------|------|------| +| `unique_code` | query | string | 是 | 题目唯一标识 | + +**成功响应** `200`: + +```json +{ + "unique_code": "web_sql_injection_01", + "hint": "尝试在登录表单的用户名字段使用单引号测试注入点" +} +``` + +| 字段 | 类型 | 说明 | +|------|------|------| +| `unique_code` | string | 题目唯一标识 | +| `hint` | string \| null | 提示内容,无提示时为 null | + +**业务异常**: + +| HTTP 状态码 | code | 触发场景 | +|-------------|------|----------| +| **404** | `challenge_not_found` | `unique_code` 不存在于当前任务的用例集中 | +| **409** | `invalid_state` | 该题目已通关(所有 flag 均已正确提交),不允许再查看提示 | + +--- + +### 5.4 提交 Flag + +提交答案 flag 进行判定。 + +``` +POST /openapi/v1/challenges/submit +Content-Type: application/json +``` + +**请求体**: + +```json +{ + "unique_code": "web_sql_injection_01", + "flag": "flag{example_flag_value}" +} +``` + +| 字段 | 类型 | 必填 | 约束 | 说明 | +|------|------|------|------|------| +| `unique_code` | string | 是 | - | 题目唯一标识 | +| `flag` | string | 是 | 长度 1~4096 | 提交的 flag 值 | + +**成功响应** `200`: + +```json +{ + "correct": true, + "awarded": 50, + "cumulative_score": 80, + "correct_flag_count": 2, + "total_flag_count": 3, + "matched_flag_index": 1 +} +``` + +| 字段 | 类型 | 说明 | +|------|------|------| +| `correct` | bool | 本次提交是否正确 | +| `awarded` | int | 本次获得的分数(错误时为 0) | +| `cumulative_score` | int | 该题累计总得分 | +| `correct_flag_count` | int | 该题已正确提交的 flag 数 | +| `total_flag_count` | int | 该题 flag 总数 | +| `matched_flag_index` | int \| null | 匹配的 flag 索引(错误时为 null) | + +**业务异常**: + +| HTTP 状态码 | code | 触发场景 | +|-------------|------|----------| +| **404** | `challenge_not_found` | `unique_code` 不存在于当前任务的用例集中 | +| **409** | `duplicate` | 同一个 flag(相同 flag_index)被重复正确提交,幂等保护 | +| **422** | (框架级验证错误) | `flag` 字段长度不在 1~4096 范围,详见 §4.3 | + +--- + +### 5.5 关闭题目容器 + +关闭并释放指定题目的靶场容器资源。 + +``` +POST /openapi/v1/challenges/close?unique_code={unique_code} +``` + +**请求参数**: + +| 参数 | 位置 | 类型 | 必填 | 说明 | +|------|------|------|------|------| +| `unique_code` | query | string | 是 | 题目唯一标识 | + +**成功响应** `200`: + +```json +{ + "unique_code": "web_sql_injection_01", + "closed": true +} +``` + +| 字段 | 类型 | 说明 | +|------|------|------| +| `unique_code` | string | 题目唯一标识 | +| `closed` | bool | 是否成功关闭 | + +**业务异常**: + +| HTTP 状态码 | code | 触发场景 | +|-------------|------|----------| +| **404** | `challenge_not_found` | `unique_code` 不存在于当前任务的用例集中 | diff --git a/backend/platform/tsecbench/__init__.py b/backend/platform/tsecbench/__init__.py new file mode 100644 index 0000000..e0655cf --- /dev/null +++ b/backend/platform/tsecbench/__init__.py @@ -0,0 +1,17 @@ +from backend.platform.tsecbench.adapter import ( + TSECBenchAdapter, + TSECBenchAuthError, + TSECBenchClient, + TSECBenchError, + TSECBenchInstanceError, + TSECBenchPreflightError, +) + +__all__ = [ + "TSECBenchAdapter", + "TSECBenchAuthError", + "TSECBenchClient", + "TSECBenchError", + "TSECBenchInstanceError", + "TSECBenchPreflightError", +] diff --git a/backend/platform/tsecbench/adapter.py b/backend/platform/tsecbench/adapter.py new file mode 100644 index 0000000..798cbd3 --- /dev/null +++ b/backend/platform/tsecbench/adapter.py @@ -0,0 +1,590 @@ +from __future__ import annotations + +import hashlib +import socket +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +import requests + +from backend.platform._common import PLATFORM_CATEGORIES +from backend.platform.adapter import PlatformAdapter +from backend.platform.mapping import FieldMapping, PlatformChallenge + +_DEFAULT_TIMEOUT = 30 + +_CATEGORY_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("pwn", ( + "pwn", "pickle", "反序列化", "沙箱", "sandbox", "逃逸", "shellcode", + "内存破坏", "溢出", "overflow", "eval", "uaf", "heap", "rop", + "命令执行", "拿到服务器 shell", + )), + ("crypto", ( + "crypto", "rsa", "aes", "crc", "lcg", "csidh", "cipher", "mac", + "spn", "dsa", "otp", "口令", "密钥", "加密", "解密", "密码", "素数", + "背包", "置换群", "预言机", "bloom", "布隆", + )), + ("web", ( + "web", "http", "sql", "waf", "jwt", "spring", "flask", "博客", + "登录", "注册", "网站", "注入", "表单", "渲染", "ssti", "网关", + "cookie", "反向代理", "版本库", "git", + )), + ("reverse", ( + "逆向", "reverse", "加壳", "packed", "反调试", "二进制", "binary", + "可执行文件", "verilog", "固件", "文字冒险", "游戏", "crush", + )), + ("misc", ( + "取证", "应急响应", "forensics", "流量", "抓包", "邮件", "eml", + "日志", "隐写", "pcap", + )), +) + + +class TSECBenchError(RuntimeError): + """Classified platform failure; ``reason`` drives scheduler retries.""" + + def __init__( + self, message: str, *, reason: str = "transient", retry_after: int = 60 + ) -> None: + super().__init__(message) + self.reason = reason + self.retry_after = int(retry_after) + + +class TSECBenchAuthError(TSECBenchError): + """The benchmark token is missing or was rejected by the task.""" + + def __init__(self, message: str) -> None: + super().__init__(message, reason="auth") + + +class TSECBenchPreflightError(TSECBenchError): + """The task cannot be started as configured (no challenges, no base URL).""" + + def __init__(self, message: str) -> None: + super().__init__(message, reason="preflight") + + +class TSECBenchInstanceError(TSECBenchError): + """Container lifecycle failure with a scheduler-actionable reason.""" + + +def _classify_error(response: Any) -> TSECBenchError: + """Translate one HTTP failure into a classified error. + + The scheduler reacts differently per reason: a quota or rate limit is + retried later, an invalid token needs a new credential, a missing + challenge is permanent, and an ended task rejects the operation outright. + """ + + status = int(getattr(response, "status_code", 0) or 0) + code = "" + message = "" + try: + payload = response.json() + if isinstance(payload, dict): + code = str(payload.get("code") or "") + message = str(payload.get("message") or "") + except (ValueError, TypeError): + pass + detail = f": {message}" if message else "" + if status in (401, 403) or code == "task_not_found": + return TSECBenchAuthError( + f"the benchmark token was rejected (HTTP {status or 'unknown'}{detail})" + ) + headers = getattr(response, "headers", {}) or {} + try: + retry_after = max(1, int(headers.get("Retry-After", "60"))) + except (TypeError, ValueError): + retry_after = 60 + if status == 429: + return TSECBenchError( + f"the platform rate limited the request{detail}", + reason="rate_limited", retry_after=retry_after, + ) + if code == "challenge_not_found": + reason = "missing" + elif code == "duplicate": + reason = "duplicate" + elif code in ("resource_unavailable", "internal_error"): + reason = "transient" + elif code == "invalid_state": + if any(marker in message for marker in ("上限", "limit", "quota", "活跃")): + reason = "quota" + else: + reason = "ended" + elif status >= 500: + reason = "transient" + elif status == 404: + reason = "missing" + elif status == 409: + reason = "ended" + else: + reason = "rejected" + description = f"TSecBench request failed (HTTP {status or 'unknown'}" + description += f" {code})" if code else ")" + return TSECBenchError( + f"{description}{detail}", reason=reason, retry_after=retry_after + ) + + +class TSECBenchClient: + """Minimal OpenAPI client for one benchmark task. + + Authentication is a single ``BENCHMARK_TOKEN`` header; the token scopes + every request to exactly one task, so there is no login flow, game id or + session state to maintain. + """ + + PATH_CHALLENGES = "/openapi/v1/challenges" + PATH_START = "/openapi/v1/challenges/start" + PATH_HINT = "/openapi/v1/challenges/hint" + PATH_SUBMIT = "/openapi/v1/challenges/submit" + PATH_CLOSE = "/openapi/v1/challenges/close" + + def __init__( + self, + base_url: str = "", + token: str = "", + *, + timeout: float = _DEFAULT_TIMEOUT, + session: Any = None, + ) -> None: + self.base_url = (base_url or "").strip().rstrip("/") + self.token = (token or "").strip() + self.timeout = timeout + self.session = session or requests.Session() + + def _require_config(self) -> None: + if not self.base_url: + raise TSECBenchPreflightError("TSecBench base_url is not configured") + if not self.token: + raise TSECBenchAuthError("the benchmark token is not configured") + + def _request( + self, + method: str, + path: str, + *, + params: dict[str, Any] | None = None, + json_body: dict[str, Any] | None = None, + ) -> Any: + self._require_config() + try: + response = self.session.request( + method, + f"{self.base_url}{path}", + params=params, + json=json_body, + headers={ + "BENCHMARK_TOKEN": self.token, + "Accept": "application/json", + }, + timeout=self.timeout, + ) + except requests.RequestException as exc: + raise TSECBenchError( + f"TSecBench request failed: {exc}", reason="transient" + ) from exc + if int(getattr(response, "status_code", 0) or 0) >= 400: + raise _classify_error(response) + if not getattr(response, "content", b""): + return None + try: + return response.json() + except ValueError as exc: + raise TSECBenchError( + "TSecBench returned a non-JSON response", reason="transient" + ) from exc + + def list_challenges(self) -> list[dict[str, Any]]: + payload = self._request("GET", self.PATH_CHALLENGES) + if not isinstance(payload, list): + raise TSECBenchError( + "TSecBench challenge list was not a JSON array", reason="transient" + ) + return [item for item in payload if isinstance(item, dict)] + + def start_container(self, unique_code: str) -> dict[str, Any]: + payload = self._request( + "POST", self.PATH_START, params={"unique_code": unique_code} + ) + return payload if isinstance(payload, dict) else {} + + def close_container(self, unique_code: str) -> dict[str, Any]: + payload = self._request( + "POST", self.PATH_CLOSE, params={"unique_code": unique_code} + ) + return payload if isinstance(payload, dict) else {} + + def hint(self, unique_code: str) -> dict[str, Any]: + payload = self._request( + "GET", self.PATH_HINT, params={"unique_code": unique_code} + ) + return payload if isinstance(payload, dict) else {} + + def submit_flag(self, unique_code: str, flag: str) -> dict[str, Any]: + payload = self._request( + "POST", + self.PATH_SUBMIT, + json_body={"unique_code": unique_code, "flag": flag}, + ) + return payload if isinstance(payload, dict) else {} + + def close(self) -> None: + closer = getattr(self.session, "close", None) + if callable(closer): + closer() + + +def _positive_int(value: Any, default: int) -> int: + try: + number = int(value) + except (TypeError, ValueError): + return default + return number if number >= 0 else default + + +class TSECBenchAdapter(PlatformAdapter): + """Challenge/instance/submit adapter for a TSecBench benchmark task.""" + + DEFAULT_INSTANCE_LIMIT = 3 + + def __init__( + self, + client: TSECBenchClient, + mapping: FieldMapping, + *, + max_attachment_bytes: int | None = None, + competition_id: str = "", + team_id: str = "", + ) -> None: + if mapping.platform != "tsecbench": + raise ValueError("TSECBenchAdapter requires a tsecbench FieldMapping") + self.client = client + self.mapping = mapping + self.max_attachment_bytes = max_attachment_bytes + self.competition_id = str(competition_id or "") + self.team_id = str(team_id or "") + self._by_id: dict[str, PlatformChallenge] = {} + self._instance_views: dict[str, dict[str, Any]] = {} + self._task_identity = "" + + @property + def identity(self) -> str: + scope = f"{self.client.base_url}/benchmark" + if self.competition_id: + scope += f"/task/{self.competition_id}" + if self.team_id: + scope += f"/team/{self.team_id}" + if self._task_identity: + scope += f"/set/{self._task_identity}" + return scope + + @property + def supports_submit(self) -> bool: + return True + + @property + def supports_instances(self) -> bool: + return all( + callable(getattr(self.client, name, None)) + for name in ("start_container", "close_container") + ) + + def instance_limit(self) -> int | None: + """The platform's documented concurrent-container cap. + + The OpenAPI exposes no limit field, so the published cap (three + active containers per task) is the only truthful value. + """ + return self.DEFAULT_INSTANCE_LIMIT + + def _category_override(self, text: str) -> str: + for keyword, category in self.mapping.category_map.items(): + if str(keyword).casefold() in text: + value = str(category).strip().lower() + return value if value in PLATFORM_CATEGORIES else "misc" + return "" + + def _category(self, raw: dict[str, Any]) -> str: + override = self.mapping.category_map.get(str(raw.get("unique_code", ""))) or self.mapping.category_map.get("*") + if override in PLATFORM_CATEGORIES: + return override + if _positive_int(raw.get("flag_count"), 0) > 1: + return "pentest" + field = str(self.mapping.category_field or "").strip() + if field and raw.get(field) not in (None, ""): + value = str(raw.get(field)).strip().lower() + mapped = self._category_override(value) or value + if mapped in PLATFORM_CATEGORIES: + return mapped + haystack = f"{raw.get('unique_code', '')} {raw.get('description', '')}".casefold() + override = self._category_override(haystack) + if override: + return override + for category, keywords in _CATEGORY_KEYWORDS: + if any(keyword in haystack for keyword in keywords): + return category + return "misc" + + def _normalise(self, raw: dict[str, Any]) -> PlatformChallenge | None: + external = raw.get("unique_code") or raw.get(self.mapping.id_field) + if external in (None, ""): + return None + container_status = str(raw.get("container_status") or "").strip().lower() + addresses = [str(item) for item in raw.get("container_addr") or [] if item] + platform_data: dict[str, Any] = { + "difficulty": str(raw.get("difficulty") or ""), + "level": _positive_int(raw.get("level"), 1), + "score": raw.get("total_score"), + "total_score": raw.get("total_score"), + "flag_count": _positive_int(raw.get("flag_count"), 0), + "correct_flag_count": _positive_int(raw.get("correct_flag_count"), 0), + "container_status": container_status, + } + if addresses: + platform_data["container_addr"] = addresses + return PlatformChallenge( + external_id=str(external), + title=str(external), + category=self._category(raw), + description=str(raw.get("description") or ""), + attachment_urls=[], + remote=bool(container_status) or bool(addresses), + solved=bool(raw.get("is_completed")), + hints=[], + platform_data=platform_data, + ) + + def fetch_challenges(self) -> list[PlatformChallenge]: + raw_challenges = self.client.list_challenges() + if self.mapping.max_challenges: + raw_challenges = raw_challenges[: self.mapping.max_challenges] + result: list[PlatformChallenge] = [] + seen: set[str] = set() + refreshed: dict[str, PlatformChallenge] = {} + for raw in raw_challenges: + challenge = self._normalise(raw) + if challenge is None: + continue + if challenge.external_id in seen: + raise TSECBenchError( + f"duplicate TSecBench challenge id: {challenge.external_id}" + ) + seen.add(challenge.external_id) + refreshed[challenge.external_id] = challenge + result.append(challenge) + self._by_id = refreshed + if seen: + digest = hashlib.sha256(",".join(sorted(seen)).encode()).hexdigest() + self._task_identity = digest[:12] + return result + + def challenges(self) -> list[PlatformChallenge]: + self.fetch_challenges() + return [item.model_copy(deep=True) for item in self._by_id.values()] + + def preflight(self) -> list[PlatformChallenge]: + raw = self.client.list_challenges() + if not raw: + raise TSECBenchPreflightError( + "the benchmark task exposes no challenges; verify the benchmark token" + ) + return self.fetch_challenges() + + def network_diagnostics(self) -> dict: + """Check existing targets without starting an instance or buying a hint.""" + checks = [] + addresses = [str(addr) for item in self._by_id.values() + for addr in item.platform_data.get("container_addr", [])][:6] + for address in addresses: + try: + parsed = urlsplit(address if "://" in address else "//" + address) + if not parsed.hostname or not parsed.port: + raise ValueError("target address must include a host and port") + with socket.create_connection((parsed.hostname, parsed.port), timeout=0.5): + pass + checks.append({"address": address, "reachable": True}) + except (OSError, ValueError) as exc: + checks.append({"address": address, "reachable": False, "reason": str(exc)[:300]}) + return {"api_reachable": True, "vpn_required": True, "checked_from": "application", + "targets": checks, "sandbox_connectivity": "must be checked from the task container", + "guidance": "Connect the benchmark VPN for both application and task containers. Targets can be checked once an instance address is available."} + + def _challenge(self, external_id: str) -> PlatformChallenge: + if external_id not in self._by_id: + self.fetch_challenges() + challenge = self._by_id.get(str(external_id)) + if challenge is None: + raise TSECBenchError( + f"unknown TSecBench challenge: {external_id}", reason="missing" + ) + return challenge + + @staticmethod + def _instance_view(payload: Any) -> dict[str, Any]: + addresses = [ + str(item) for item in (payload or {}).get("container_addr") or [] if item + ] + view: dict[str, Any] = {"state": "ready" if addresses else "pending"} + if addresses: + view["entry"] = addresses[0] + view["url"] = addresses[0] + view["addresses"] = addresses + return view + + def instances(self) -> list[dict[str, Any]]: + """Report the containers this task currently holds. + + The list endpoint carries the live ``container_status`` for every + challenge, which is the only place TSecBench exposes an existing + instance; the reconciliation reader uses it to retire orphans. + """ + if not self.supports_instances: + return [] + found: list[dict[str, Any]] = [] + for challenge in self.challenges(): + data = challenge.platform_data or {} + addresses = [str(item) for item in data.get("container_addr") or [] if item] + if str(data.get("container_status") or "") == "available" and addresses: + found.append( + { + "external_id": challenge.external_id, + **self._instance_view({"container_addr": addresses}), + } + ) + return found + + def start_instance(self, external_id: str) -> dict[str, Any]: + self._challenge(external_id) + if not self.supports_instances: + raise ValueError("TSecBench client does not expose the container lifecycle") + try: + payload = self.client.start_container(external_id) + except TSECBenchError as exc: + raise TSECBenchInstanceError( + str(exc), reason=exc.reason, retry_after=exc.retry_after + ) from exc + view = self._instance_view(payload) + if view["state"] != "ready": + raise TSECBenchInstanceError( + f"TSecBench returned no container address for challenge {external_id}", + reason="transient", retry_after=30, + ) + self._instance_views[str(external_id)] = view + return dict(view) + + def stop_instance(self, external_id: str) -> None: + if not self.supports_instances: + return + try: + self.client.close_container(external_id) + except TSECBenchError as exc: + if exc.reason == "missing": + return + raise TSECBenchInstanceError( + str(exc), reason=exc.reason, retry_after=exc.retry_after + ) from exc + self._instance_views.pop(str(external_id), None) + + def renew_instance(self, external_id: str) -> dict[str, Any]: + """Confirm a running container; TSecBench has no lease to extend. + + Containers live until closed or the task ends, so renewal is a + truthful re-read of the challenge list instead: a container that + silently stopped is reported expired so the scheduler repairs it. + """ + if not self.supports_instances: + raise ValueError("TSecBench client does not expose the container lifecycle") + container_status = "" + addresses: list[str] = [] + for raw in self.client.list_challenges(): + if str(raw.get("unique_code") or "") == str(external_id): + container_status = str(raw.get("container_status") or "").strip().lower() + addresses = [ + str(item) for item in raw.get("container_addr") or [] if item + ] + break + if container_status == "available" and addresses: + view = self._instance_view({"container_addr": addresses}) + self._instance_views[str(external_id)] = view + return dict(view) + raise TSECBenchInstanceError( + f"TSecBench container for challenge {external_id} is " + f"{container_status or 'gone'}", + reason="expired", retry_after=30, + ) + + def rebuild_instance(self, external_id: str) -> dict[str, Any]: + """Replace a container that could not be confirmed as running.""" + try: + self.client.close_container(external_id) + except TSECBenchError: + pass + return self.start_instance(external_id) + + def submit(self, external_id: str, flag: str) -> dict[str, Any]: + try: + payload = self.client.submit_flag(external_id, flag) + except TSECBenchError as exc: + if exc.reason == "auth": + return {"verdict": "auth_required"} + if exc.reason == "duplicate": + # Duplicate is per flag index, never a proof that the whole task is solved. + progress = next((item for item in self.fetch_challenges() if item.external_id == str(external_id)), None) + metadata = progress.platform_data if progress else {} + return {"verdict": "correct", "reason": "duplicate", "duplicate": True, + "challenge_complete": bool(progress and progress.solved), + "correct_flag_count": metadata.get("correct_flag_count"), + "total_flag_count": metadata.get("flag_count")} + if exc.reason == "rate_limited": + return { + "verdict": "rate_limited", + "retry_after": getattr(exc, "retry_after", 60), + } + if exc.reason == "transient": + return {"verdict": "unknown", "reason": str(exc)} + return {"verdict": "rejected", "reason": str(exc)} + return self._normalise_submission(payload) + + @staticmethod + def _normalise_submission(payload: Any) -> dict[str, Any]: + if not isinstance(payload, dict): + return {"verdict": "unknown"} + correct = payload.get("correct") + if correct is True: + return { + "verdict": "correct", + "challenge_complete": (_positive_int(payload.get("total_flag_count"), 0) > 0 + and _positive_int(payload.get("correct_flag_count"), 0) >= _positive_int(payload.get("total_flag_count"), 0)), + "awarded": payload.get("awarded"), + "cumulative_score": payload.get("cumulative_score"), + "correct_flag_count": payload.get("correct_flag_count"), + "total_flag_count": payload.get("total_flag_count"), + "matched_flag_index": payload.get("matched_flag_index"), + } + if correct is False: + return {"verdict": "wrong"} + return {"verdict": "unknown"} + + def hint(self, external_id: str) -> dict[str, Any]: + """Read the optional hint. Reading it discounts later flag scores.""" + if not self.mapping.allow_hints: + return {"error": "Hints reduce the score; enable allow_hints explicitly to use them.", "reason": "hint_disabled"} + try: + return self.client.hint(external_id) + except TSECBenchError as exc: + return {"error": str(exc), "reason": getattr(exc, "reason", "transient")} + + def download_attachments( + self, challenge: PlatformChallenge, dest_dir: str | Path + ) -> list[Path]: + """TSecBench exposes no attachment endpoint. + + Challenge material (scripts, captures, binaries) lives inside each + challenge container and is served from the live target after the + container starts, so the import stage has nothing to download. + """ + return [] diff --git a/backend/platform/tsecbench/workflow.py b/backend/platform/tsecbench/workflow.py new file mode 100644 index 0000000..c0ccc94 --- /dev/null +++ b/backend/platform/tsecbench/workflow.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from urllib.parse import urlsplit + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from backend.core.config import CATEGORIES +from backend.ops.models import PlatformWorkflowSpec, _validate_http_url + +TSECBENCH_TEMPLATE_VERSION = 1 +TEMPLATE = {'id': 'tsecbench', 'name': 'TSecBench', 'version': 1, 'default': False, 'required': ['base_url'], 'optional': ['name', 'team_id', 'remote_concurrency', 'allow_private_networks', 'category_map', 'allow_hints'], 'secret_names': ['benchmark_token'], 'capabilities': ['challenges', 'submit', 'instances', 'hint']} + + +class TSECBenchWorkflowInput(BaseModel): + """Personalization for one benchmark task; identity comes from the token. + + The task has no ids of its own, so ``competition_id`` and ``team_id`` get + stable operational defaults that keep run identity keys deterministic; + the token in the secret store is what actually scopes the challenge set. + """ + + model_config = ConfigDict(extra="forbid") + + base_url: str = Field(max_length=2048) + name: str = Field(default="", max_length=120) + team_id: str = Field(default="", max_length=256) + remote_concurrency: int = Field(default=0, ge=0, le=3) + allow_hints: bool = False + category_map: dict[str, str] = Field(default_factory=dict) + allow_private_networks: bool = False + secrets: dict[str, str] = Field(default_factory=dict, repr=False, exclude=True) + + @field_validator("category_map") + @classmethod + def validate_category_map(cls, value): + if any(not key or len(key) > 256 or category not in CATEGORIES for key, category in value.items()): + raise ValueError("category_map must map challenge IDs, category names or * to a supported category") + return value + + @field_validator("base_url") + @classmethod + def validate_base_url(cls, value: str) -> str: + value = _validate_http_url(value).rstrip("/") + parsed = urlsplit(value) + if parsed.query or parsed.path not in ("", "/"): + raise ValueError("base_url must be the TSecBench platform origin, for example https://tsecbench.example.com") + return value + + @field_validator("secrets") + @classmethod + def validate_secrets(cls, values: dict[str, str]) -> dict[str, str]: + allowed = {"benchmark_token", "platform_token"} + if set(values) - allowed: + raise ValueError("TSecBench credentials must use benchmark_token") + if any(not value.strip() or len(value) > 16384 for value in values.values()): + raise ValueError("credentials must contain 1 to 16384 characters") + return values + + def workflow_spec(self) -> PlatformWorkflowSpec: + return PlatformWorkflowSpec.model_validate({ + "name": self.name.strip() or "TSecBench Benchmark", + "competition_id": "tsecbench", "team_id": self.team_id.strip() or "benchmark", + "template_id": "tsecbench", "template_version": TSECBENCH_TEMPLATE_VERSION, + "remote_concurrency": self.remote_concurrency, + "allow_private_networks": self.allow_private_networks, + "challenges": {"platform": "tsecbench", "list_url": self.base_url, "category_map": self.category_map, "allow_hints": self.allow_hints}, + }) diff --git a/backend/platform/verdict.py b/backend/platform/verdict.py index 8e73327..01bea85 100644 --- a/backend/platform/verdict.py +++ b/backend/platform/verdict.py @@ -1,4 +1,3 @@ -"""Platform verdicts are explicit application evidence, never HTTP success.""" from __future__ import annotations from dataclasses import dataclass diff --git a/backend/sandbox/command_policy.py b/backend/sandbox/command_policy.py index 94c924e..57315b9 100644 --- a/backend/sandbox/command_policy.py +++ b/backend/sandbox/command_policy.py @@ -1,6 +1,3 @@ -"""Reject member commands that reach outside the task container. -""" - from __future__ import annotations import re diff --git a/backend/sandbox/container_pool.py b/backend/sandbox/container_pool.py index 3e40b02..ed28b4d 100644 --- a/backend/sandbox/container_pool.py +++ b/backend/sandbox/container_pool.py @@ -1,8 +1,8 @@ from __future__ import annotations -from contextlib import suppress import re import threading +from contextlib import suppress from pathlib import Path from backend.sandbox.resource_limiter import TaskSlotLimiter diff --git a/backend/sandbox/docker_manager.py b/backend/sandbox/docker_manager.py index a82e2a3..26e62ce 100644 --- a/backend/sandbox/docker_manager.py +++ b/backend/sandbox/docker_manager.py @@ -1,6 +1,5 @@ from __future__ import annotations - import importlib import sys from pathlib import Path diff --git a/backend/sandbox/network_manager.py b/backend/sandbox/network_manager.py index b45fba9..c092bcc 100644 --- a/backend/sandbox/network_manager.py +++ b/backend/sandbox/network_manager.py @@ -1,6 +1,6 @@ from __future__ import annotations - +import logging import threading from dataclasses import dataclass, field from pathlib import Path @@ -12,6 +12,7 @@ classify_docker_startup_error, ) +_recovery_logger = logging.getLogger(__name__) @dataclass class ChallengeEnv: @@ -106,6 +107,7 @@ def _docker_up(self, env: ChallengeEnv) -> bool: env.error = str(env.startup_error) return False except Exception as exc: + _recovery_logger.debug('Error handled in _docker_up', exc_info=True) env.startup_error = classify_docker_startup_error( exc, "start challenge environment", @@ -137,4 +139,4 @@ def _docker_down(self, env: ChallengeEnv) -> None: else: subprocess.run(["docker", "rm", "-f", env.network_name], check=False, capture_output=True) except Exception: - pass + _recovery_logger.debug('Error handled in _docker_down', exc_info=True) diff --git a/backend/sandbox/output.py b/backend/sandbox/output.py new file mode 100644 index 0000000..744d670 --- /dev/null +++ b/backend/sandbox/output.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +from pathlib import Path + +DEFAULT_OUTPUT_BYTES = 16_384 + + +def read_page_file(path: Path, *, offset: int = 0, limit: int = 8192) -> dict: + if offset < 0 or not 1 <= limit <= 8192: + raise ValueError("invalid artifact page range") + with path.open("rb") as handle: + handle.seek(offset) + raw = handle.read(limit) + while True: + try: + data, encoding = raw.decode("utf-8"), "utf8" + except UnicodeDecodeError: + data, encoding = base64.b64encode(raw).decode("ascii"), "base64" + result = {"path": str(path), "offset": offset, "next_offset": offset + len(raw), + "total_bytes": path.stat().st_size, "encoding": encoding, "data": data} + if len(json.dumps(result, ensure_ascii=False).encode()) <= DEFAULT_OUTPUT_BYTES: + return result + raw = raw[:max(1, len(raw) // 2)] + + +def preview_bytes(value: bytes, limit: int = DEFAULT_OUTPUT_BYTES) -> str: + limit = max(512, limit) + if len(value) <= limit: + return value.decode("utf-8", "replace") + marker = f"\n[output truncated; {len(value)} bytes total; read the output artifact in pages]\n".encode() + side = max(1, (limit - len(marker)) // 2) + return (value[:side].decode("utf-8", "ignore") + marker.decode() + + value[-side:].decode("utf-8", "ignore")) + + +def preview_text(value: str, limit: int = DEFAULT_OUTPUT_BYTES) -> str: + return preview_bytes(value.encode("utf-8"), limit) + + +def read_output_file(path: Path, limit: int, *, visible_path: str | None = None) -> tuple[str, dict | None]: + size = path.stat().st_size + with path.open("rb") as handle: + if size <= limit: + return handle.read().decode("utf-8", "replace"), None + head = handle.read(max(1, (limit - 128) // 2)) + handle.seek(max(0, size - max(1, (limit - 128) // 2))) + tail = handle.read() + handle.seek(0) + digest = hashlib.sha256() + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + marker = f"\n[output truncated; {size} bytes total; read the artifact in pages]\n" + return head.decode("utf-8", "ignore") + marker + tail.decode("utf-8", "ignore"), { + "path": visible_path or str(path), "bytes": size, "sha256": digest.hexdigest(), + } + + +OUTPUT_FINALIZER = r''' +import hashlib,json,os,sys +limit=max(512,int(sys.argv[4])) +result={"exit_code":int(sys.argv[3]),"stdout":"","stderr":"","artifacts":{}} +for stream,path in zip(("stdout","stderr"),sys.argv[1:3]): + size=os.path.getsize(path) + with open(path,"rb") as f: + if size<=limit: + result[stream]=f.read().decode("utf-8","replace") + else: + side=max(1,(limit-128)//2) + head=f.read(side);f.seek(max(0,size-side));tail=f.read() + result[stream]=head.decode("utf-8","ignore")+"\n[output truncated; read the output artifact in pages]\n"+tail.decode("utf-8","ignore") + digest=hashlib.sha256();f.seek(0) + while True: + chunk=f.read(1048576) + if not chunk:break + digest.update(chunk) + result["artifacts"][stream]={"path":path,"bytes":size,"sha256":digest.hexdigest()} + if stream not in result["artifacts"]:os.unlink(path) +print(json.dumps(result,ensure_ascii=False)) +''' + +OUTPUT_PAGE_READER = r''' +import base64,json,pathlib,sys +path=pathlib.Path(sys.argv[1]).resolve() +if not path.is_relative_to(pathlib.Path("/workspace")):raise ValueError("artifact path must stay in the workspace") +offset=max(0,int(sys.argv[2]));limit=max(1,min(8192,int(sys.argv[3]))) +with path.open("rb") as f: + f.seek(offset);raw=f.read(limit) +while True: + try:data,encoding=raw.decode("utf-8"),"utf8" + except UnicodeDecodeError:data,encoding=base64.b64encode(raw).decode("ascii"),"base64" + result={"path":str(path),"offset":offset,"next_offset":offset+len(raw),"total_bytes":path.stat().st_size,"encoding":encoding,"data":data} + if len(json.dumps(result,ensure_ascii=False).encode())<=16000:break + raw=raw[:max(1,len(raw)//2)] +print(json.dumps(result,ensure_ascii=False)) +''' diff --git a/backend/sandbox/resource_limiter.py b/backend/sandbox/resource_limiter.py index 9611727..00892ea 100644 --- a/backend/sandbox/resource_limiter.py +++ b/backend/sandbox/resource_limiter.py @@ -6,40 +6,55 @@ @dataclass class TaskSlotLimiter: - """Concurrency gate for CTF tasks. + """Track tasks and gate work admitted by the standalone scheduler. - Each running task (project) occupies one slot. Memory is no longer - reserved per agent - a task owns a single Docker container shared by all of - its Members, and that container is not memory-capped. The only global limit - is how many tasks may run at once. + Competition work has already passed its own local/remote capacity and + global Member-seat checks. Track it for cleanup without charging a second + standalone slot. """ max_concurrent_tasks: int = 5 total_cpu: int = 4 _active: set[str] = field(default_factory=set) + _coordinated: set[str] = field(default_factory=set) _lock: threading.Lock = field(default_factory=threading.Lock) def can_admit(self, project_id: str) -> bool: with self._lock: if project_id in self._active: return True - return len(self._active) < self.max_concurrent_tasks + return len(self._active - self._coordinated) < self.max_concurrent_tasks - def acquire(self, project_id: str) -> bool: + def acquire(self, project_id: str, *, externally_scheduled: bool = False) -> bool: """Reserve a task slot. Idempotent for a project that already holds one.""" with self._lock: if project_id in self._active: return True - if len(self._active) >= self.max_concurrent_tasks: + if not externally_scheduled and len(self._active - self._coordinated) >= self.max_concurrent_tasks: return False self._active.add(project_id) + if externally_scheduled: + self._coordinated.add(project_id) return True def release(self, project_id: str) -> None: with self._lock: self._active.discard(project_id) + self._coordinated.discard(project_id) def active_tasks(self) -> list[str]: with self._lock: return sorted(self._active) + + def update_limits(self, *, max_concurrent_tasks: int | None = None, total_cpu: int | None = None) -> None: + """Apply configuration changes without interrupting admitted tasks.""" + with self._lock: + if max_concurrent_tasks is not None: + if int(max_concurrent_tasks) < 0: + raise ValueError("max_concurrent_tasks must be non-negative") + self.max_concurrent_tasks = int(max_concurrent_tasks) + if total_cpu is not None: + if int(total_cpu) < 0: + raise ValueError("total_cpu must be non-negative") + self.total_cpu = int(total_cpu) diff --git a/backend/sandbox/sandbox.py b/backend/sandbox/sandbox.py index da4b209..bf4f212 100644 --- a/backend/sandbox/sandbox.py +++ b/backend/sandbox/sandbox.py @@ -6,11 +6,18 @@ import signal import subprocess import threading +import time +import uuid from contextlib import suppress -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Protocol +from backend.sandbox.output import ( + DEFAULT_OUTPUT_BYTES, + read_output_file, + read_page_file, +) from backend.sandbox.webui_proxy import webui_proxy_manager @@ -91,6 +98,8 @@ class ExecResult: stdout: str stderr: str timed_out: bool = False + artifacts: dict = field(default_factory=dict) + duration_seconds: float | None = None @property def ok(self) -> bool: @@ -123,6 +132,10 @@ def __init__(self, name: str, workspace: str | Path, env: dict[str, str] | None self._started = False self._webui_keys: set[tuple[str, str]] = set() self._jobs: dict[str, subprocess.Popen] = {} + self.output_limit = DEFAULT_OUTPUT_BYTES + + def configure_output(self, limit: int) -> None: + self.output_limit = max(1024, int(limit)) def start(self) -> None: with self._lock: @@ -146,28 +159,36 @@ def exec(self, command: str, timeout: int = 60) -> ExecResult: popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP else: popen_kwargs["start_new_session"] = True - proc = subprocess.Popen( - args, - shell=use_shell, - cwd=str(self.workspace), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - encoding="utf-8", - errors="replace", - env=full_env, - **popen_kwargs, - ) - try: - out, err = proc.communicate(timeout=timeout) - return ExecResult(int(proc.returncode or 0), out or "", err or "") - except subprocess.TimeoutExpired: - _kill_process_tree(proc) + directory = self.workspace / ".ipc" / "output" + directory.mkdir(parents=True, exist_ok=True) + identifier = uuid.uuid4().hex + paths = [directory / f"{identifier}.{stream}" for stream in ("stdout", "stderr")] + started = time.monotonic() + timed_out = False + with paths[0].open("wb") as stdout, paths[1].open("wb") as stderr: + proc = subprocess.Popen( + args, shell=use_shell, cwd=str(self.workspace), stdout=stdout, stderr=stderr, + env=full_env, **popen_kwargs, + ) try: - out, err = proc.communicate(timeout=10) + proc.wait(timeout=timeout) except subprocess.TimeoutExpired: - out, err = "", "" - return ExecResult(124, out or "", err or "", timed_out=True) + timed_out = True + _kill_process_tree(proc) + with suppress(subprocess.TimeoutExpired): + proc.wait(timeout=10) + previews = [] + artifacts = {} + for stream, path in zip(("stdout", "stderr"), paths): + preview, artifact = read_output_file(path, self.output_limit // 2) + previews.append(preview) + if artifact: + artifacts[stream] = artifact + else: + path.unlink(missing_ok=True) + return ExecResult(124 if timed_out else int(proc.returncode or 0), *previews, + timed_out=timed_out, artifacts=artifacts, + duration_seconds=round(time.monotonic() - started, 6)) def write_file(self, rel_path: str, content: str) -> None: target = self._safe_path(rel_path) @@ -317,6 +338,9 @@ def read_file(self, rel_path: str) -> str | None: return None return target.read_text(encoding="utf-8", errors="replace") + def read_page(self, path: str, *, offset: int = 0, limit: int = 8192) -> dict: + return read_page_file(self._safe_path(path), offset=offset, limit=limit) + def stop(self) -> None: for project_id, member in list(self._webui_keys): webui_proxy_manager.close_member(project_id, member) diff --git a/backend/sandbox/task_sandbox.py b/backend/sandbox/task_sandbox.py index ab16361..f4c07a0 100644 --- a/backend/sandbox/task_sandbox.py +++ b/backend/sandbox/task_sandbox.py @@ -1,11 +1,15 @@ from __future__ import annotations -from contextlib import suppress import io +import json +import logging import os import shlex import tarfile import threading +import time +import uuid +from contextlib import suppress from pathlib import Path, PurePosixPath from backend.sandbox.docker_manager import _load_docker_sdk @@ -15,9 +19,16 @@ SandboxStartupError, classify_docker_startup_error, ) +from backend.sandbox.output import ( + DEFAULT_OUTPUT_BYTES, + OUTPUT_FINALIZER, + OUTPUT_PAGE_READER, +) from backend.sandbox.sandbox import ExecResult from backend.sandbox.webui_proxy import webui_proxy_manager +_recovery_logger = logging.getLogger(__name__) + WORKSPACE = "/workspace" ATTACHMENTS_DIR = f"{WORKSPACE}/attachments" SHARED_DIR = f"{WORKSPACE}/shared" @@ -70,8 +81,8 @@ def job_launch_script(directory: str, command: str, workdir: str) -> str: f"nohup bash -c {shlex.quote(supervisor)} >/dev/null 2>&1 < /dev/null &", "child=$!", f"printf '%s' \"$child\" > {quoted}/pid", - f"(sed 's/.*) //' /proc/$child/stat 2>/dev/null | awk '{{print $20}}' " - f"|| true) > {quoted}/starttime", + (f"(sed 's/.*) //' /proc/$child/stat 2>/dev/null | awk '{{print $20}}' " + f"|| true) > {quoted}/starttime"), f"cat {quoted}/pid", )) @@ -221,6 +232,7 @@ def _shared_network_name(self) -> str | None: try: current = self._docker().containers.get(hostname) except Exception: + _recovery_logger.debug('Error handled in _shared_network_name', exc_info=True) return None networks = current.attrs.get("NetworkSettings", {}).get("Networks", {}) if not networks: @@ -388,6 +400,40 @@ def _raw_exec(self, command: str, workdir: str, timeout: int = 60) -> ExecResult timed_out = res.exit_code in (124, 137) return ExecResult(res.exit_code if res.exit_code is not None else -1, stdout, stderr, timed_out) + def exec_bounded(self, command: str, workdir: str, timeout: int = 60) -> ExecResult: + if self._container is None: + self.start() + identifier = uuid.uuid4().hex + directory = f"{SHARED_DIR}/.ipc-output" + stdout, stderr = (f"{directory}/{identifier}.{stream}" for stream in ("stdout", "stderr")) + limit = max(1024, int(getattr(self, "output_limit", DEFAULT_OUTPUT_BYTES))) + script = "\n".join([ + f"mkdir -p {shlex.quote(directory)}", + f"timeout -k 5s {timeout}s bash -lc {shlex.quote(command)} >{shlex.quote(stdout)} 2>{shlex.quote(stderr)}", + "ipc_output_status=$?", + (f"python3 -c {shlex.quote(OUTPUT_FINALIZER)} {shlex.quote(stdout)} {shlex.quote(stderr)} " + f'"$ipc_output_status" {limit // 2}'), + ]) + started = time.monotonic() + envelope = self._raw_exec(script, workdir=workdir, timeout=timeout + 15) + if not envelope.ok: + return envelope + try: + payload = json.loads(envelope.stdout) + code = int(payload["exit_code"]) + return ExecResult(code, payload["stdout"], payload["stderr"], code in (124, 137), + payload["artifacts"], round(time.monotonic() - started, 6)) + except (ValueError, KeyError, TypeError): + raise RuntimeError("sandbox output finalizer returned an invalid envelope") from None + + + def read_page(self, path: str, *, workdir: str, offset: int = 0, limit: int = 8192) -> dict: + target = str(PurePosixPath(workdir) / path) if not PurePosixPath(path).is_absolute() else path + result = self._raw_exec(f"python3 -c {shlex.quote(OUTPUT_PAGE_READER)} {shlex.quote(target)} {int(offset)} {int(limit)}", + workdir=workdir, timeout=10) + if not result.ok: + raise ValueError("artifact is unavailable or outside this workspace") + return json.loads(result.stdout) def exec_detached(self, job_id: str, command: str, workdir: str) -> int: """Start ``command`` with no timeout and return its process-group PID. @@ -503,7 +549,13 @@ def start(self) -> None: self._task.start() def exec(self, command: str, timeout: int = 60) -> ExecResult: - return self._task.exec(command, workdir=self.workdir, timeout=timeout) + return self._task.exec_bounded(command, workdir=self.workdir, timeout=timeout) + + def configure_output(self, limit: int) -> None: + self._task.output_limit = max(1024, int(limit)) + + def read_page(self, path: str, *, offset: int = 0, limit: int = 8192) -> dict: + return self._task.read_page(path, workdir=self.workdir, offset=offset, limit=limit) def exec_detached(self, job_id: str, command: str) -> int: diff --git a/backend/server/app.py b/backend/server/app.py index 03fbfb6..6369c4a 100644 --- a/backend/server/app.py +++ b/backend/server/app.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os from contextlib import asynccontextmanager from pathlib import Path @@ -10,25 +11,28 @@ from backend.api import agent as agent_router from backend.api import auth as auth_router -from backend.api import config as config_router from backend.api import competition as competition_router +from backend.api import config as config_router from backend.api import flags as flags_router from backend.api import graph as graph_router from backend.api import logs as logs_router from backend.api import memory as memory_router from backend.api import ops_agent as ops_agent_router +from backend.api import pentest as pentest_router from backend.api import platform as platform_router from backend.api import project as project_router -from backend.api import solve as solve_router -from backend.api import skills as skills_router from backend.api import questions as questions_router +from backend.api import skills as skills_router +from backend.api import solve as solve_router from backend.api import wp as wp_router from backend.auth import AuthManager from backend.auth.middleware import AuthenticationMiddleware from backend.core.state import AppState +from backend.core.telemetry import RuntimeTelemetry from backend.ops.ipc_mcp import build_ipc_mcp from backend.sandbox.webui_proxy import webui_proxy_manager +_recovery_logger = logging.getLogger(__name__) def _resolve_frontend_dir(app_root: Path) -> Path: """Locate the top-level ``frontend/`` web UI directory. @@ -64,12 +68,16 @@ def create_app(root: str | Path | None = None) -> FastAPI: async def lifespan(app: FastAPI): state = AppState(root=app_root) app.state.ipc = state + telemetry = RuntimeTelemetry(state.artifact_root) + app.state.telemetry = telemetry + telemetry.start() try: from backend.core.orchestrator import Orchestrator state.orchestrator = Orchestrator(state) state.orchestrator.start() except Exception as exc: + _recovery_logger.debug('Error handled in lifespan', exc_info=True) app.state.orchestrator_error = str(exc) try: from backend.competition.service import CompetitionService @@ -77,11 +85,13 @@ async def lifespan(app: FastAPI): state.competition = CompetitionService(state) state.competition.start_worker() except Exception as exc: + _recovery_logger.debug('Error handled in lifespan', exc_info=True) app.state.competition_error = str(exc) try: async with ipc_mcp_app.router.lifespan_context(ipc_mcp_app): yield finally: + await telemetry.close() try: competition = getattr(state, "competition", None) if competition is not None: @@ -109,6 +119,7 @@ async def lifespan(app: FastAPI): app.include_router(auth_router.router) app.include_router(project_router.router) + app.include_router(pentest_router.router) app.include_router(solve_router.router) app.include_router(graph_router.router) app.include_router(memory_router.router) @@ -134,9 +145,14 @@ def index(): return FileResponse(frontend_dir / "index.html") @app.get("/health", include_in_schema=False) - def health(): + async def health(): return {"status": "ok", "setup_required": False} + @app.get("/health/details", include_in_schema=False) + async def health_details(): + telemetry = getattr(app.state, "telemetry", None) + return {"status": "ok", "runtime": telemetry.snapshot(getattr(app.state, "ipc", None)) if telemetry else {}} + if frontend_dir.exists(): app.mount("/static", StaticFiles(directory=str(frontend_dir)), name="static") diff --git a/backend/tools/catalog.py b/backend/tools/catalog.py index b20c87e..649907e 100644 --- a/backend/tools/catalog.py +++ b/backend/tools/catalog.py @@ -87,7 +87,7 @@ def load( path: str | Path = CATALOG_PATH, *, validate_documents: bool = True, - ) -> "ToolCatalog": + ) -> ToolCatalog: data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {} groups = {str(k): str(v) for k, v in (data.get("groups") or {}).items()} entries = [CatalogEntry(**item) for item in data.get("entries", [])] diff --git a/backend/tools/tool_inventory.py b/backend/tools/tool_inventory.py index 51f255d..53cfc09 100644 --- a/backend/tools/tool_inventory.py +++ b/backend/tools/tool_inventory.py @@ -3,7 +3,6 @@ from functools import lru_cache from pathlib import Path - _INVENTORY_PATH = Path(__file__).with_name("member_tools.txt") diff --git a/backend/tools/tool_mcp.py b/backend/tools/tool_mcp.py index c11e746..ec39e19 100644 --- a/backend/tools/tool_mcp.py +++ b/backend/tools/tool_mcp.py @@ -1,7 +1,8 @@ from __future__ import annotations import asyncio -from typing import Any, Callable, Collection +from collections.abc import Callable, Collection +from typing import Any from backend.mcp.mcp_server import MCPServer, create_mcp_server from backend.tools.tool_registry import ToolRegistry diff --git a/backend/tools/tool_registry.py b/backend/tools/tool_registry.py index af81632..393fe66 100644 --- a/backend/tools/tool_registry.py +++ b/backend/tools/tool_registry.py @@ -4,9 +4,9 @@ import threading import time from collections import OrderedDict +from collections.abc import Collection from dataclasses import dataclass from pathlib import Path -from typing import Collection import yaml @@ -90,7 +90,7 @@ def all_tools(self) -> list[Tool]: return list(self._tools) def by_category(self, category: str) -> list[Tool]: - return [t for t in self._tools if t.category == category] + return [t for t in self._tools if t.category == ("web" if category == "pentest" else category)] @staticmethod def _available(tool: Tool, available_mcps: Collection[str] | None) -> bool: diff --git a/coverage-baseline.json b/coverage-baseline.json new file mode 100644 index 0000000..bc01ff2 --- /dev/null +++ b/coverage-baseline.json @@ -0,0 +1,12 @@ +{ + "agent-core": { + "covered_lines": 3124, + "statements": 3841, + "line_percent": 81.33 + }, + "competition": { + "covered_lines": 2559, + "statements": 3162, + "line_percent": 80.93 + } +} diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml new file mode 100644 index 0000000..e9ac391 --- /dev/null +++ b/docker-compose.e2e.yml @@ -0,0 +1,68 @@ +name: ipc-e2e +services: + e2e-postgres: + image: postgres:16-alpine + ports: + - "127.0.0.1:15433:5432" + environment: + POSTGRES_DB: ipc_e2e + POSTGRES_USER: ipc + POSTGRES_PASSWORD: ipc + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ipc -d ipc_e2e"] + interval: 2s + timeout: 3s + retries: 20 + + e2e-platform: + image: python:3.12-slim + volumes: + - ./docker/e2e/fake_platform.py:/opt/fake_platform.py:ro + command: ["python", "/opt/fake_platform.py"] + + e2e-app: + image: ipc-ctfagent:latest + command: ["sh", "-c", "alembic upgrade head && exec uvicorn app:app --app-dir /app/docker/e2e --host 0.0.0.0 --port 8000"] + environment: + IPC_ROOT: /app + IPC_ARTIFACT_ROOT: /app/data/artifacts + IPC_DATABASE_URL: postgresql://ipc:ipc@e2e-postgres:5432/ipc_e2e + IPC_ALLOW_LOCAL_SANDBOX: "1" + IPC_LOG_ENABLED: "true" + volumes: + - e2e-data:/app/data + - ./docker/e2e/config:/app/backend/config:ro + - ./docker/e2e:/app/docker/e2e:ro + - ./backend:/app/backend:ro + - ./frontend:/app/frontend:ro + ports: + - "18000:8000" + depends_on: + e2e-postgres: + condition: service_healthy + e2e-platform: + condition: service_started + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/auth/status', timeout=3)"] + interval: 2s + timeout: 4s + retries: 30 + + e2e-runner: + build: + context: . + dockerfile: docker/e2e/runner.Dockerfile + image: ipc-e2e-browser:latest + environment: + IPC_E2E_BASE_URL: http://e2e-app:8000 + IPC_E2E_ARTIFACT_DIR: /artifacts + volumes: + - ./scripts/playwright_e2e.py:/opt/playwright_e2e.py:ro + - ./data/exports/e2e:/artifacts + command: ["python", "/opt/playwright_e2e.py"] + depends_on: + e2e-app: + condition: service_healthy + +volumes: + e2e-data: diff --git a/docker-compose.yml b/docker-compose.yml index 2003dbb..a0e3d2b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,8 @@ services: ipc-postgres: image: postgres:16-alpine + ports: + - "15432:5432" environment: - POSTGRES_DB=${IPC_POSTGRES_DB:-ipc} - POSTGRES_USER=${IPC_POSTGRES_USER:-ipc} @@ -30,6 +32,8 @@ services: build: context: . dockerfile: Dockerfile + args: + INSTALL_WSRX: ${IPC_INSTALL_WSRX:-1} image: ipc-ctfagent:latest container_name: ipc-app ports: diff --git a/docker/e2e/app.py b/docker/e2e/app.py new file mode 100644 index 0000000..512f2e7 --- /dev/null +++ b/docker/e2e/app.py @@ -0,0 +1,83 @@ +"""Test-only entrypoint: production orchestration with scripted mock Members.""" + +from contextlib import asynccontextmanager + +from backend.core.config import MEMBER_NAMES +from backend.core.orchestrator import Orchestrator +from backend.server.app import create_app + +SCRIPT = [ + {"action": "bash", "command": "sleep 2; printf 'e2e-sandbox-command-ok\\n'"}, + {"action": "tool", "server": "memory", "tool": "memory_catalog", "args": {}}, + {"action": "flag", "flag": "flag{e2e-deterministic}", + "description": "Reproduced sandbox command and memory MCP evidence in the acceptance fixture."}, +] + + +class FixtureScripts(dict): + """Choose the next deterministic turn from durable platform progress.""" + def __init__(self, state): + super().__init__({name: SCRIPT for name in MEMBER_NAMES}) + self.state = state + + def get(self, key, default=None): + if isinstance(key, tuple): + with self.state.db.connect() as conn: + row = conn.execute( + "SELECT p.platform,c.metadata FROM projects p JOIN competition_challenges c ON c.project_id=p.id WHERE p.id=%s", + (key[0],), + ).fetchone() + if row and row["platform"] == "tsecbench": + count = (row["metadata"].get("flag_progress") or {}).get("correct_flag_count", 0) + flag = "flag{e2e-tsec-two}" if count else "flag{e2e-tsec-one}" + return [*SCRIPT[:-1], {"action": "flag", "flag": flag, "description": "Verified the next deterministic benchmark objective."}] + return super().get(key, default) + +app = create_app() +original_lifespan = app.router.lifespan_context + + +@asynccontextmanager +async def lifespan(application): + async with original_lifespan(application): + state = application.state.ipc + state.orchestrator.shutdown() + state.orchestrator = Orchestrator( + state, scripts=FixtureScripts(state) + ) + state.orchestrator.start() + yield + + +app.router.lifespan_context = lifespan + + +@app.get("/e2e/runs/{run_id}/evidence") +def evidence(run_id: str): + """Read durable evidence. No production API or solver behavior is replaced.""" + state = app.state.ipc + store = state.competition.store + assignments = store.assignments(run_id) + events = [event for session in {a["session_id"] for a in assignments} + for event in store.events(session, limit=500)] + tool_names = [e["payload"].get("name") for e in events if e["kind"] == "tool_call"] + with state.db.connect() as connection: + jobs = connection.execute( + "SELECT status,attempts,artifact_path FROM competition_wp_jobs WHERE run_id=%s", + (run_id,), + ).fetchall() + backlog = connection.execute( + """SELECT count(*) AS count FROM competition_messages m + LEFT JOIN competition_message_cursors c + ON c.session_id=m.envelope->>'receiver_session_id' + WHERE m.run_id=%s AND m.sequence>COALESCE(c.sequence,0)""", + (run_id,), + ).fetchone()["count"] + return { + "assignments": assignments, + "tool_names": tool_names, + "tool_results": [e["payload"] for e in events if e["kind"] == "tool_result"], + "wp_jobs": jobs, + "collaboration_backlog": int(backlog), + "run_events": store.run_events(run_id, limit=1000), + } diff --git a/docker/e2e/config/config.yaml b/docker/e2e/config/config.yaml new file mode 100644 index 0000000..ee13f43 --- /dev/null +++ b/docker/e2e/config/config.yaml @@ -0,0 +1,17 @@ +log_enabled: true +diamond: + api_format: mock + api_surface: auto + reasoning_effort: high +member: + api_format: mock + api_surface: auto + reasoning_effort: high +runtime: + interval: 1 + sandbox_backend: local + max_member_steps: 10 + max_member_actions_per_task: 5 +limits: + max_concurrent_tasks: 2 + total_cpu: 2 diff --git a/docker/e2e/fake_platform.py b/docker/e2e/fake_platform.py new file mode 100644 index 0000000..dffdd8b --- /dev/null +++ b/docker/e2e/fake_platform.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import json +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, urlsplit + +TSEC_FLAGS: set[str] = set() +TSEC_ACTIVE = False +TSEC_CLOSES = 0 +TSEC_HINTS = 0 +TSEC_CODE = "e2e-pentest-multi" +TSEC_ANSWERS = ("flag{e2e-tsec-one}", "flag{e2e-tsec-two}") + + +CHALLENGE = { + "id": "e2e-web-1", + "name": "Deterministic E2E Web", + "category": "web", + "description": "The deterministic acceptance flag is flag{e2e-deterministic}.", + "files": [], + "score": 100, +} + + +class Handler(BaseHTTPRequestHandler): + def _send(self, status: int, payload: object) -> None: + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + global TSEC_HINTS + path = urlsplit(self.path).path + if path == "/e2e/tsecbench/state": + self._send(200, {"active": TSEC_ACTIVE, "correct_flags": len(TSEC_FLAGS), "close_calls": TSEC_CLOSES, "hint_calls": TSEC_HINTS}) + return + if path.startswith("/openapi/v1/challenges"): + if self.headers.get("BENCHMARK_TOKEN") != "fixture-benchmark-token": + self._send(404, {"code": "task_not_found", "message": "fixture task not found"}) + elif path.endswith("/hint"): + TSEC_HINTS += 1 + self._send(200, {"hint": "Score-reducing fixture hint"}) + else: + self._send(200, [{"unique_code": TSEC_CODE, "description": "Deterministic two-flag pentest", "difficulty": "easy", "level": 1, + "total_score": 100, "flag_count": 2, "correct_flag_count": len(TSEC_FLAGS), "is_completed": len(TSEC_FLAGS) == 2, + "container_status": "available" if TSEC_ACTIVE else "stopped", "container_addr": ["e2e-platform:8080"] if TSEC_ACTIVE else []}]) + return + if self.path == "/api/challenges": + self._send(200, {"data": [CHALLENGE]}) + else: + self._send(404, {"error": "not found"}) + + def do_POST(self) -> None: + global TSEC_ACTIVE, TSEC_CLOSES, TSEC_HINTS + parsed = urlsplit(self.path) + if parsed.path == "/e2e/tsecbench/reset": + TSEC_FLAGS.clear() + TSEC_ACTIVE = False + TSEC_CLOSES = TSEC_HINTS = 0 + self._send(200, {"ok": True}) + return + if parsed.path.startswith("/openapi/v1/challenges/"): + if self.headers.get("BENCHMARK_TOKEN") != "fixture-benchmark-token": + self._send(404, {"code": "task_not_found", "message": "fixture task not found"}) + return + if parsed.path.endswith("/submit"): + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", "0"))) or b"{}") + flag = body.get("flag") + if body.get("unique_code") != TSEC_CODE: + self._send(404, {"code": "challenge_not_found", "message": "wrong unique_code"}) + elif flag in TSEC_FLAGS: + self._send(409, {"code": "duplicate", "message": "flag already counted"}) + else: + correct = flag in TSEC_ANSWERS + if correct: + TSEC_FLAGS.add(flag) + self._send(200, {"correct": correct, "awarded": 50 if correct else 0, "cumulative_score": 50 * len(TSEC_FLAGS), + "correct_flag_count": len(TSEC_FLAGS), "total_flag_count": 2, "matched_flag_index": TSEC_ANSWERS.index(flag) if correct else None}) + return + if parse_qs(parsed.query).get("unique_code") != [TSEC_CODE]: + self._send(404, {"code": "challenge_not_found", "message": "missing unique_code query"}) + elif parsed.path.endswith("/start"): + TSEC_ACTIVE = True + self._send(200, {"unique_code": TSEC_CODE, "container_addr": ["e2e-platform:8080"]}) + elif parsed.path.endswith("/close"): + TSEC_ACTIVE = False + TSEC_CLOSES += 1 + self._send(200, {"unique_code": TSEC_CODE, "closed": True}) + else: + self._send(404, {"code": "challenge_not_found", "message": "unknown fixture path"}) + return + if self.path == "/api/submit": + length = int(self.headers.get("Content-Length", "0")) + payload = json.loads(self.rfile.read(length) or b"{}") + verdict = "correct" if payload.get("flag") == "flag{e2e-deterministic}" else "wrong" + self._send(200, {"verdict": verdict, "submission_id": "e2e-submission-1"}) + return + self._send(404, {"error": "not found"}) + + def log_message(self, *_args) -> None: + return + + +if __name__ == "__main__": + ThreadingHTTPServer(("0.0.0.0", 8080), Handler).serve_forever() diff --git a/docker/e2e/probe.py b/docker/e2e/probe.py new file mode 100644 index 0000000..95f9dbe --- /dev/null +++ b/docker/e2e/probe.py @@ -0,0 +1,68 @@ +import time + +import requests + +base = "http://127.0.0.1:8000" +workflow = { + "name": "Deterministic E2E Competition", + "competition_id": "e2e-competition", + "team_id": "e2e-team", + "allow_private_networks": True, + "challenges": { + "platform": "http_json", + "list_url": "http://e2e-platform:8080/api/challenges", + "list_path": "data", + "id_field": "id", + "title_field": "name", + "category_field": "category", + "description_field": "description", + "attachments_field": "files", + }, + "submit": { + "url": "http://e2e-platform:8080/api/submit", + "method": "POST", + "json_template": {"challenge_id": "{{external_id}}", "flag": "{{flag}}"}, + "success_path": "verdict", + "success_values": ["correct"], + "wrong_values": ["wrong"], + }, +} +created = requests.post(f"{base}/api/ops/workflows", json={"workflow": workflow}) +print("create", created.status_code, created.text) +workflow_id = created.json()["id"] +confirmed = requests.post( + f"{base}/api/ops/workflows/{workflow_id}/confirm", + json={"confirmation_phrase": f"CONFIRM WORKFLOW {workflow_id}"}, +) +print("confirm", confirmed.status_code, confirmed.text) +print("preflight", requests.post(f"{base}/api/workflows/{workflow_id}/preflight").text) +started = requests.post( + f"{base}/api/workflows/{workflow_id}/start", + json={"idempotency_key": f"probe-start-{int(time.time())}"}, +) +print("start", started.status_code, started.text[:2000]) +payload = started.json() +run_id = payload["run"]["id"] +assignment = None +for _ in range(30): + snapshot = requests.get(f"{base}/api/runs/{run_id}").json() + challenge = snapshot["challenges"][0] + if challenge["assignments"]: + assignment = challenge["assignments"][0] + break + time.sleep(0.5) +if assignment: + response = requests.post( + f"{base}/api/challenges/{challenge['id']}/candidates", + json={"run_id": run_id, "session_id": assignment["session_id"], + "flag": "flag{e2e-deterministic}", + "evidence": "deterministic evidence", + "instance_generation": challenge["instance_generation"]}, + ) + print("candidate", response.status_code, response.text) + for _ in range(30): + snapshot = requests.get(f"{base}/api/runs/{run_id}").json() + if snapshot["challenges"][0]["state"] == "solved": + print("solved", run_id) + break + time.sleep(0.5) diff --git a/docker/e2e/runner.Dockerfile b/docker/e2e/runner.Dockerfile new file mode 100644 index 0000000..f115a33 --- /dev/null +++ b/docker/e2e/runner.Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.12-slim +RUN sed -i 's|http://deb.debian.org/debian|https://mirrors.tuna.tsinghua.edu.cn/debian|g; s|http://deb.debian.org/debian-security|https://mirrors.tuna.tsinghua.edu.cn/debian-security|g' /etc/apt/sources.list.d/debian.sources \ + && apt-get update \ + && apt-get install -y --no-install-recommends chromium \ + && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir playwright==1.50.0 +ENV IPC_PLAYWRIGHT_EXECUTABLE=/usr/bin/chromium +CMD ["python", "/opt/playwright_e2e.py"] diff --git a/docker/tests/postgres.Dockerfile b/docker/tests/postgres.Dockerfile new file mode 100644 index 0000000..33259db --- /dev/null +++ b/docker/tests/postgres.Dockerfile @@ -0,0 +1,10 @@ +# Run pytest-postgresql against its own temporary server as a non-root user. +FROM ipc-ctfagent:latest +RUN apt-get update \ + && apt-get install -y --no-install-recommends postgresql \ + && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir pytest pytest-cov pytest-postgresql httpx +ENV PYTHONDONTWRITEBYTECODE=1 +WORKDIR /workspace +USER postgres +CMD ["python", "-m", "pytest", "-q", "-p", "no:cacheprovider", "--basetemp=/tmp/ipc-native-pytest"] diff --git a/docs/Example.md b/docs/Example.md deleted file mode 100644 index dbcffa5..0000000 --- a/docs/Example.md +++ /dev/null @@ -1,15 +0,0 @@ -首先我们挑选一道新生赛题目(难度在 Low ~ Medium 间),然后设定为: - -![](../image/1.png) - -然后让GPT5.4跑一下,结果是:![](../image/2.png) - -结果GPT5.4在1m16s解出来了,给了给非预期 - -~~我们的 IPC 耗时 45m 调用了全部 Agent 然后给了个预期解没绷住.....~~ - -经过多次测试......再没有特定优化某一类题型和更改模型的前提下,取得了阶段性胜利,这次还调用库失败了,不然应该会快几步 - -![](../image/3.png) - -简单题由于短任务运行,只要花费时间不是过分长就是在预料之内,等这几天看看中难问题 \ No newline at end of file diff --git a/docs/ret2shell.md b/docs/ret2shell.md deleted file mode 100644 index 7b8b33d..0000000 --- a/docs/ret2shell.md +++ /dev/null @@ -1,92 +0,0 @@ -# ret2shell 平台适配 - -适配 [ret2shell](https://github.com/ret2shell/ret2shell)(Rust/axum 开源 CTF 平台)。 -**已对真实平台验证通过**:西电 `https://ctf.xidian.edu.cn`(MoeCTF 2026,game 37,125 题), -端到端冒烟(导入 → 解题 → flag → writeup)已跑通。 - -## 配置(环境变量) - -| 变量 | 默认 | 说明 | -|---|---|---| -| `IPC_R2S_BASE_URL` | `https://ctf.xidian.edu.cn` | 平台根地址 | -| `IPC_R2S_GAME_ID` | `37` | 比赛 id(数字) | -| `IPC_R2S_USERNAME` / `IPC_R2S_PASSWORD` | 空 | 参赛账号(登录换 JWT Bearer token) | -| `IPC_R2S_TOKEN` | 空 | 已有的 Bearer token(可替代账密) | - -Docker Compose 部署时把这些写进项目根目录的 `.env`(compose 自动读取并注入 -ipc-app 容器;注意 `.env` 不能带 UTF-8 BOM,PowerShell `Set-Content` 后需检查)。 -**认证细节**:登录需要验证码时客户端自动走 `GET /api/account/captcha/cli` 的 -PoW(难度 4:答案以 challenge 随机串开头、其 SHA-256 十六进制前 4 位为 0), -无需人工介入;token 在响应头 `Set-Token` 返回并自动滚动更新。 -**密码连续错 5 次账号冻结 30 分钟**。 - -## 使用流程 - -```bash -export PYTHONPATH=. -export IPC_R2S_USERNAME=... IPC_R2S_PASSWORD=... - -# 1) 只读冒烟:登录 + 比赛状态 + 题目列表 -python scripts/ret2shell_platform_check.py - -# 2) 拉取题目清单(含分值/分类/已解状态)到 platform_targets/ret2shell.yaml -python scripts/fetch_ret2shell_targets.py - -# 3) 批量导入题目为项目。附件下载与数据库写入已分离(两阶段), -# 但大括量仍建议分批(脚本每批 15 题、自动重试): -docker cp scripts/import_ret2shell_batch.py ipc-app:/tmp/ -docker exec ipc-app python /tmp/import_batch.py -# 或直接对运行中的后端调 POST /api/platform/import(mapping 见下)。 - -# 4) Member 在沙箱里通过 ret2shell MCP 管理动态靶机(pwn 题拿 host:port) - -# 5) 提交 flag:默认 dry-run,--no-dry-run 才真正提交 -python scripts/ret2shell_submit.py --challenge --flag 'flag{...}' -``` - -**运维代理(IPC 对话)路径**:ops agent 的平台工作流也支持 -`challenges.platform: "ret2shell"`(backend/ops/service.py 的 `_adapter` 分支), -在 IPC 对话里提交含 `"challenges": {"platform": "ret2shell", "game_id": 37}` -的 workflow 描述并确认后,agent 可自行拉题导入。凭据始终只从环境变量读取。 - -**中文分类映射**(MoeCTF 2026 实测):Web 安全与渗透测试→web、 -现代密码学→crypto、二进制漏洞审计 / Python 沙箱逃逸→pwn、 -软件逆向工程→reverse、大语言模型应用安全→ai、其余(取证杂项/从此开始/ -开发运维基础/策略博弈/美工设计)→misc。 - -## 关键契约(与 GZCTF 的差异) - -- **异步判题**:`POST .../submit` 返回 `solved: null` 的 Submission,客户端轮询 - `GET .../submit?id=` 直到 `solved != null`(默认 1s × 7 次)。 -- **提交限额**:每账号 5 分钟 10 次(HTTP 429)。客户端内置同额度限速器, - 提交前还会做已解出 preflight,避免浪费平台尝试次数。 -- **动态靶机**:`POST/PATCH/DELETE .../instance` 启动/续期/销毁, - `GET /api/game/{gid}/instance` 轮询 `exposed_ports` 拿实际连接地址。 -- 列表接口返回 `[items, total]` 二元组;错误响应是纯文本 body。 - -## Member 侧能力(ret2shell MCP) - -配置了 `IPC_R2S_USERNAME`(或 `IPC_R2S_TOKEN`)后,后端自动向所有 Member 注册 -`ret2shell` MCP(凭据留在后端,不进沙箱): - -- `instance_start(challenge_id)`:启动并等待靶机可达,返回 endpoints -- `instance_status` / `instance_renew` / `instance_stop` -- `challenge_status(challenge_id)`:已解状态与全站解出数 - -**flag 提交不暴露给 Member**——Member 只能通过 `flag` 动作上报候选 flag。 -对 platform 为 `ret2shell` 的项目,候选先落 `flag_submissions`(pending), -由 orchestrator 的判定 worker 经平台客户端提交并轮询异步判题:判对才 -`accept_verified_flag` 置 solved;判错记 `rejected`、写入 `rejected-flag` -经验记忆(Member 上下文会列出,防止重复上报),项目回到 running 继续解; -限流/网络/判题超时等瞬时失败重试 3 次后把候选停为 `error` 并恢复求解。 -手动补交仍可用 `scripts/ret2shell_submit.py`(默认 dry-run)。 - -## 已知限制与运维注意 - -- **RAM 黑板**:解题图谱/项目列表在内存里,`docker compose restart` 会清空 - 项目(附件文件保留在 `projects/`,重新导入即恢复,external_id 对应关系不变)。 - 比赛期间尽量避免重启容器。 -- **SQLite 锁**:memdb 不支持 WAL,长事务与 UI 轮询互斥。导入已改为 - 两阶段(先下载后写库),但请勿在导入进行时并行发起第二个导入。 -- **附件去重**:导入脚本超时重试可能产生重复项目(同一 external_id 多个 - project),可按 external_id 保留最早一个、DELETE 其余。 diff --git a/frontend/index.html b/frontend/index.html index 9c0278e..5015c62 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -4,7 +4,7 @@ IPC Agent - + @@ -105,7 +105,7 @@ + + @@ -129,7 +131,7 @@
@@ -137,7 +139,7 @@ Memory - @@ -214,8 +216,8 @@ class="mt-1.5 truncate font-mono text-[10px] text-sky-600" x-text="p.runtime_phase">
-
No projects yet.
-
No project matches this filter.
+
No projects yet.
+
No project matches this filter.
@@ -260,6 +262,26 @@ +
+
+ + + + + + + + + + +
+
+ + +
+ +

+
Select or create a project to view its flow graph. @@ -268,6 +290,7 @@
@@ -578,23 +674,26 @@

Member seats

Challenges

- +
- - - +
ChallengeCategoryStateMembersDeadlineWrong
@@ -642,13 +741,19 @@

Live events

- +
+ + + + +
- + @keydown.enter="onQuestionEnter($event, question.id)"> @@ -1158,7 +1263,7 @@

Skill

- No workflow selected: IPC will individually adapt a new platform from your description. + No workflow selected: use the built-in GZCTF workflow setup, then provide only your platform and competition details.