diff --git a/.gitignore b/.gitignore index 218282d..b80f7ff 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ data/ memory/ projects/ Wp/ +docs/ exports/ examples/ log.txt @@ -237,3 +238,4 @@ __marimo__/ # Streamlit .streamlit/secrets.toml +.ipc-agent/ diff --git a/Dockerfile b/Dockerfile index ea2da7e..229bac7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,19 @@ COPY scripts /app/scripts RUN pip install --no-cache-dir -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple -e ".[docker]" +# wsrx (WebSocket Reflector X CLI from XDSEC, static musl build) tunnels the +# ret2shell platform's ws://-proxied challenge ports to local TCP ports so +# Members can reach dynamic instances over the docker network. The ret2shell +# backend itself is only installed in this image because the tunnel +# subprocesses are owned by the backend process (Members just connect to +# ipc-app:). +ARG WSRX_VERSION=0.6.1 +RUN python -c "import urllib.request; urllib.request.urlretrieve('https://github.com/XDSEC/WebSocketReflectorX/releases/download/${WSRX_VERSION}/wsrx-cli-${WSRX_VERSION}-linux-musl-x86_64.tar.gz', '/tmp/wsrx.tar.gz')" \ + && tar -xzf /tmp/wsrx.tar.gz -C /tmp \ + && install -m 0755 /tmp/wsrx /usr/local/bin/wsrx \ + && rm -f /tmp/wsrx /tmp/wsrx.tar.gz \ + && wsrx --version + # /app/data is bind-mounted by docker-compose.yml. PostgreSQL owns runtime # facts; filesystem workspaces and generated files share one Artifact tree. RUN mkdir -p /app/data/artifacts/projects \ diff --git a/README.md b/README.md index 0a779a6..002a3be 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ Member 在题目容器中使用按分类注册的 CTF 工具,并可通过 MCP - `browser`:基于 Chromium 的浏览器自动化、下载与截图 Artifact。 - `reverse`:PyGhidra 与 radare2 逆向分析。 - `zap`:可选 OWASP ZAP 服务,必须同时开启运行配置和 Compose profile。 +- `ret2shell`:ret2shell 平台的动态实例控制(`instance_start` / `instance_status` / `instance_renew` / `instance_stop` / `challenge_status`),仅在配置 `IPC_R2S_USERNAME` 或 `IPC_R2S_TOKEN` 后注册;ws:// 隧道由镜像内置的 wsrx 转发到本地端口。 分类覆盖 `web`、`pwn`、`reverse`、`crypto`、`misc`、`ai`、`osint`。浏览器下载、截图等内容以 Artifact 保存,避免将大型输出直接放入模型上下文。 @@ -141,7 +142,7 @@ Docker Compose 将数据库放入命名卷,将需要人工访问或跨容器 | Docker Socket | 必需 | App 需要创建题目容器;IPC Runner 同样依赖 Socket | | Linux Docker 主机 | 推荐/已验证 | Compose 会挂载 Docker Socket 与 Compose 插件 | | LLM 端点 | 开始解题时需要 | Diamond 与已启用 Member 使用 OpenAI、Anthropic、Claude Code、DeepSeek、Pi 或 Mock 适配器 | -| 浏览器 | 可选 | 用于打开本地 Web UI | +| 浏览器 | 可选 | 用于打开本地浏览器 | > [!WARNING] > 沙箱降低风险,但不能替代隔离主机或虚拟机。请只将题目、附件、模型密钥和 Docker Socket 放入可信环境。 @@ -357,6 +358,7 @@ IPC_CTFAgent/ │ ├── memory/ # 经验记忆与工具目录 │ ├── ops/ # IPC 行动代理与平台工作流 │ ├── persistence/ # PostgreSQL schema 与 Alembic migrations +│ ├── platform/ # 平台适配层(HTTP JSON / GZCTF / ret2shell 客户端) │ ├── sandbox/ # Docker/本地任务沙箱 │ └── tools/ # 工具注册表、目录与文档 ├── frontend/ # 单页 Web UI @@ -424,11 +426,11 @@ docker run --rm \ xz w + + xz w +

- ---- - ## 🤝 Contribution 欢迎提交 bug 报告、稳定性测试、工具集成、文档和架构改进。 diff --git a/backend/api/platform.py b/backend/api/platform.py index 182280b..e6efa37 100644 --- a/backend/api/platform.py +++ b/backend/api/platform.py @@ -1,6 +1,9 @@ from __future__ import annotations +import re import shutil +import time +from pathlib import Path import requests from fastapi import APIRouter, Depends, HTTPException @@ -9,12 +12,18 @@ from backend.api.deps import get_state from backend.blackboard import graph_store from backend.core.state import AppState -from backend.platform.adapter import HttpJsonAdapter +from backend.platform.adapter import HttpJsonAdapter, PlatformAdapter from backend.platform.mapping import FieldMapping, PlatformChallenge +from backend.platform.ret2shell import Ret2ShellAdapter, Ret2ShellClient, Ret2ShellError router = APIRouter(prefix="/api/platform", tags=["platform"]) +def safe_name(external_id: str) -> str: + cleaned = re.sub(r"[^A-Za-z0-9._-]", "_", external_id).strip("._") or "challenge" + return cleaned[:80] + + class ChallengeRequest(BaseModel): model_config = ConfigDict(extra="forbid") mapping: FieldMapping @@ -24,12 +33,25 @@ class ImportRequest(ChallengeRequest): select: list[str] | None = None -def _fetch(mapping: FieldMapping) -> tuple[HttpJsonAdapter, list[PlatformChallenge]]: - adapter = HttpJsonAdapter(mapping) +def _build_adapter(mapping: FieldMapping) -> PlatformAdapter: + if mapping.platform == "ret2shell": + client = Ret2ShellClient(base_url=mapping.list_url, game_id=mapping.game_id) + return Ret2ShellAdapter( + client, + game_id=mapping.game_id or None, + category_map=mapping.category_map, + ) + return HttpJsonAdapter(mapping) + + +def _fetch(mapping: FieldMapping) -> tuple[PlatformAdapter, list[PlatformChallenge]]: + adapter = _build_adapter(mapping) try: return adapter, adapter.fetch_challenges() except requests.RequestException as exc: raise HTTPException(502, f"platform request failed: {exc}") from exc + except Ret2ShellError as exc: + raise HTTPException(502, f"ret2shell request failed: {exc}") from exc except (TypeError, ValueError) as exc: raise HTTPException(400, str(exc)) from exc @@ -52,29 +74,58 @@ def import_challenges(body: ImportRequest, state: AppState = Depends(get_state)) raise HTTPException(400, f"unknown external_id values: {missing}") selected = [by_id[external_id] for external_id in body.select] + # Phase 1 downloads every attachment while NO database connection is + # open. Holding the write transaction across network downloads starved + # the UI polling loop (and itself) with "database is locked" on large + # imports. Phase 2 then only performs fast local writes. + staging_root = state.root / "staging" / f"import_{int(time.time() * 1000)}" + staged: dict[str, list[Path]] = {} + try: + for challenge in selected: + staged[challenge.external_id] = adapter.download_attachments( + challenge, + staging_root / safe_name(challenge.external_id), + ) + except requests.RequestException as exc: + shutil.rmtree(staging_root, ignore_errors=True) + raise HTTPException(502, f"attachment download failed: {exc}") from exc + except Ret2ShellError as exc: + shutil.rmtree(staging_root, ignore_errors=True) + raise HTTPException(502, f"ret2shell attachment download failed: {exc}") from exc + except (OSError, TypeError, ValueError) as exc: + shutil.rmtree(staging_root, ignore_errors=True) + raise HTTPException(400, str(exc)) from exc + imported: list[dict[str, str]] = [] created_project_ids: list[str] = [] try: with state.db.connect() as conn: for challenge in selected: + # Mirror the ops workflow importer: the challenge statement + # (connection commands, flag format) rides in the origin fact + # so the Member context's challenge_description picks it up. + origin = body.mapping.list_url + if challenge.description: + origin = f"{origin}\n\n{challenge.description}" project_id = graph_store.create_project( conn, challenge.title, - body.mapping.list_url, + origin, "capture the flag", challenge.category, external_id=challenge.external_id, ) created_project_ids.append(project_id) - for path in adapter.download_attachments( - challenge, - state.attachments_dir(project_id), - ): + attachment_dir = state.attachments_dir(project_id) + attachment_dir.mkdir(parents=True, exist_ok=True) + for path in staged.get(challenge.external_id, []): + target = attachment_dir / path.name + shutil.move(str(path), str(target)) graph_store.create_attachment( conn, project_id, - path.name, - str(path), + target.name, + str(target), ) imported.append( { @@ -84,14 +135,12 @@ def import_challenges(body: ImportRequest, state: AppState = Depends(get_state)) "category": challenge.category, } ) - except requests.RequestException as exc: - for project_id in created_project_ids: - shutil.rmtree(state.projects_dir / project_id, ignore_errors=True) - raise HTTPException(502, f"attachment download failed: {exc}") from exc except (OSError, TypeError, ValueError) as exc: for project_id in created_project_ids: shutil.rmtree(state.projects_dir / project_id, ignore_errors=True) raise HTTPException(400, str(exc)) from exc + finally: + shutil.rmtree(staging_root, ignore_errors=True) for item in imported: state.logger.project( diff --git a/backend/blackboard/edge_store.py b/backend/blackboard/edge_store.py index bdeab27..2214182 100644 --- a/backend/blackboard/edge_store.py +++ b/backend/blackboard/edge_store.py @@ -330,41 +330,3 @@ def expire_workers(conn: Any, timeout: int, project_id: str | None = None) -> No query += " AND project_id = %s" params += (project_id,) conn.execute(query, params) - - -def claim_next_intent( - conn: Any, - project_id: str, - worker: str, - lease_owner: str, - *, - timeout: int = 30, -) -> tuple[str, str] | None: - """Atomically select and claim one open Intent across app instances.""" - - candidate = conn.execute( - """ - SELECT id - FROM intents - WHERE project_id = %s AND concluded_at IS NULL - AND ( - lease_owner IS NULL - OR (lease_expires_at IS NOT NULL AND lease_expires_at <= now()) - ) - ORDER BY created_at, id - FOR UPDATE SKIP LOCKED - LIMIT 1 - """, - (project_id,), - ).fetchone() - if candidate is None: - return None - token = claim_intent( - conn, - project_id, - candidate["id"], - worker, - lease_owner=lease_owner, - timeout=timeout, - ) - return (candidate["id"], token) if token else None diff --git a/backend/blackboard/node_store.py b/backend/blackboard/node_store.py index 6e4af4c..2877c99 100644 --- a/backend/blackboard/node_store.py +++ b/backend/blackboard/node_store.py @@ -1,11 +1,58 @@ from __future__ import annotations +import re from typing import Any from backend.blackboard.ids import next_fact_id, utcnow from backend.blackboard.models import Fact +# Agent reports often restate the same source finding with a different lead-in +# (for example, "Nday candidate" versus "confirmed candidate"). Keeping the +# full prose is useful for audit, but these stable tokens are enough to prevent +# a repeated conclusion from inflating the graph and retriggering Diamond. +_FINDING_RE = re.compile( + r"(?P[A-Za-z0-9_./-]+\.(?:c|cc|cpp|cxx|h|hpp|py|js|ts|java))" + r"(?::|\s+lines?\s+)(?P\d+(?:-\d+)?)" + r".*?(?P\b[0-9a-f]{7,40}\b)", + re.IGNORECASE | re.DOTALL, +) + + +def _finding_key(description: str) -> tuple[str, str, str] | None: + match = _FINDING_RE.search(description) + if match is None: + return None + return ( + match.group("path").replace("\\", "/").lower(), + match.group("line"), + match.group("commit").lower(), + ) + + +def _existing_equivalent_fact(conn: Any, project_id: str, description: str) -> Fact | None: + normalized = " ".join(description.split()) + row = conn.execute( + "SELECT id, description, created_at FROM facts " + "WHERE project_id = %s AND description = %s ORDER BY created_at, id LIMIT 1", + (project_id, normalized), + ).fetchone() + if row is not None: + return Fact(id=row["id"], description=row["description"], created_at=row["created_at"]) + + key = _finding_key(normalized) + if key is None: + return None + rows = conn.execute( + "SELECT id, description, created_at FROM facts WHERE project_id = %s ORDER BY created_at, id", + (project_id,), + ).fetchall() + for row in rows: + if _finding_key(row["description"]) == key: + return Fact(id=row["id"], description=row["description"], created_at=row["created_at"]) + return None + + def insert_fact(conn: Any, project_id: str, fact_id: str, description: str) -> Fact: now = utcnow() conn.execute( @@ -26,6 +73,10 @@ def reserve_fact(conn: Any, project_id: str, description: str) -> Fact: def create_fact(conn: Any, project_id: str, description: str) -> Fact: + description = " ".join(description.split()) + existing = _existing_equivalent_fact(conn, project_id, description) + if existing is not None: + return existing fid = next_fact_id(conn, project_id) return insert_fact(conn, project_id, fid, description) diff --git a/backend/config/limits.yaml b/backend/config/limits.yaml index 1a6904d..f806a99 100644 --- a/backend/config/limits.yaml +++ b/backend/config/limits.yaml @@ -1,4 +1,4 @@ limits: - total_cpu: 4 # cores across all containers - max_concurrent_tasks: 5 # max CTF tasks running at once (one container each) + total_cpu: 12 # cores across all containers (4P+8E) + max_concurrent_tasks: 8 # max CTF tasks running at once (one container each) network: true # all containers may reach the network diff --git a/backend/core/config.py b/backend/core/config.py index 3e1520d..9d2c4f3 100644 --- a/backend/core/config.py +++ b/backend/core/config.py @@ -10,7 +10,6 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator # CTF challenge categories the user may pick when creating a project. -Category = Literal["pwn", "reverse", "crypto", "web", "misc", "ai", "osint"] CATEGORIES: tuple[str, ...] = ("pwn", "reverse", "crypto", "web", "misc", "ai", "osint") # Supported LLM wire formats. base_url is always user-provided. diff --git a/backend/core/ipc.py b/backend/core/ipc.py index a1f239b..37d3f41 100644 --- a/backend/core/ipc.py +++ b/backend/core/ipc.py @@ -139,17 +139,3 @@ def verify_postprocess(db, project_id: str, wp_dir: Path) -> dict: ): reasons.extend(f"invalid writeup: {error}" for error in errors) return {"ok": not reasons, "flag": flag, "wp_path": wp_path, "reasons": reasons} - - -def verify_flag_and_wp(db, project_id: str, wp_dir: Path) -> dict: - """Compatibility aggregate used by older callers and export checks.""" - - flag = verify_flag(db, project_id) - postprocess = verify_postprocess(db, project_id, wp_dir) - reasons = [*flag.get("reasons", []), *postprocess.get("reasons", [])] - return { - "ok": not reasons, - "flag": flag.get("flag"), - "wp_path": postprocess.get("wp_path"), - "reasons": reasons, - } diff --git a/backend/core/lifecycle.py b/backend/core/lifecycle.py index adb9fb3..e07e1ca 100644 --- a/backend/core/lifecycle.py +++ b/backend/core/lifecycle.py @@ -28,9 +28,6 @@ def status(self, project_id: str) -> str | None: row = graph_store.get_project_row(conn, project_id) return row["status"] if row else None - def can_transition(self, current: str, target: str) -> bool: - return target in TRANSITIONS.get(current, set()) - def transition(self, project_id: str, target: str) -> str: with self.db.connect() as conn: row = graph_store.get_project_row(conn, project_id) diff --git a/backend/core/logging_util.py b/backend/core/logging_util.py index bb11200..3343920 100644 --- a/backend/core/logging_util.py +++ b/backend/core/logging_util.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import shutil import threading from collections.abc import Callable from datetime import UTC, datetime @@ -101,10 +100,6 @@ def delete_project_logs(self, project_id: str) -> None: path = self.root / sub / self._project_filename(project_id) path.unlink(missing_ok=True) - def clear_all(self) -> None: - with self._lock: - shutil.rmtree(self.root, ignore_errors=True) - @staticmethod def _read_array(path: Path) -> list[dict]: if not path.exists(): @@ -140,7 +135,3 @@ def _tail(path: Path, limit: int | None) -> list[dict]: @staticmethod def jsonl_text(entries: list[dict]) -> str: return "".join(json.dumps(entry, ensure_ascii=False) + "\n" for entry in entries) - - @staticmethod - def write_jsonl(path: Path, entries: list[dict]) -> None: - path.write_text(IPCLogger.jsonl_text(entries), encoding="utf-8") diff --git a/backend/core/orchestrator.py b/backend/core/orchestrator.py index b8516a6..f8c0377 100644 --- a/backend/core/orchestrator.py +++ b/backend/core/orchestrator.py @@ -7,6 +7,7 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass import json +import sys from time import monotonic from typing import Any import uuid @@ -56,6 +57,8 @@ class PostprocessLeaseLost(RuntimeError): class Orchestrator: _MEMBER_RETRY_DELAYS = (10, 30, 90, 300) + _MAX_CONSECUTIVE_STALLS = 3 + _STALL_DEFER_SECONDS = 180 _POSTPROCESS_LEASE_SECONDS = 120 def __init__(self, state, max_workers: int | None = None, scripts: dict | None = None): @@ -88,6 +91,7 @@ def __init__(self, state, max_workers: int | None = None, scripts: dict | None = self._startup_futures: dict[str, Any] = {} self._member_failure_counts: dict[tuple[str, str], int] = {} self._member_retry_not_before: dict[tuple[str, str], float] = {} + self._member_stall_counts: dict[tuple[str, str], int] = {} self._project_leases: dict[str, str] = {} self._intent_leases: dict[tuple[str, str], IntentLease] = {} self._lock = threading.Lock() @@ -1211,7 +1215,13 @@ def _run_loop(self) -> None: try: self._tick() except Exception as exc: - self.state.logger.project("scheduler_error", "system", error=str(exc)) + # The fallback handler itself must never kill the loop: it + # logs through the DB-backed resolver, which is exactly what + # fails when the tick died on a database error. + try: + self.state.logger.project("scheduler_error", "system", error=str(exc)) + except Exception: + print(f"[ipc-orchestrator] tick failed: {type(exc).__name__}: {exc}", file=sys.stderr, flush=True) self._stop.wait(interval) def _tick(self) -> None: @@ -1438,6 +1448,32 @@ def _schedule_member_retry( )[:1500], ) + def _record_member_stall(self, project_id: str, intent_id: str) -> tuple[int, bool]: + """Defer a repeatedly stalled intent so a concrete alternate branch can run.""" + key = (project_id, intent_id) + with self._lock: + count = self._member_stall_counts.get(key, 0) + 1 + self._member_stall_counts[key] = count + should_defer = count >= self._MAX_CONSECUTIVE_STALLS + if should_defer: + self._member_retry_not_before[key] = monotonic() + self._STALL_DEFER_SECONDS + if should_defer: + # The finished-future path has already released the intent claim; + # only the redispatch backoff is needed here. + self.state.logger.project( + "member_intent_deferred", + project_id, + intent=intent_id, + stalls=count, + delay_seconds=self._STALL_DEFER_SECONDS, + reason="repeated_stalls", + ) + return count, should_defer + + def _clear_member_stalls(self, project_id: str, intent_id: str) -> None: + with self._lock: + self._member_stall_counts.pop((project_id, intent_id), None) + def _clear_member_retry(self, project_id: str, intent_id: str) -> None: key = (project_id, intent_id) with self._lock: @@ -1454,6 +1490,8 @@ def _clear_project_member_retries(self, project_id: str) -> None: self._member_failure_counts.pop(key, None) for key in [key for key in self._member_retry_not_before if key[0] == project_id]: self._member_retry_not_before.pop(key, None) + for key in [key for key in self._member_stall_counts if key[0] == project_id]: + self._member_stall_counts.pop(key, None) def _reconcile_resources(self) -> None: with self.state.db.connect() as conn: @@ -1711,6 +1749,7 @@ def _reap_finished_futures(self) -> None: continue self._clear_member_retry(project_id, intent_id) if result.status == "stalled": + stalls, deferred = self._record_member_stall(project_id, intent_id) self.state.logger.project( "member_task_stalled", project_id, @@ -1719,14 +1758,19 @@ def _reap_finished_futures(self) -> None: retryable=result.retryable, error_kind=result.error_kind, error=result.error, + consecutive_stalls=stalls, + deferred=deferred, ) elif result.status == "done": + self._clear_member_stalls(project_id, intent_id) self.state.logger.project("member_task_done", project_id, intent=intent_id, steps=result.steps) elif result.status == "concluded": + self._clear_member_stalls(project_id, intent_id) self.state.logger.project("member_task_concluded", project_id, intent=intent_id, fact=result.fact_id) if result.fact_id is not None: self._broadcast_fact(project_id, result.fact_id, self._intent_worker(project_id, intent_id)) elif result.status == "flag": + self._clear_member_stalls(project_id, intent_id) self.state.logger.project("member_task_flag", project_id, intent=intent_id, flag=result.flag) def _intent_worker(self, project_id: str, intent_id: str) -> str | None: diff --git a/backend/core/state.py b/backend/core/state.py index 9d5d50a..cc76362 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import re import shutil import socket import threading @@ -14,6 +15,8 @@ from backend.mcp.mcp_client import MCPRegistry from backend.memory.memory_mcp import build_memory_mcp from backend.memory.memory_store import MemoryStore +from backend.platform.ret2shell import Ret2ShellClient +from backend.platform.ret2shell_mcp import build_ret2shell_mcp from backend.sandbox.container_pool import ContainerPool from backend.sandbox.network_manager import NetworkManager from backend.sandbox.resource_limiter import TaskSlotLimiter @@ -57,6 +60,7 @@ def __init__(self, root: str | Path = ".", config_dir: Path | None = None): self.db = Database().configure() with self.db.connect() as conn: graph_store.reset_project_counter_if_empty(conn) + self._reserve_existing_project_ids(conn) self.memory = MemoryStore(self.db, export_dir=None).configure() self.registry = ToolRegistry().load() self.catalog = ToolCatalog.load(self.registry) @@ -96,10 +100,39 @@ def __init__(self, root: str | Path = ".", config_dir: Path | None = None): self.mcps = MCPRegistry() self.mcps.register(build_memory_mcp(self.memory, catalog=self.catalog)) self.mcps.register(build_tool_search_mcp(self.registry)) + # The ret2shell competition MCP (dynamic instance control) is only + # registered when participant credentials are configured, so members + # never see a platform they cannot reach. + if os.getenv("IPC_R2S_USERNAME") or os.getenv("IPC_R2S_TOKEN"): + self.mcps.register(build_ret2shell_mcp(Ret2ShellClient())) # Attached by module 8. self.orchestrator = None + def _reserve_existing_project_ids(self, conn) -> None: + """Keep a fresh database from reusing surviving workspace ids. + + PostgreSQL rows are durable, but the database can still be recreated + (e.g. a replaced container volume) while project workspace directories + on the shared Artifact root survive. Reserve the largest surviving + numeric ID so a newly created project never inherits old progress or + artifacts merely because the database started empty. + """ + if not self.projects_dir.is_dir(): + return + highest = 0 + for entry in self.projects_dir.iterdir(): + if not entry.is_dir(): + continue + match = re.fullmatch(r"proj_(\d+)", entry.name) + if match: + highest = max(highest, int(match.group(1))) + if highest: + conn.execute( + "UPDATE counters SET value = GREATEST(value, %s) WHERE name = 'project'", + (highest,), + ) + def _clean_runtime_state(self) -> None: # Database rows are intentionally not deleted by a local clean start. # Durable exports are preserved; only live workspaces and generated @@ -135,8 +168,14 @@ def close(self) -> None: self.db.close() def project_log_filename(self, project_id: str) -> str | None: - with self.db.connect() as conn: - return graph_store.project_log_filename(conn, project_id) + # Called from logging paths, including the scheduler's crash handler; + # a failing database must degrade to the plain id instead of raising + # through the logger and killing the caller. + try: + with self.db.connect() as conn: + return graph_store.project_log_filename(conn, project_id) + except Exception: + return project_id def attachments_dir(self, project_id: str) -> Path: d = self.projects_dir / project_id / "attachments" diff --git a/backend/mcp/mcp_client.py b/backend/mcp/mcp_client.py index 7d3a850..3e3d91e 100644 --- a/backend/mcp/mcp_client.py +++ b/backend/mcp/mcp_client.py @@ -12,11 +12,10 @@ from mcp import ClientSession, StdioServerParameters, types from mcp.client.stdio import stdio_client -from mcp.client.streamable_http import streamablehttp_client from mcp.server.fastmcp import FastMCP from mcp.shared.memory import create_connected_server_and_client_session -MCPClientTarget = FastMCP | StdioServerParameters | str +MCPClientTarget = FastMCP | StdioServerParameters class MCPToolError(RuntimeError): @@ -54,11 +53,9 @@ def __init__( target: MCPClientTarget, *, read_timeout: float | None = 120, - headers: Mapping[str, str] | None = None, ) -> None: self.target = target self.read_timeout = read_timeout - self.headers = dict(headers or {}) self._stack: AsyncExitStack | None = None self._session: ClientSession | None = None @@ -81,16 +78,6 @@ def stdio( **kwargs, ) - @classmethod - def streamable_http( - cls, - url: str, - *, - headers: Mapping[str, str] | None = None, - **kwargs: Any, - ) -> MCPClient: - return cls(url, headers=headers, **kwargs) - @property def session(self) -> ClientSession: if self._session is None: @@ -112,17 +99,7 @@ async def __aenter__(self) -> MCPClient: ) ) else: - if isinstance(self.target, StdioServerParameters): - read_stream, write_stream = await stack.enter_async_context(stdio_client(self.target)) - else: - read_stream, write_stream, _ = await stack.enter_async_context( - streamablehttp_client( - self.target, - headers=self.headers, - timeout=self.read_timeout or 30, - sse_read_timeout=self.read_timeout or 300, - ) - ) + read_stream, write_stream = await stack.enter_async_context(stdio_client(self.target)) session = await stack.enter_async_context( ClientSession(read_stream, write_stream, read_timeout_seconds=timeout) ) @@ -259,10 +236,6 @@ async def call_tool( async with self.session() as session: return await session.call_tool(server, tool, arguments) - async def describe(self) -> dict[str, list[dict[str, Any]]]: - async with self.session() as session: - return {name: await session.describe_tools(name) for name in self.names()} - async def _run_cli(args: argparse.Namespace) -> int: server_args = [ diff --git a/backend/members/adapters.py b/backend/members/adapters.py index 7b326d3..9969cf8 100644 --- a/backend/members/adapters.py +++ b/backend/members/adapters.py @@ -277,7 +277,11 @@ def decide(self, context: dict) -> MemberAction: messages, system_prompt=_SYSTEM_PROMPT, temperature=0.0 if deepseek else (None if reasoning_model else 0.4), - max_tokens=4096 if deepseek or reasoning_model else None, + # Forced-reasoning models (e.g. kimi-for-coding) spend the output + # budget on hidden reasoning before any content; 4096 was observed + # being fully consumed by reasoning alone on large contexts, leaving + # an empty response with finish_reason=length. + max_tokens=16384 if reasoning_model else (4096 if deepseek else None), structured=True, reasoning_effort=reasoning_effort, thinking="disabled" if deepseek else None, @@ -298,7 +302,7 @@ def decide(self, context: dict) -> MemberAction: [*messages, {"role": "user", "content": repair_message}], system_prompt=_SYSTEM_PROMPT, temperature=0.0 if not reasoning_model else None, - max_tokens=4096 if deepseek or reasoning_model else 1024, + max_tokens=16384 if reasoning_model else (4096 if deepseek else 1024), structured=True, reasoning_effort=reasoning_effort, thinking="disabled" if deepseek else None, @@ -362,7 +366,15 @@ def _default_profile( thinking: str | None, ) -> _OpenAIProfile: reasoning_model = _is_reasoning_model(self.config.model) - native_or_reasoning = _is_native_openai(self.config.base_url) or reasoning_model + # Reasoning-model names alone do not guarantee that a compatible + # gateway implements OpenAI's strict JSON Schema wire shape. When an + # operator explicitly selects Chat Completions on a non-native + # endpoint, prefer the broadly supported JSON Object mode (and its + # matching token parameter). Native OpenAI keeps the strict schema + # profile used by Responses/Chat reasoning models. + native_or_reasoning = _is_native_openai(self.config.base_url) or ( + reasoning_model and self.config.api_surface == "auto" + ) if not structured: structured_mode = "none" elif self.config.api_format == "deepseek": @@ -771,8 +783,14 @@ def _provider_error_from_response(response: Any, provider: str) -> ProviderError status = _response_status(response) retry_after = _retry_after_seconds(response) error_type = RetryableProviderError if _is_retryable_status(status) else NonRetryableProviderError + # Surface the gateway's own reason (rate limit, content filter, schema + # rejection...) — a bare "HTTP 400" is undiagnosable from logs. + detail = (getattr(response, "text", "") or "").strip()[:500] + message = f"{provider} endpoint returned HTTP {status}" + if detail: + message = f"{message} | gateway response: {detail}" return error_type( - f"{provider} endpoint returned HTTP {status}", + message, provider=provider, status_code=status, retry_after=retry_after, @@ -882,6 +900,9 @@ def _http_error_summary(exc: Exception) -> str: def _is_reasoning_model(model: str) -> bool: name = (model or "").strip().lower().rsplit("/", 1)[-1] + # Kimi For Coding always thinks; give it the reasoning-model budget. + if "kimi-for-coding" in name or name.startswith("kimi-k"): + return True if name.startswith("gpt-"): version = name[4:].split("-", 1)[0] try: diff --git a/backend/members/base_member.py b/backend/members/base_member.py index d2151ea..508e808 100644 --- a/backend/members/base_member.py +++ b/backend/members/base_member.py @@ -1214,6 +1214,36 @@ def _build_context(self, project_id, intent_id, category, step, is_initial, eval if callable(attachment_path): for attachment in attachments: attachment["path"] = attachment_path(attachment["filename"], attachment["path"]) + runtime_notes = [ + "If sandbox_backend is LocalSandbox, use host shell-compatible commands only.", + "If sandbox_backend is MemberSandbox, use Linux commands inside the shared task container.", + "Flag search priority for this round: try /flag first, then environment variables, then other methods.", + "If attachment_true is true, inspect the listed attachments before blind target probing.", + ( + "challenge_description contains the original challenge statement " + "(connection commands like nc/ssh, service addresses, flag format). " + "external_id is this challenge's platform id — use it with ret2shell MCP tools." + ), + ] + # Platform verdicts flow back through the experience memory: surface + # any flags the platform already rejected for THIS challenge so the + # member never resubmits them and re-derives a complete flag instead. + try: + rejected_flags: list[str] = [] + for m in d.memory.list(None): + if m.project_id != project_id or "rejected-flag" not in m.tags: + continue + rejected_flags.extend(re.findall(r"'([^']+)'", m.content)) + if rejected_flags: + runtime_notes.append( + "Platform already REJECTED these flag strings for this challenge — " + f"do NOT resubmit them: {', '.join(sorted(set(rejected_flags)))}. " + "Re-derive the complete flag from the challenge material (check every " + "file/resource, including binary XML and encoded blobs; recombine ALL " + "fragments) and only report a flag that reads as a coherent whole." + ) + except Exception: + pass return { "role": self.name, "role_blurb": self.role_blurb, @@ -1228,16 +1258,15 @@ def _build_context(self, project_id, intent_id, category, step, is_initial, eval "or new evidence justifies escalation. Do not repeat previous attempts." ), "sandbox_backend": getattr(d.sandbox, "__class__", type(d.sandbox)).__name__, - "runtime_notes": [ - "If sandbox_backend is LocalSandbox, use host shell-compatible commands only.", - "If sandbox_backend is MemberSandbox, use Linux commands inside the shared task container.", - "Flag search priority for this round: try /flag first, then environment variables, then other methods.", - "If attachment_true is true, inspect the listed attachments before blind target probing.", - ], + "runtime_notes": runtime_notes, "evaluate_now": evaluate_now, "eval_interval": d.eval_interval, "is_initial": is_initial, "expected_flag": d.expected_flag, + "challenge_description": next( + (f.description for f in detail.facts if f.id == "origin"), "" + ), + "external_id": detail.project.external_id, "goal": next((f.description for f in detail.facts if f.id == "goal"), ""), "assigned_intent": {"id": intent_id, "description": assigned.description if assigned else ""}, "assigned_intent_sources": assigned.from_ if assigned else ["origin"], diff --git a/backend/memory/memory_store.py b/backend/memory/memory_store.py index dd5bf2c..ba46d95 100644 --- a/backend/memory/memory_store.py +++ b/backend/memory/memory_store.py @@ -26,10 +26,6 @@ class Memory(BaseModel): source: str = "diamond" created_at: str - def keywords(self) -> set[str]: - text = f"{self.title} {self.content} {' '.join(self.tags)}".lower() - return {word for word in _tokenize(text) if len(word) >= 2} - def _tokenize(text: str) -> list[str]: output: list[str] = [] diff --git a/backend/ops/ipc_mcp.py b/backend/ops/ipc_mcp.py index 20971dc..bede1c6 100644 --- a/backend/ops/ipc_mcp.py +++ b/backend/ops/ipc_mcp.py @@ -388,7 +388,11 @@ def task_sandbox_health(self, project_id: str) -> dict[str, Any]: return self._run_project_tool(project_id, "task_sandbox_health") def task_sandbox_exec(self, project_id: str, command: str, timeout: int) -> dict[str, Any]: - return self._run_project_tool(project_id, "task_sandbox_exec", project_id, command, timeout) + # ``_run_project_tool`` injects project_id into the executor payload; + # pass only the command arguments here. Passing project_id twice made + # the MCP bridge fail with ``too many values to unpack`` before the + # sandbox command could run. + return self._run_project_tool(project_id, "task_sandbox_exec", command, timeout) def host_exec(self, command: str, timeout: int) -> dict[str, Any]: try: diff --git a/backend/ops/models.py b/backend/ops/models.py index b004c72..a8a22a8 100644 --- a/backend/ops/models.py +++ b/backend/ops/models.py @@ -82,7 +82,9 @@ def validate_prefix(cls, value: str) -> str: class ChallengeMappingSpec(BaseModel): model_config = ConfigDict(extra="forbid") - list_url: str + platform: Literal["http_json", "ret2shell"] = "http_json" + game_id: int | None = None + list_url: str = "" list_path: str = "data" id_field: str = "id" title_field: str = "name" @@ -96,7 +98,13 @@ class ChallengeMappingSpec(BaseModel): @field_validator("list_url") @classmethod def validate_list_url(cls, value: str) -> str: - return _validate_http_url(value) + return _validate_http_url(value) if value.strip() else "" + + @model_validator(mode="after") + def require_list_url_for_http_json(self) -> ChallengeMappingSpec: + if self.platform == "http_json" and not self.list_url.strip(): + raise ValueError("list_url is required for the http_json platform") + return self @field_validator("attachment_base_url") @classmethod @@ -127,6 +135,8 @@ def unique_headers(self) -> ChallengeMappingSpec: def to_field_mapping(self, headers: dict[str, str]) -> FieldMapping: return FieldMapping( + platform=self.platform, + game_id=self.game_id, list_url=self.list_url, list_path=self.list_path, id_field=self.id_field, diff --git a/backend/ops/service.py b/backend/ops/service.py index 573c7fa..2c5f837 100644 --- a/backend/ops/service.py +++ b/backend/ops/service.py @@ -25,7 +25,8 @@ from backend.ops.claude_runner import ClaudeCodeRunner, ClaudeCodeRunnerError from backend.ops.store import OpsStore from backend.ops.tools import OpsToolError, OpsToolExecutor, tool_prompt -from backend.platform.adapter import HttpJsonAdapter +from backend.platform.adapter import HttpJsonAdapter, PlatformAdapter +from backend.platform.ret2shell import Ret2ShellAdapter, Ret2ShellClient _JSON_DECODER = json.JSONDecoder() _MAX_TOOL_ROUNDS = 8 @@ -1237,10 +1238,19 @@ def workflow_view(self, workflow: dict[str, Any]) -> dict[str, Any]: "confirmation_phrase": f"CONFIRM WORKFLOW {workflow['id']}", } - def _adapter(self, workflow_id: str, spec: PlatformWorkflowSpec) -> HttpJsonAdapter: + def _adapter(self, workflow_id: str, spec: PlatformWorkflowSpec) -> PlatformAdapter: secrets_values = self.store.workflow_secrets(workflow_id) headers = _resolve_headers(spec.challenges.headers, secrets_values) mapping = spec.challenges.to_field_mapping(headers) + if mapping.platform == "ret2shell": + # Credentials come from IPC_R2S_* environment variables inside the + # backend process; the workflow spec carries no secrets. + return Ret2ShellAdapter( + Ret2ShellClient(game_id=mapping.game_id), + game_id=mapping.game_id or None, + category_map=mapping.category_map, + max_attachment_bytes=spec.max_attachment_bytes, + ) return HttpJsonAdapter( mapping, request_get=self._http_client(spec).get, diff --git a/backend/persistence/database.py b/backend/persistence/database.py index f28cff5..0a110ac 100644 --- a/backend/persistence/database.py +++ b/backend/persistence/database.py @@ -1,8 +1,6 @@ from __future__ import annotations import os -import random -import time from collections.abc import Generator, Iterable, Mapping from contextlib import contextmanager from datetime import date, datetime @@ -10,7 +8,6 @@ from typing import Any from psycopg import Connection -from psycopg.errors import DeadlockDetected, SerializationFailure from psycopg.rows import dict_row from psycopg_pool import ConnectionPool @@ -163,16 +160,6 @@ def connect(self) -> Generator[DatabaseConnection, None, None]: with raw.transaction(): yield DatabaseConnection(raw) - def run_transaction(self, callback, *, retries: int = 3): - for attempt in range(retries): - try: - with self.connect() as connection: - return callback(connection) - except (DeadlockDetected, SerializationFailure): - if attempt + 1 >= retries: - raise - time.sleep((0.05 * (2**attempt)) + random.random() * 0.05) - def close(self) -> None: if self._opened: self._pool.close() diff --git a/backend/platform/gzctf.py b/backend/platform/gzctf.py new file mode 100644 index 0000000..f15e9af --- /dev/null +++ b/backend/platform/gzctf.py @@ -0,0 +1,448 @@ +from __future__ import annotations + +import os +from datetime import UTC, datetime +from typing import Any + +import requests + + +class GZCTFError(RuntimeError): + pass + + +class GZCTFLoginError(GZCTFError): + pass + + +class GZCTFPreflightError(GZCTFError): + """A submission was refused before it could consume a platform attempt.""" + + +def _track_id(item: dict[str, Any]) -> str: + for key in ("trackId", "track_id", "id"): + value = item.get(key) + if value: + return str(value) + return "" + + +def _track_entries(challenge: dict[str, Any]) -> list[dict[str, Any]]: + entries = challenge.get("tracks") + if isinstance(entries, list): + flags = challenge.get("flags") if isinstance(challenge.get("flags"), list) else [] + result: list[dict[str, Any]] = [] + for item in entries: + if not isinstance(item, dict) or not _track_id(item): + continue + # GZCTF exposes track metadata and per-level state as sibling + # arrays. Join them here so preflight sees one uniform shape. + joined = dict(item) + if not isinstance(joined.get("levels"), list): + joined["levels"] = [ + flag + for flag in flags + if isinstance(flag, dict) and _track_id(flag) == _track_id(item) + ] + result.append(joined) + return result + # Some GZCTF versions expose the per-track level state under `flags`. + entries = challenge.get("flags") + if isinstance(entries, list): + return [item for item in entries if isinstance(item, dict) and _track_id(item)] + return [] + + +def _level_entries(track: dict[str, Any]) -> list[dict[str, Any]]: + entries = track.get("levels") + if isinstance(entries, list): + return [item for item in entries if isinstance(item, dict)] + entries = track.get("flags") + if isinstance(entries, list): + return [item for item in entries if isinstance(item, dict)] + return [] + + +def _find_level(track: dict[str, Any], level: int) -> dict[str, Any] | None: + for item in _level_entries(track): + raw = item.get("level", item.get("levelId")) + try: + if int(raw) == level: + return item + except (TypeError, ValueError): + continue + return None + + +def _parse_time(value: Any) -> datetime | None: + if not value or not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + + +def _active_cooldown(*objects: dict[str, Any]) -> str | None: + now = datetime.now(UTC) + for obj in objects: + cooldown = obj.get("cooldown") + if isinstance(cooldown, dict): + value = cooldown.get("cooldownUntil") + else: + value = obj.get("cooldownUntil") + until = _parse_time(value) + if until and until > now: + return str(value) + return None + + +def _normalize_answer(answer: Any) -> str: + return str(answer).strip() + + +def validate_submission( + challenge: dict[str, Any], + *, + level: int, + answer: Any, + track_id: str | None = None, + evidence: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Validate one submission against live challenge state. + + This is intentionally conservative. It refuses ambiguous tracks, + locked/solved levels, active cooldowns, unverified evidence and + malformed answers before the POST endpoint is called. + """ + + try: + level = int(level) + except (TypeError, ValueError) as exc: + raise GZCTFPreflightError("level must be a positive integer") from exc + normalized_answer = _normalize_answer(answer) + if not normalized_answer: + raise GZCTFPreflightError("answer must not be empty") + + challenge_lock = challenge.get("challengeLock", challenge.get("challengeLocked", False)) + if isinstance(challenge_lock, dict): + # Current GZCTF returns a status object such as + # ``{"isLocked": false, "unlocksAt": null}``; treating the + # object itself as truthy would incorrectly reject every preflight. + challenge_lock = challenge_lock.get( + "isLocked", challenge_lock.get("locked", challenge_lock.get("is_locked", False)) + ) + if bool(challenge_lock): + raise GZCTFPreflightError("challenge is currently locked") + + tracks = _track_entries(challenge) + enabled = [item for item in tracks if item.get("isEnabled", item.get("is_enabled", True))] + if not enabled: + raise GZCTFPreflightError("live challenge has no enabled track state") + if track_id: + track = next((item for item in enabled if _track_id(item) == str(track_id)), None) + if track is None: + raise GZCTFPreflightError(f"trackId is not an enabled track: {track_id}") + elif len(enabled) == 1: + track = enabled[0] + track_id = _track_id(track) + else: + raise GZCTFPreflightError( + "challenge exposes multiple enabled tracks; an explicit trackId is required" + ) + + level_state = _find_level(track, level) + if level_state is None: + raise GZCTFPreflightError(f"track {track_id} has no level {level} state") + status = str(level_state.get("status", level_state.get("state", ""))).strip().lower() + if status in {"solved", "complete", "completed"} or bool(level_state.get("solved")): + raise GZCTFPreflightError(f"level {level} is already solved") + if status in {"locked", "unavailable", "banned"}: + raise GZCTFPreflightError(f"level {level} is locked") + current_level = track.get("currentLevel", track.get("current_level")) + try: + if current_level is not None and level > int(current_level): + raise GZCTFPreflightError( + f"level {level} is locked until level {int(current_level)} is solved" + ) + except (TypeError, ValueError): + pass + + cooldown = _active_cooldown(challenge, track, level_state) + if cooldown: + raise GZCTFPreflightError(f"submission cooldown active until {cooldown}") + cooldown_obj = level_state.get("cooldown") + if not isinstance(cooldown_obj, dict): + cooldown_obj = track.get("cooldown") if isinstance(track.get("cooldown"), dict) else challenge.get("cooldown", {}) + if isinstance(cooldown_obj, dict): + wrong_count = cooldown_obj.get("wrongCount", cooldown_obj.get("wrong_count")) + wrong_limit = cooldown_obj.get("wrongAnswerLimit", cooldown_obj.get("wrong_answer_limit")) + try: + if wrong_count is not None and wrong_limit is not None and int(wrong_count) >= int(wrong_limit): + raise GZCTFPreflightError("wrong-answer limit reached; wait for the platform cooldown") + except (TypeError, ValueError): + pass + + if evidence is not None: + status_value = str(evidence.get("status", "candidate")).strip().lower() + answers = evidence.get("answers") if isinstance(evidence.get("answers"), dict) else {} + evidence_answer = answers.get(str(level), answers.get(level)) + if evidence_answer is None or _normalize_answer(evidence_answer) != normalized_answer: + raise GZCTFPreflightError("submitted answer does not match the evidence report") + if status_value not in {"verified", "confirmed"}: + raise GZCTFPreflightError("candidate evidence has not been verified") + + return { + "ok": True, + "challenge_id": challenge.get("id"), + "track_id": track_id, + "level": level, + "answer": normalized_answer, + "status": status or "unsolved", + "current_level": current_level, + } + + +class GZCTFClient: + """Cookie-authenticated GZCTF participant API client. + + GZCTF flag submission requires the normal user cookie session, not the + admin-only API token. This client keeps that distinction explicit. + """ + + def __init__( + self, + base_url: str = "", + username: str = "", + password: str = "", + *, + timeout: float = 30, + ) -> None: + self.base_url = (base_url or os.getenv("IPC_GZ_BASE_URL", "")).rstrip("/") + self.username = username or os.getenv("IPC_GZ_USERNAME", "") + self.password = password or os.getenv("IPC_GZ_PASSWORD", "") + self.timeout = timeout + self.session = requests.Session() + self.logged_in = False + + def _url(self, path: str) -> str: + return f"{self.base_url}{path}" + + def _require_base_url(self) -> None: + if not self.base_url: + raise GZCTFError("GZCTF base_url is not configured; set IPC_GZ_BASE_URL") + + def login(self) -> None: + self._require_base_url() + if not self.username or not self.password: + raise GZCTFLoginError("GZCTF username/password are not configured") + response = self.session.post( + self._url("/api/Account/LogIn"), + json={"userName": self.username, "password": self.password}, + timeout=self.timeout, + ) + if response.status_code != 200: + try: + message = response.json().get("title", "login failed") + except ValueError: + message = "login failed" + raise GZCTFLoginError(message) + self.logged_in = True + + def _authorized_get(self, path: str, *, params: dict[str, Any] | None = None) -> Any: + self._require_base_url() + if not self.logged_in: + self.login() + response = self.session.get( + self._url(path), params=params, timeout=self.timeout + ) + response.raise_for_status() + return response.json() + + def _authorized_post( + self, + path: str, + json: dict[str, Any] | None = None, + *, + params: dict[str, Any] | None = None, + ) -> Any: + self._require_base_url() + if not self.logged_in: + self.login() + response = self.session.post( + self._url(path), + json=json or {}, + params=params, + timeout=self.timeout, + ) + if response.status_code >= 400: + # Keep the platform's validation reason in the exception. A bare + # ``raise_for_status`` only exposed ``400 Bad Request`` and made + # it impossible to distinguish a contract mismatch from a wrong + # answer without issuing another blind attempt. + detail = response.text.strip().replace("\r", " ").replace("\n", " ") + if len(detail) > 1000: + detail = detail[:1000] + "..." + suffix = f": {detail}" if detail else "" + raise requests.HTTPError( + f"{response.status_code} Client Error for url: {response.url}{suffix}", + response=response, + ) + return response.json() + + def list_games(self, *, query: dict[str, Any] | None = None) -> Any: + """List visible competitions without selecting one implicitly.""" + + return self._authorized_get("/api/game", params=query) + + def list_teams(self) -> Any: + return self._authorized_get("/api/team") + + def create_team(self, payload: dict[str, Any]) -> Any: + return self._authorized_post("/api/team", payload) + + def accept_team_invite(self, payload: dict[str, Any]) -> Any: + return self._authorized_post("/api/team/accept", payload) + + def join_game(self, game_id: int, payload: dict[str, Any] | None = None) -> Any: + return self._authorized_post(f"/api/game/{game_id}", payload or {}) + + def leave_game(self, game_id: int) -> Any: + self._require_base_url() + if not self.logged_in: + self.login() + response = self.session.delete( + self._url(f"/api/game/{game_id}"), timeout=self.timeout + ) + response.raise_for_status() + return response.json() if response.content else {} + + def get_profile(self) -> dict[str, Any]: + return self._authorized_get("/api/Account/Profile") + + def get_game(self, game_id: int) -> dict[str, Any]: + return self._authorized_get(f"/api/Game/{game_id}") + + def get_game_details(self, game_id: int) -> dict[str, Any]: + return self._authorized_get(f"/api/Game/{game_id}/Details") + + def get_game_check(self, game_id: int) -> dict[str, Any]: + return self._authorized_get(f"/api/Game/{game_id}/Check") + + def get_game_participation(self, game_id: int) -> Any: + return self._authorized_get(f"/api/game/{game_id}/participations") + + def get_scoreboard(self, game_id: int) -> dict[str, Any]: + return self._authorized_get(f"/api/Game/{game_id}/Scoreboard") + + def get_challenge( + self, + game_id: int, + challenge_id: int, + ) -> dict[str, Any]: + return self._authorized_get( + f"/api/Game/{game_id}/Challenges/{challenge_id}" + ) + + def preflight_submission( + self, + game_id: int, + challenge_id: int, + *, + level: int, + answer: Any, + track_id: str | None = None, + evidence: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Fetch live state and validate without issuing a submission POST.""" + + challenge = self.get_challenge(game_id, challenge_id) + return validate_submission( + challenge, + level=level, + answer=answer, + track_id=track_id, + evidence=evidence, + ) + + def submit_level( + self, + game_id: int, + challenge_id: int, + answer: Any, + *, + level: int = 1, + track_id: str | None = None, + evidence: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Run preflight immediately before the single allowed POST.""" + + decision = self.preflight_submission( + game_id, + challenge_id, + level=level, + answer=answer, + track_id=track_id, + evidence=evidence, + ) + return self.submit_flag( + game_id, + challenge_id, + decision["answer"], + level=decision["level"], + track_id=decision["track_id"], + ) + + def submit_flag( + self, + game_id: int, + challenge_id: int, + flag: str, + *, + level: int = 1, + track_id: str | None = None, + ) -> dict[str, Any]: + body: dict[str, Any] = {"flag": flag, "level": level} + if track_id: + body["trackId"] = track_id + result = self._authorized_post( + f"/api/Game/{game_id}/Challenges/{challenge_id}", + body, + ) + if not isinstance(result, dict): + raise GZCTFError( + f"GZCTF submit_flag returned unexpected type: {type(result).__name__}" + ) + return result + + def get_submission_status( + self, + game_id: int, + challenge_id: int, + submission_id: int, + ) -> dict[str, Any]: + return self._authorized_get( + f"/api/Game/{game_id}/Challenges/{challenge_id}/Status/{submission_id}" + ) + + def get_my_submissions(self, *, query: dict[str, Any] | None = None) -> Any: + return self._authorized_get("/api/ext/submissions/mine", params=query) + + def get_my_tickets(self, *, query: dict[str, Any] | None = None) -> Any: + return self._authorized_get("/api/ext/tickets/mine", params=query) + + def get_ticket(self, ticket_id: int | str) -> Any: + return self._authorized_get(f"/api/ext/tickets/{ticket_id}") + + def create_ticket(self, payload: dict[str, Any]) -> Any: + return self._authorized_post("/api/ext/tickets", payload) + + def close(self) -> None: + self.session.close() + + def __enter__(self) -> GZCTFClient: + return self + + def __exit__(self, *exc: object) -> None: + self.close() diff --git a/backend/platform/mapping.py b/backend/platform/mapping.py index b1c3a77..44fda79 100644 --- a/backend/platform/mapping.py +++ b/backend/platform/mapping.py @@ -1,12 +1,19 @@ from __future__ import annotations -from pydantic import BaseModel, ConfigDict, Field, field_validator +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class FieldMapping(BaseModel): model_config = ConfigDict(extra="forbid") - list_url: str + # ``http_json`` fetches any JSON list endpoint with static headers. + # ``ret2shell`` uses the authenticated ret2shell client (credentials come + # from IPC_R2S_* environment variables, never from the request body). + platform: Literal["http_json", "ret2shell"] = "http_json" + game_id: int | None = None + list_url: str = "" list_path: str = "data" id_field: str = "id" title_field: str = "name" @@ -21,10 +28,16 @@ class FieldMapping(BaseModel): @classmethod def validate_list_url(cls, value: str) -> str: value = value.strip() - if not value.startswith(("http://", "https://")): + if value and not value.startswith(("http://", "https://")): raise ValueError("list_url must use http or https") return value + @model_validator(mode="after") + def require_list_url_for_http_json(self) -> FieldMapping: + if self.platform == "http_json" and not self.list_url: + raise ValueError("list_url is required for the http_json platform") + return self + class PlatformChallenge(BaseModel): external_id: str diff --git a/backend/platform/ret2shell.py b/backend/platform/ret2shell.py new file mode 100644 index 0000000..bdc1f26 --- /dev/null +++ b/backend/platform/ret2shell.py @@ -0,0 +1,596 @@ +from __future__ import annotations + +import hashlib +import os +import threading +import time +from pathlib import Path +from typing import Any, Callable + +import requests + +from backend.core.config import CATEGORIES +from backend.filename_util import numbered_filename, safe_stem +from backend.platform.adapter import PlatformAdapter +from backend.platform.mapping import PlatformChallenge + +DEFAULT_BASE_URL = "https://ctf.xidian.edu.cn" +DEFAULT_GAME_ID = 37 + +# ret2shell limits each account to 10 flag submissions per 5-minute window +# (HTTP 429). The client-side limiter guards that shared quota. +SUBMIT_WINDOW_SECONDS = 300.0 +SUBMIT_LIMIT = 10 +# ret2shell judges submissions asynchronously: the POST returns a Submission +# with ``solved: null`` and the result must be polled. +SUBMIT_POLL_ATTEMPTS = 7 +SUBMIT_POLL_INTERVAL = 1.0 +# Instance exposed_ports appear asynchronously after a start request. +INSTANCE_WAIT_TIMEOUT = 60.0 +INSTANCE_WAIT_INTERVAL = 2.0 + + +class Ret2ShellError(RuntimeError): + pass + + +class Ret2ShellAuthError(Ret2ShellError): + pass + + +class Ret2ShellPreflightError(Ret2ShellError): + """A submission was refused before it could consume a platform attempt.""" + + +class Ret2ShellRateLimitError(Ret2ShellError): + pass + + +class _SubmitRateLimiter: + """Guard the shared 10-submissions-per-5-minutes account quota.""" + + def __init__( + self, + *, + limit: int = SUBMIT_LIMIT, + window: float = SUBMIT_WINDOW_SECONDS, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self.limit = limit + self.window = window + self._monotonic = monotonic + self._lock = threading.Lock() + self._attempts: list[float] = [] + + def _trim(self, now: float) -> None: + self._attempts = [ts for ts in self._attempts if now - ts < self.window] + + def reserve(self) -> None: + with self._lock: + now = self._monotonic() + self._trim(now) + if len(self._attempts) >= self.limit: + oldest = min(self._attempts) + retry_in = self.window - (now - oldest) + raise Ret2ShellRateLimitError( + f"local submit quota exhausted ({self.limit} per " + f"{int(self.window)}s); retry in {retry_in:.0f}s" + ) + self._attempts.append(now) + + def used(self) -> int: + with self._lock: + self._trim(self._monotonic()) + return len(self._attempts) + + +def _error_detail(response: requests.Response) -> str: + # ret2shell errors are plain-text bodies, not JSON. + detail = response.text.strip().replace("\r", " ").replace("\n", " ") + if len(detail) > 500: + detail = detail[:500] + "..." + return f"{response.status_code}: {detail}" if detail else str(response.status_code) + + +class Ret2ShellClient: + """ret2shell (Rust/axum) participant API client. + + Auth is a JWT Bearer token issued by ``POST /api/account/login`` and + returned in the ``Set-Token`` response header (never in the body). Any + later response may carry a refreshed ``Set-Token``, so every response is + checked and the token rolled forward. All timestamps are unix seconds + and list endpoints return ``[items, total]`` tuples. + """ + + def __init__( + self, + base_url: str = "", + game_id: int | None = None, + username: str = "", + password: str = "", + token: str = "", + *, + timeout: float = 30, + session: requests.Session | None = None, + poll_attempts: int = SUBMIT_POLL_ATTEMPTS, + poll_interval: float = SUBMIT_POLL_INTERVAL, + sleep: Callable[[float], None] = time.sleep, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self.base_url = (base_url or os.getenv("IPC_R2S_BASE_URL", "") or DEFAULT_BASE_URL).rstrip("/") + raw_game_id = game_id if game_id is not None else os.getenv("IPC_R2S_GAME_ID", "") + try: + self.game_id = int(raw_game_id) if str(raw_game_id).strip() else DEFAULT_GAME_ID + except (TypeError, ValueError) as exc: + raise Ret2ShellError(f"invalid IPC_R2S_GAME_ID: {raw_game_id!r}") from exc + self.username = username or os.getenv("IPC_R2S_USERNAME", "") + self.password = password or os.getenv("IPC_R2S_PASSWORD", "") + self.timeout = timeout + self.session = session or requests.Session() + self.poll_attempts = max(1, poll_attempts) + self.poll_interval = poll_interval + self._sleep = sleep + self._monotonic = monotonic + self._token = token or os.getenv("IPC_R2S_TOKEN", "") + self._limiter = _SubmitRateLimiter(monotonic=monotonic) + + # ---- low-level request handling ---- + + def _url(self, path: str) -> str: + return f"{self.base_url}/api{path}" + + def _absorb_token(self, response: requests.Response) -> None: + token = response.headers.get("Set-Token") + if token: + self._token = token + + def _headers(self) -> dict[str, str]: + headers: dict[str, str] = {} + if self._token: + headers["Authorization"] = f"Bearer {self._token}" + return headers + + def _request( + self, + method: str, + path: str, + *, + json: Any = None, + params: dict[str, Any] | None = None, + stream: bool = False, + _relogin: bool = True, + ) -> requests.Response: + if not self._token: + self.login() + response = self.session.request( + method, + self._url(path), + json=json, + params=params, + headers=self._headers(), + timeout=self.timeout, + stream=stream, + ) + self._absorb_token(response) + if response.status_code == 401 and _relogin: + self._token = "" + self.login() + return self._request( + method, path, json=json, params=params, stream=stream, _relogin=False + ) + return response + + def _request_ok( + self, + method: str, + path: str, + *, + json: Any = None, + params: dict[str, Any] | None = None, + ) -> Any: + response = self._request(method, path, json=json, params=params) + if response.status_code >= 400: + raise Ret2ShellError(_error_detail(response)) + if not response.content: + return None + try: + return response.json() + except ValueError as exc: + raise Ret2ShellError( + f"unexpected non-JSON response from {path}: {response.text[:200]!r}" + ) from exc + + # ---- auth ---- + + @staticmethod + def solve_pow(challenge: str, *, max_attempts: int = 5_000_000) -> str: + """Solve a ret2shell CLI proof-of-work captcha. + + The server issues ``"#"`` and accepts any answer + that starts with ```` and whose SHA-256 hex digest starts + with ``difficulty`` zero characters (crates/captcha/src/pow.rs). + """ + + difficulty_raw, _, prefix = challenge.partition("#") + try: + difficulty = int(difficulty_raw) + except ValueError as exc: + raise Ret2ShellError(f"invalid pow challenge: {challenge!r}") from exc + target = "0" * difficulty + for counter in range(max_attempts): + answer = f"{prefix}{counter}" + if hashlib.sha256(answer.encode()).hexdigest().startswith(target): + return answer + raise Ret2ShellError(f"pow challenge not solved within {max_attempts} attempts") + + def _fresh_captcha(self) -> dict[str, str]: + # The CLI endpoint always issues a PoW (difficulty 4), which the + # agent can solve autonomously — unlike the browser image captcha. + response = self.session.get(self._url("/account/captcha/cli"), timeout=self.timeout) + if response.status_code != 200: + raise Ret2ShellAuthError(f"captcha fetch failed: {_error_detail(response)}") + payload = response.json() + if not isinstance(payload, dict) or not payload.get("id"): + raise Ret2ShellAuthError("captcha response has no id") + challenge = str(payload.get("challenge", "")) + # A non-PoW challenge (image/hcaptcha) cannot be solved by the agent; + # pass it through and let login fail with the platform's reason. + answer = self.solve_pow(challenge) if "#" in challenge else "" + return {"captcha_id": str(payload["id"]), "captcha_answer": answer} + + def login(self) -> None: + if not self.username or not self.password: + raise Ret2ShellAuthError( + "ret2shell username/password are not configured; " + "set IPC_R2S_USERNAME/IPC_R2S_PASSWORD or IPC_R2S_TOKEN" + ) + body = { + "account": self.username, + "password": self.password, + "captcha_id": "", + "captcha_answer": "", + } + # The platform may enable captchas (410 "captcha is outdate" otherwise). + # The CLI PoW endpoint works headlessly, so try to pre-solve one; if + # the deployment disabled captchas the fields are simply ignored. + try: + body.update(self._fresh_captcha()) + except Ret2ShellAuthError: + pass + response = self.session.post( + self._url("/account/login"), + json=body, + timeout=self.timeout, + ) + self._absorb_token(response) + if response.status_code != 200 or not self._token: + raise Ret2ShellAuthError(f"ret2shell login failed: {_error_detail(response)}") + + # ---- games / challenges ---- + + def ping(self) -> str: + response = self.session.get(self._url("/ping"), timeout=self.timeout) + if response.status_code != 200: + raise Ret2ShellError(_error_detail(response)) + return response.text + + def get_profile(self) -> dict[str, Any]: + return self._request_ok("GET", "/account/profile") + + def get_game(self, game_id: int | None = None) -> dict[str, Any]: + return self._request_ok("GET", f"/game/{game_id or self.game_id}") + + def list_games(self) -> list[dict[str, Any]]: + payload = self._request_ok("GET", "/game") + items = payload[0] if isinstance(payload, list) and payload and isinstance(payload[0], list) else payload + return items if isinstance(items, list) else [] + + def list_challenges(self, game_id: int | None = None) -> list[dict[str, Any]]: + payload = self._request_ok("GET", f"/game/{game_id or self.game_id}/challenge") + if not (isinstance(payload, list) and payload and isinstance(payload[0], list)): + raise Ret2ShellError("challenge list response is not an [items, total] tuple") + return payload[0] + + def get_challenge(self, challenge_id: int, game_id: int | None = None) -> dict[str, Any]: + return self._request_ok( + "GET", f"/game/{game_id or self.game_id}/challenge/{challenge_id}" + ) + + def challenge_status(self, challenge_id: int, game_id: int | None = None) -> dict[str, Any]: + """Own solve state plus the platform-wide solve count.""" + + return self._request_ok( + "GET", f"/game/{game_id or self.game_id}/challenge/{challenge_id}/submit" + ) + + def has_environment(self, challenge_id: int, game_id: int | None = None) -> bool: + """Whether the challenge ships a dynamic instance environment. + + The env endpoint returns ``null`` for plain challenges and an + ``{"images": [...]}`` object for those with an online environment. + """ + + response = self._request( + "GET", f"/game/{game_id or self.game_id}/challenge/{challenge_id}/env" + ) + if response.status_code >= 400: + return False + try: + payload = response.json() + except ValueError: + return False + return bool(payload) + + # ---- attachments ---- + + def list_files(self, challenge_id: int, game_id: int | None = None) -> list[dict[str, Any]]: + payload = self._request_ok( + "GET", f"/game/{game_id or self.game_id}/challenge/{challenge_id}/file" + ) + if not isinstance(payload, list): + raise Ret2ShellError("challenge file response is not a list") + return [item for item in payload if isinstance(item, dict)] + + def download_file( + self, + challenge_id: int, + folder: str, + file: str, + dest_dir: str | Path, + game_id: int | None = None, + *, + max_bytes: int | None = None, + ) -> Path: + destination = Path(dest_dir) + destination.mkdir(parents=True, exist_ok=True) + existing = [path.name for path in destination.iterdir()] + source = Path(file) + stem = safe_stem(source.stem, fallback="attachment") + suffix = source.suffix[:20] + target = destination / numbered_filename( + stem, suffix or ".bin", existing, fallback="attachment" + ) + response = self._request( + "GET", + f"/game/{game_id or self.game_id}/challenge/{challenge_id}/file", + params={"folder": folder, "file": file}, + stream=True, + ) + if response.status_code >= 400: + raise Ret2ShellError(_error_detail(response)) + written = 0 + try: + with target.open("wb") as handle: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + written += len(chunk) + if max_bytes is not None and written > max_bytes: + raise Ret2ShellError( + f"attachment exceeds {max_bytes} byte limit: {file}" + ) + handle.write(chunk) + except Exception: + target.unlink(missing_ok=True) + raise + return target + + # ---- dynamic instances ---- + + def list_instances(self, game_id: int | None = None) -> list[dict[str, Any]]: + payload = self._request_ok("GET", f"/game/{game_id or self.game_id}/instance") + return payload if isinstance(payload, list) else [] + + def find_instance( + self, challenge_id: int, game_id: int | None = None + ) -> dict[str, Any] | None: + for instance in self.list_instances(game_id): + if instance.get("challenge_id") == challenge_id: + return instance + return None + + def start_instance(self, challenge_id: int, game_id: int | None = None) -> None: + response = self._request( + "POST", f"/game/{game_id or self.game_id}/challenge/{challenge_id}/instance" + ) + if response.status_code >= 400: + raise Ret2ShellError(_error_detail(response)) + + def renew_instance(self, challenge_id: int, game_id: int | None = None) -> None: + response = self._request( + "PATCH", f"/game/{game_id or self.game_id}/challenge/{challenge_id}/instance" + ) + if response.status_code >= 400: + raise Ret2ShellError(_error_detail(response)) + + def destroy_instance(self, challenge_id: int, game_id: int | None = None) -> None: + response = self._request( + "DELETE", f"/game/{game_id or self.game_id}/challenge/{challenge_id}/instance" + ) + if response.status_code >= 400: + raise Ret2ShellError(_error_detail(response)) + + def wait_for_instance( + self, + challenge_id: int, + game_id: int | None = None, + *, + timeout: float = INSTANCE_WAIT_TIMEOUT, + interval: float = INSTANCE_WAIT_INTERVAL, + ) -> dict[str, Any]: + """Poll until the challenge instance reports reachable exposed_ports.""" + + deadline = self._monotonic() + timeout + last: dict[str, Any] | None = None + while True: + instance = self.find_instance(challenge_id, game_id) + if instance is not None: + last = instance + if instance.get("exposed_ports"): + return instance + if self._monotonic() >= deadline: + state = (last or {}).get("state", "unknown") + raise Ret2ShellError( + f"instance for challenge {challenge_id} did not become reachable " + f"within {timeout:.0f}s (last state: {state})" + ) + self._sleep(interval) + + # ---- flag submission (async judged) ---- + + def get_submission( + self, + challenge_id: int, + submission_id: int, + game_id: int | None = None, + ) -> dict[str, Any]: + return self._request_ok( + "GET", + f"/game/{game_id or self.game_id}/challenge/{challenge_id}/submit", + params={"id": submission_id}, + ) + + def submit_flag( + self, + challenge_id: int, + flag: str, + game_id: int | None = None, + *, + check_solved: bool = True, + ) -> dict[str, Any]: + """Submit one flag and poll the async judge for its verdict. + + Preflight refuses to POST when the challenge is already solved (no + platform attempt should be spent) and the local limiter protects the + shared 10-per-5-minutes quota. Returns the final Submission dict + with ``solved`` and ``result`` filled in when the judge answered in + time; ``solved`` stays ``None`` on a poll timeout. + """ + + flag = str(flag).strip() + if not flag: + raise Ret2ShellPreflightError("flag must not be empty") + if check_solved: + status = self.challenge_status(challenge_id, game_id) + if isinstance(status, dict) and status.get("solved"): + raise Ret2ShellPreflightError( + f"challenge {challenge_id} is already solved; no submission was sent" + ) + self._limiter.reserve() + response = self._request( + "POST", + f"/game/{game_id or self.game_id}/challenge/{challenge_id}/submit", + json={"content": flag}, + ) + if response.status_code == 429: + raise Ret2ShellRateLimitError(_error_detail(response)) + if response.status_code >= 400: + raise Ret2ShellError(_error_detail(response)) + submission = response.json() if response.content else {} + submission_id = submission.get("id") + if submission_id is None: + raise Ret2ShellError(f"submit response has no submission id: {submission}") + for _ in range(self.poll_attempts): + if submission.get("solved") is not None: + return submission + self._sleep(self.poll_interval) + submission = self.get_submission(challenge_id, submission_id, game_id) + return submission + + # ---- lifecycle ---- + + def close(self) -> None: + self.session.close() + + def __enter__(self) -> Ret2ShellClient: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + +def primary_tag(challenge: dict[str, Any]) -> str: + tags = challenge.get("tag") + if isinstance(tags, list): + for tag in tags: + if isinstance(tag, dict) and tag.get("name"): + return str(tag["name"]) + if tags and isinstance(tags[0], str): + return str(tags[0]) + return "" + + +def normalize_category(raw: str) -> str: + lowered = raw.strip().lower() + return lowered if lowered in CATEGORIES else "misc" + + +class Ret2ShellAdapter(PlatformAdapter): + """Import ret2shell challenges through the generic platform mechanism.""" + + def __init__( + self, + client: Ret2ShellClient, + *, + game_id: int | None = None, + category_map: dict[str, str] | None = None, + max_attachment_bytes: int | None = None, + ) -> None: + self.client = client + self.game_id = game_id or client.game_id + self.category_map = category_map or {} + self.max_attachment_bytes = max_attachment_bytes + + def fetch_challenges(self) -> list[PlatformChallenge]: + challenges: list[PlatformChallenge] = [] + for raw in self.client.list_challenges(self.game_id): + if not isinstance(raw, dict) or raw.get("id") is None: + continue + # The list endpoint returns content: null; the full statement + # (connection commands, flag format hints) only comes from the + # per-challenge detail endpoint. + content = raw.get("content") + if not content: + try: + content = self.client.get_challenge(int(raw["id"]), self.game_id).get("content") + except Ret2ShellError: + content = None + raw_category = primary_tag(raw) + mapped = self.category_map.get(raw_category, raw_category) + challenges.append( + PlatformChallenge( + external_id=str(raw["id"]), + title=str(raw.get("name", "")), + category=normalize_category(mapped), + description=str(content or ""), + attachment_urls=[], + ) + ) + return challenges + + def download_attachments( + self, + challenge: PlatformChallenge, + dest_dir: str | Path, + ) -> list[Path]: + # ret2shell attachments are listed per challenge via the file API, so + # there are no attachment URLs at fetch time. + destination = Path(dest_dir) + destination.mkdir(parents=True, exist_ok=True) + downloaded: list[Path] = [] + for item in self.client.list_files(int(challenge.external_id), self.game_id): + folder = str(item.get("folder", "static")) + file = item.get("file") + if not file: + continue + downloaded.append( + self.client.download_file( + int(challenge.external_id), + folder, + str(file), + destination, + self.game_id, + max_bytes=self.max_attachment_bytes, + ) + ) + return downloaded diff --git a/backend/platform/ret2shell_mcp.py b/backend/platform/ret2shell_mcp.py new file mode 100644 index 0000000..7f024ce --- /dev/null +++ b/backend/platform/ret2shell_mcp.py @@ -0,0 +1,354 @@ +from __future__ import annotations + +import asyncio +import os +import shutil +import socket +import subprocess +import threading +import time +from typing import Any, Callable +from urllib.parse import urlparse + +from backend.mcp.mcp_server import MCPServer, close_lifespan, create_mcp_server +from backend.platform.ret2shell import Ret2ShellClient + +# ret2shell does not expose direct host:port pairs: each running instance +# carries a ``traffic`` token plus the container ``ports``, and the platform +# web app connects through wsrx (WebSocket Reflector X, XDSEC) links of the +# form ``wss:///api/traffic/?port=``. The backend +# keeps one ``wsrx connect`` subprocess per (challenge, port) alive so that +# Members — running in their own containers on the same docker network — can +# simply connect to ``ipc-app:``. +WSRX_BINARY = os.getenv("IPC_R2S_WSRX_BINARY", "wsrx") +WSRX_BIND_HOST = os.getenv("IPC_R2S_WSRX_BIND_HOST", "0.0.0.0") +WSRX_ENDPOINT_HOST = os.getenv("IPC_R2S_WSRX_ENDPOINT_HOST", "ipc-app") +WSRX_BASE_PORT = int(os.getenv("IPC_R2S_WSRX_BASE_PORT", "20000")) +WSRX_PORT_SPAN = 10000 +WSRX_STARTUP_TIMEOUT = 15.0 +WSRX_STARTUP_INTERVAL = 0.3 + + +def local_wsrx_port(challenge_id: int, index: int = 0) -> int: + """Deterministic local port for a challenge's nth tunneled port.""" + + return WSRX_BASE_PORT + (challenge_id % WSRX_PORT_SPAN) + index + + +def _platform_host(client: Ret2ShellClient) -> str: + return urlparse(client.base_url).netloc + + +def _ws_scheme(client: Ret2ShellClient) -> str: + return "wss" if urlparse(client.base_url).scheme == "https" else "ws" + + +def wsrx_remotes(instance: dict[str, Any], client: Ret2ShellClient) -> list[str]: + """wsrx websocket URLs for one instance. + + Handles both shapes: explicit ``ws://``/``wss://`` entries inside + ``exposed_ports`` (GZCTF-style platform proxy) and the ret2shell shape + where the instance carries a ``traffic`` token plus ``ports``. + """ + + remotes: list[str] = [] + exposed = instance.get("exposed_ports") + if isinstance(exposed, list): + for entry in exposed: + if isinstance(entry, str) and entry.startswith(("ws://", "wss://")): + remotes.append(entry) + traffic = instance.get("traffic") + ports = instance.get("ports") + if traffic and isinstance(ports, list): + scheme = _ws_scheme(client) + host = _platform_host(client) + for port in ports: + remotes.append(f"{scheme}://{host}/api/traffic/{traffic}?port={port}") + return remotes + + +class _Tunnel: + def __init__(self, remote: str, local_port: int, process: Any) -> None: + self.remote = remote + self.local_port = local_port + self.process = process + + @property + def alive(self) -> bool: + return self.process.poll() is None + + +class WsrxTunnelManager: + """Own the ``wsrx connect`` subprocesses, keyed by challenge id. + + ``ensure`` is idempotent: a challenge whose tunnels are still running for + the same set of remotes is reused instead of spawning duplicates. The + spawner/probe are injectable so tests can run without the wsrx binary. + """ + + def __init__( + self, + *, + binary: str = WSRX_BINARY, + bind_host: str = WSRX_BIND_HOST, + endpoint_host: str = WSRX_ENDPOINT_HOST, + spawner: Callable[..., Any] | None = None, + probe: Callable[[int], bool] | None = None, + startup_timeout: float = WSRX_STARTUP_TIMEOUT, + startup_interval: float = WSRX_STARTUP_INTERVAL, + terminator: Callable[[Any], None] | None = None, + ) -> None: + self.binary = binary + self.bind_host = bind_host + self.endpoint_host = endpoint_host + self.startup_timeout = startup_timeout + self.startup_interval = startup_interval + self._spawner = spawner or self._spawn_process + self._probe = probe or self._probe_port + self._terminator = terminator or self._terminate_process + self._lock = threading.Lock() + self._tunnels: dict[int, list[_Tunnel]] = {} + + # ---- default subprocess plumbing ---- + + def _spawn_process(self, remote: str, local_port: int) -> Any: + binary = shutil.which(self.binary) + if binary is None: + raise RuntimeError( + f"wsrx binary {self.binary!r} not found on PATH; cannot tunnel {remote}" + ) + return subprocess.Popen( + [ + binary, + "connect", + "--host", + self.bind_host, + "--port", + str(local_port), + remote, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + start_new_session=True, + ) + + @staticmethod + def _probe_port(local_port: int) -> bool: + try: + with socket.create_connection(("127.0.0.1", local_port), timeout=1.0): + return True + except OSError: + return False + + @staticmethod + def _terminate_process(process: Any) -> None: + try: + process.terminate() + try: + process.wait(timeout=5) + except Exception: + process.kill() + except Exception: + pass + + # ---- public API ---- + + def _alive(self, challenge_id: int) -> list[_Tunnel]: + tunnels = [t for t in self._tunnels.get(challenge_id, []) if t.alive] + self._tunnels[challenge_id] = tunnels + return tunnels + + def ensure(self, challenge_id: int, remotes: list[str]) -> dict[str, Any]: + """Guarantee one live tunnel per remote; return local endpoints. + + Returns ``{"endpoints": [...], "started": [...]}`` where endpoints are + ``:`` strings reachable from Member containers. + Raises RuntimeError when a tunnel cannot be established. + """ + + with self._lock: + existing = self._alive(challenge_id) + needed = list(remotes) + tunnels: list[_Tunnel] = [] + for tunnel in existing: + if tunnel.remote in needed: + needed.remove(tunnel.remote) + tunnels.append(tunnel) + else: + self._terminator(tunnel.process) + started: list[str] = [] + for remote in needed: + local_port = local_wsrx_port(challenge_id, remotes.index(remote)) + process = self._spawner(remote, local_port) + tunnels.append(_Tunnel(remote, local_port, process)) + started.append(remote) + self._tunnels[challenge_id] = tunnels + # Probe outside the lock, on a snapshot, so a slow platform-side + # websocket handshake never blocks other challenges' tool calls. + deadline = time.monotonic() + self.startup_timeout + for tunnel in tunnels: + while time.monotonic() < deadline: + if not tunnel.alive: + raise RuntimeError( + f"wsrx tunnel for {tunnel.remote} exited immediately " + f"(code {tunnel.process.returncode}); is wsrx installed?" + ) + if self._probe(tunnel.local_port): + break + time.sleep(self.startup_interval) + else: + raise RuntimeError( + f"wsrx tunnel port {tunnel.local_port} did not become " + f"reachable within {self.startup_timeout:.0f}s" + ) + return { + "endpoints": [ + f"{self.endpoint_host}:{tunnel.local_port}" + for tunnel in sorted(tunnels, key=lambda t: t.local_port) + ], + "started": started, + } + + def stop(self, challenge_id: int) -> list[int]: + """Terminate every tunnel subprocess of one challenge.""" + + with self._lock: + tunnels = self._tunnels.pop(challenge_id, []) + for tunnel in tunnels: + self._terminator(tunnel.process) + return [tunnel.local_port for tunnel in tunnels] + + def endpoints(self, challenge_id: int) -> list[str]: + with self._lock: + tunnels = self._alive(challenge_id) + return [ + f"{self.endpoint_host}:{tunnel.local_port}" + for tunnel in sorted(tunnels, key=lambda t: t.local_port) + ] + + def stop_all(self) -> None: + with self._lock: + challenge_ids = list(self._tunnels) + for challenge_id in challenge_ids: + self.stop(challenge_id) + + +def build_ret2shell_mcp( + client: Ret2ShellClient, *, tunnel_manager: WsrxTunnelManager | None = None +) -> MCPServer: + """Expose ret2shell dynamic-instance control to Members. + + Credentials stay in the backend process; Members only ever see instance + endpoints and challenge state. Flag submission is deliberately NOT + exposed here — the shared 10-per-5-minutes quota is spent only by the + operator through scripts/ret2shell_submit.py. + """ + + manager = tunnel_manager or WsrxTunnelManager() + + server = create_mcp_server( + name="ret2shell", + description=( + "ret2shell competition platform: dynamic challenge instances and " + "solve state" + ), + lifespan=close_lifespan(client.close), + ) + + def _direct_endpoints(instance: dict[str, Any]) -> Any: + # Direct host:port style exposure (no wsrx needed): pass through. + exposed = instance.get("exposed_ports") + if isinstance(exposed, list) and exposed and not wsrx_remotes(instance, client): + return exposed + return None + + async def _resolve_endpoints(challenge_id: int, instance: dict[str, Any]) -> dict[str, Any]: + remotes = wsrx_remotes(instance, client) + if not remotes: + return {"endpoints": _direct_endpoints(instance)} + result = await asyncio.to_thread(manager.ensure, challenge_id, remotes) + return { + "endpoints": result["endpoints"], + "wsrx_remotes": remotes, + "tunnels_started": result["started"], + } + + @server.tool( + name="instance_start", + description=( + "Start (or reuse) the dynamic instance for one challenge and wait " + "until its endpoints are reachable. wsrx-proxied instances are " + "tunneled automatically: connect to the returned local " + "host:port pairs with netcat/pwntools. Requires the challenge id." + ), + ) + async def instance_start(challenge_id: int) -> dict[str, Any]: + existing = await asyncio.to_thread(client.find_instance, challenge_id) + if existing is None: + await asyncio.to_thread(client.start_instance, challenge_id) + instance = await asyncio.to_thread(client.wait_for_instance, challenge_id) + resolved = await _resolve_endpoints(challenge_id, instance) + return { + "challenge_id": challenge_id, + "state": instance.get("state"), + **resolved, + "renew_count": instance.get("renew_count"), + } + + @server.tool( + name="instance_status", + description=( + "Report the current instance state and endpoints for one challenge, " + "or null when no instance is running." + ), + ) + async def instance_status(challenge_id: int) -> dict[str, Any]: + instance = await asyncio.to_thread(client.find_instance, challenge_id) + if instance is None: + return {"challenge_id": challenge_id, "running": False} + remotes = wsrx_remotes(instance, client) + if remotes: + endpoints = await asyncio.to_thread(manager.endpoints, challenge_id) + resolved = {"endpoints": endpoints, "wsrx_remotes": remotes} + else: + resolved = {"endpoints": _direct_endpoints(instance)} + return { + "challenge_id": challenge_id, + "running": instance.get("state") == "Running", + "state": instance.get("state"), + **resolved, + "renew_count": instance.get("renew_count"), + } + + @server.tool( + name="instance_renew", + description="Extend the lifetime of the running instance for one challenge.", + ) + async def instance_renew(challenge_id: int) -> dict[str, Any]: + await asyncio.to_thread(client.renew_instance, challenge_id) + return {"challenge_id": challenge_id, "renewed": True} + + @server.tool( + name="instance_stop", + description="Destroy the running instance for one challenge.", + ) + async def instance_stop(challenge_id: int) -> dict[str, Any]: + torn_down = await asyncio.to_thread(manager.stop, challenge_id) + await asyncio.to_thread(client.destroy_instance, challenge_id) + return {"challenge_id": challenge_id, "stopped": True, "tunnels_closed": torn_down} + + @server.tool( + name="challenge_status", + description=( + "Report whether the team already solved one challenge and how many " + "teams solved it overall. Check before planning around a challenge." + ), + ) + async def challenge_status(challenge_id: int) -> dict[str, Any]: + status = await asyncio.to_thread(client.challenge_status, challenge_id) + if not isinstance(status, dict): + return {"challenge_id": challenge_id, "error": "unexpected status payload"} + return {"challenge_id": challenge_id, **status} + + return server diff --git a/backend/sandbox/container_pool.py b/backend/sandbox/container_pool.py index 23cfd07..bab2124 100644 --- a/backend/sandbox/container_pool.py +++ b/backend/sandbox/container_pool.py @@ -7,7 +7,6 @@ from backend.sandbox.resource_limiter import TaskSlotLimiter from backend.sandbox.sandbox import LocalSandbox, Sandbox -from backend.sandbox.webui_proxy import webui_proxy_manager _SAFE_SEGMENT = re.compile(r"[^A-Za-z0-9_.-]+") @@ -114,15 +113,6 @@ def _new_task(self, project_id: str, env: dict[str, str] | None): attachments_dir=self.workspace_root / project_id / "attachments", ) - def stop_member(self, project_id: str, member: str) -> None: - """Drop a Member's view. Does NOT stop the shared task container.""" - key = self._key(project_id, member) - with self._lock: - sb = self._sandboxes.pop(key, None) - if sb is not None: - sb.stop() - webui_proxy_manager.close_member(project_id, member) - def stop_project(self, project_id: str) -> None: with self._lock: keys = [k for k in self._sandboxes if k[0] == project_id] diff --git a/backend/sandbox/resource_limiter.py b/backend/sandbox/resource_limiter.py index e9ca150..cec7d10 100644 --- a/backend/sandbox/resource_limiter.py +++ b/backend/sandbox/resource_limiter.py @@ -43,12 +43,3 @@ def release(self, project_id: str) -> None: def active_tasks(self) -> list[str]: with self._lock: return sorted(self._active) - - @property - def active_count(self) -> int: - with self._lock: - return len(self._active) - - def reset(self) -> None: - with self._lock: - self._active.clear() diff --git a/backend/sandbox/sandbox.py b/backend/sandbox/sandbox.py index e0a0c93..10e2ce2 100644 --- a/backend/sandbox/sandbox.py +++ b/backend/sandbox/sandbox.py @@ -1,6 +1,8 @@ from __future__ import annotations import os +import re +import shutil import subprocess import threading from dataclasses import dataclass @@ -10,6 +12,56 @@ from backend.sandbox.webui_proxy import webui_proxy_manager +def _git_bash_executable() -> Path | None: + """Return a Git Bash executable for Windows local sandboxes, if present. + + Members are prompted with POSIX shell commands (``find``, ``head``, pipes, + etc.). Running those commands through ``cmd.exe`` makes the local backend + behave differently from the Docker backend and causes avoidable tool + failures. Git for Windows provides a small, self-contained Bash that is + suitable for development-mode sandboxes. + """ + if os.name != "nt": + return None + + candidates: list[Path] = [] + configured = os.environ.get("IPC_LOCAL_BASH", "").strip() + if configured: + candidates.append(Path(configured)) + + git = shutil.which("git") + if git: + git_path = Path(git).resolve() + # Typical layout: \\cmd\\git.exe -> \\bin\\bash.exe. + if len(git_path.parents) >= 2: + candidates.append(git_path.parents[1] / "bin" / "bash.exe") + + for base in (os.environ.get("ProgramFiles"), os.environ.get("ProgramW6432")): + if base: + candidates.append(Path(base) / "Git" / "bin" / "bash.exe") + + for candidate in candidates: + if candidate.is_file(): + return candidate + return None + + +def _windows_paths_for_bash(command: str) -> str: + """Convert unquoted ``D:\\path`` forms produced by Members to ``/d/path``. + + This intentionally targets only absolute drive paths. It leaves normal + command text untouched and avoids rewriting arbitrary backslash escapes. + """ + def replace_path(match: re.Match[str]) -> str: + return "/" + match.group(1).lower() + "/" + match.group(2).replace("\\", "/") + + return re.sub( + r"(?]*)", + replace_path, + command, + ) + + @dataclass(slots=True) class ExecResult: exit_code: int @@ -57,13 +109,23 @@ def exec(self, command: str, timeout: int = 60) -> ExecResult: if not self._started: self.start() full_env = {**os.environ, **self.env} + full_env.setdefault("PYTHONIOENCODING", "utf-8") + full_env.setdefault("PYTHONUTF8", "1") + bash = _git_bash_executable() + args: str | list[str] = command + use_shell = True + if bash is not None: + args = [str(bash), "-lc", _windows_paths_for_bash(command)] + use_shell = False try: proc = subprocess.run( - command, - shell=True, + args, + shell=use_shell, cwd=str(self.workspace), capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=timeout, env=full_env, ) diff --git a/backend/tools/member_tools.txt b/backend/tools/member_tools.txt index 10a3d38..5d62415 100644 --- a/backend/tools/member_tools.txt +++ b/backend/tools/member_tools.txt @@ -179,3 +179,16 @@ Common first moves by challenge shape libraries; use RsaCtfTool for weak RSA. - Stego/forensics: exiftool, binwalk, zsteg/steghide/pngcheck, OCR, audio/video extraction. + + +ret2shell platform (competition) +-------------------------------- +- ret2shell MCP: dynamic instance control for challenges that need a remote + target. Tools: instance_start (starts the challenge instance and waits for + its endpoints), instance_status, instance_renew, instance_stop, + challenge_status (already-solved check). Credentials stay in the backend. +- For pwn-style challenges with a remote endpoint, call instance_start first + and attack the returned address/port; renew the instance if it is about to + expire while you are still working. +- Flag submission is operator-only (shared 10-per-5-minutes quota). When you + find the flag, emit the flag action; do not attempt to submit it yourself. diff --git a/docker-compose.yml b/docker-compose.yml index 6cfa725..e5375fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,6 +98,13 @@ services: - IPC_CLAUDE_RUNNER_URL=${IPC_CLAUDE_RUNNER_URL:-http://ipc-claude-runner:8600} - IPC_RUNNER_TOKEN=${IPC_RUNNER_TOKEN:-} - IPC_DATABASE_URL=${IPC_DATABASE_URL:-postgresql://ipc:ipc@ipc-postgres:5432/ipc} + # ret2shell platform credentials; when USERNAME (or TOKEN) is set the + # backend registers the ret2shell MCP for every Member. + - IPC_R2S_BASE_URL=${IPC_R2S_BASE_URL:-} + - IPC_R2S_GAME_ID=${IPC_R2S_GAME_ID:-} + - IPC_R2S_USERNAME=${IPC_R2S_USERNAME:-} + - IPC_R2S_PASSWORD=${IPC_R2S_PASSWORD:-} + - IPC_R2S_TOKEN=${IPC_R2S_TOKEN:-} volumes: # PostgreSQL owns runtime facts. ./data holds deployment-owned secrets, # IPC conversation files and the shared Artifact tree (workspaces, diff --git a/docs/ret2shell.md b/docs/ret2shell.md new file mode 100644 index 0000000..f7d4bfa --- /dev/null +++ b/docs/ret2shell.md @@ -0,0 +1,87 @@ +# ret2shell 平台适配 + +适配 [ret2shell](https://github.com/ret2shell/ret2shell)(Rust/axum 开源 CTF 平台)。 +**已对真实平台验证通过**:西电 `https://ctf.xidian.edu.cn`(MoeCTF 2026,game 37,125 题), +端到端冒烟(导入 → 解题 → flag → writeup)已跑通。 + +## 配置(环境变量) + +| 变量 | 默认 | 说明 | +|---|---|---| +| `IPC_R2S_BASE_URL` | `https://ctf.xidian.edu.cn` | 平台根地址 | +| `IPC_R2S_GAME_ID` | `37` | 比赛 id(数字) | +| `IPC_R2S_USERNAME` / `IPC_R2S_PASSWORD` | 空 | 参赛账号(登录换 JWT Bearer token) | +| `IPC_R2S_TOKEN` | 空 | 已有的 Bearer token(可替代账密) | + +Docker Compose 部署时把这些写进项目根目录的 `.env`(compose 自动读取并注入 +ipc-app 容器;注意 `.env` 不能带 UTF-8 BOM,PowerShell `Set-Content` 后需检查)。 +**认证细节**:登录需要验证码时客户端自动走 `GET /api/account/captcha/cli` 的 +PoW(难度 4:答案以 challenge 随机串开头、其 SHA-256 十六进制前 4 位为 0), +无需人工介入;token 在响应头 `Set-Token` 返回并自动滚动更新。 +**密码连续错 5 次账号冻结 30 分钟**。 + +## 使用流程 + +```bash +export PYTHONPATH=. +export IPC_R2S_USERNAME=... IPC_R2S_PASSWORD=... + +# 1) 只读冒烟:登录 + 比赛状态 + 题目列表 +python scripts/ret2shell_platform_check.py + +# 2) 拉取题目清单(含分值/分类/已解状态)到 platform_targets/ret2shell.yaml +python scripts/fetch_ret2shell_targets.py + +# 3) 批量导入题目为项目。附件下载与数据库写入已分离(两阶段), +# 但大括量仍建议分批(脚本每批 15 题、自动重试): +docker cp scripts/import_ret2shell_batch.py ipc-app:/tmp/ +docker exec ipc-app python /tmp/import_batch.py +# 或直接对运行中的后端调 POST /api/platform/import(mapping 见下)。 + +# 4) Member 在沙箱里通过 ret2shell MCP 管理动态靶机(pwn 题拿 host:port) + +# 5) 提交 flag:默认 dry-run,--no-dry-run 才真正提交 +python scripts/ret2shell_submit.py --challenge --flag 'flag{...}' +``` + +**运维代理(IPC 对话)路径**:ops agent 的平台工作流也支持 +`challenges.platform: "ret2shell"`(backend/ops/service.py 的 `_adapter` 分支), +在 IPC 对话里提交含 `"challenges": {"platform": "ret2shell", "game_id": 37}` +的 workflow 描述并确认后,agent 可自行拉题导入。凭据始终只从环境变量读取。 + +**中文分类映射**(MoeCTF 2026 实测):Web 安全与渗透测试→web、 +现代密码学→crypto、二进制漏洞审计 / Python 沙箱逃逸→pwn、 +软件逆向工程→reverse、大语言模型应用安全→ai、其余(取证杂项/从此开始/ +开发运维基础/策略博弈/美工设计)→misc。 + +## 关键契约(与 GZCTF 的差异) + +- **异步判题**:`POST .../submit` 返回 `solved: null` 的 Submission,客户端轮询 + `GET .../submit?id=` 直到 `solved != null`(默认 1s × 7 次)。 +- **提交限额**:每账号 5 分钟 10 次(HTTP 429)。客户端内置同额度限速器, + 提交前还会做已解出 preflight,避免浪费平台尝试次数。 +- **动态靶机**:`POST/PATCH/DELETE .../instance` 启动/续期/销毁, + `GET /api/game/{gid}/instance` 轮询 `exposed_ports` 拿实际连接地址。 +- 列表接口返回 `[items, total]` 二元组;错误响应是纯文本 body。 + +## Member 侧能力(ret2shell MCP) + +配置了 `IPC_R2S_USERNAME`(或 `IPC_R2S_TOKEN`)后,后端自动向所有 Member 注册 +`ret2shell` MCP(凭据留在后端,不进沙箱): + +- `instance_start(challenge_id)`:启动并等待靶机可达,返回 endpoints +- `instance_status` / `instance_renew` / `instance_stop` +- `challenge_status(challenge_id)`:已解状态与全站解出数 + +**flag 提交不暴露给 Member**——共享限额只由操作者经 `scripts/ret2shell_submit.py` +消耗,避免 agent 误操作烧掉提交次数。 + +## 已知限制与运维注意 + +- **RAM 黑板**:解题图谱/项目列表在内存里,`docker compose restart` 会清空 + 项目(附件文件保留在 `projects/`,重新导入即恢复,external_id 对应关系不变)。 + 比赛期间尽量避免重启容器。 +- **SQLite 锁**:memdb 不支持 WAL,长事务与 UI 轮询互斥。导入已改为 + 两阶段(先下载后写库),但请勿在导入进行时并行发起第二个导入。 +- **附件去重**:导入脚本超时重试可能产生重复项目(同一 external_id 多个 + project),可按 external_id 保留最早一个、DELETE 其余。 diff --git a/frontend/index.html b/frontend/index.html index 34d9485..a17e33e 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -962,7 +962,13 @@ }, currentElapsed(){ return this.projectElapsed(this.current?.project); }, - async loadProjects(){ this.projects = await this.api("GET","/projects"); }, + async loadProjects(){ + const list = await this.api("GET","/projects"); + // Running projects first, then queued/created, completed last; newest id first within a group. + const rank = (s) => (s === 'running' ? 0 : (s === 'completed' ? 2 : 1)); + this.projects = [...list].sort((a, b) => + (rank(a.status) - rank(b.status)) || (a.id < b.id ? 1 : -1)); + }, async loadBroadcasts(){ this.broadcasts = await this.api("GET","/broadcasts"); }, async loadProject(id){ this.current = await this.api("GET","/projects/"+id); }, async openProject(id){ diff --git a/scripts/fetch_gzctf_targets.py b/scripts/fetch_gzctf_targets.py new file mode 100644 index 0000000..4badb36 --- /dev/null +++ b/scripts/fetch_gzctf_targets.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import argparse +import os +from datetime import UTC, datetime +from pathlib import Path + +import yaml + +from backend.platform.gzctf import GZCTFClient + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +TARGETS_PATH = PROJECT_ROOT / "platform_targets" / "gzctf.yaml" + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Fetch visible GZCTF challenges for one game into a YAML target list." + ) + parser.add_argument("--base-url", default=os.getenv("IPC_GZ_BASE_URL", "")) + parser.add_argument("--username", default=os.getenv("IPC_GZ_USERNAME", "")) + parser.add_argument("--password", default=os.getenv("IPC_GZ_PASSWORD", "")) + parser.add_argument("--game", type=int, required=True) + parser.add_argument("--output", type=Path, default=TARGETS_PATH) + args = parser.parse_args() + + challenges: list[dict] = [] + with GZCTFClient( + base_url=args.base_url, + username=args.username, + password=args.password, + ) as client: + client.login() + game = client.get_game(args.game) + scoreboard = client.get_scoreboard(args.game) + for category, items in (scoreboard.get("challenges") or {}).items(): + for item in items: + challenges.append( + { + "id": item["id"], + "title": item.get("title", ""), + "category": category, + "score": item.get("score", 0), + "solved": item.get("solved", 0), + } + ) + challenges.sort(key=lambda item: (item["category"], item["id"])) + + data = { + "updated_at": datetime.now(UTC).isoformat(), + "game": { + "id": game.get("id"), + "title": game.get("title"), + "team_name": game.get("teamName"), + "status": game.get("status"), + "start": game.get("start"), + "end": game.get("end"), + }, + "challenge_count": len(challenges), + "challenges": challenges, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + yaml.safe_dump(data, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + print(f"wrote {len(challenges)} challenges to {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/fetch_ret2shell_targets.py b/scripts/fetch_ret2shell_targets.py new file mode 100644 index 0000000..805a8a9 --- /dev/null +++ b/scripts/fetch_ret2shell_targets.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import argparse +import os +from datetime import UTC, datetime +from pathlib import Path + +import yaml + +from backend.platform.ret2shell import Ret2ShellClient + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +TARGETS_PATH = PROJECT_ROOT / "platform_targets" / "ret2shell.yaml" + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Fetch ret2shell challenges for one game into a YAML target list." + ) + parser.add_argument("--base-url", default=os.getenv("IPC_R2S_BASE_URL", "")) + parser.add_argument("--game", type=int, default=None) + parser.add_argument("--username", default=os.getenv("IPC_R2S_USERNAME", "")) + parser.add_argument("--password", default=os.getenv("IPC_R2S_PASSWORD", "")) + parser.add_argument("--token", default=os.getenv("IPC_R2S_TOKEN", "")) + parser.add_argument("--output", type=Path, default=TARGETS_PATH) + args = parser.parse_args() + + challenges: list[dict] = [] + with Ret2ShellClient( + base_url=args.base_url, + game_id=args.game, + username=args.username, + password=args.password, + token=args.token, + ) as client: + game = client.get_game() + for raw in client.list_challenges(): + status = client.challenge_status(raw["id"]) + challenges.append( + { + "id": raw.get("id"), + "name": raw.get("name", ""), + "category": (raw.get("tag") or [{}])[0].get("name", "misc") + if isinstance(raw.get("tag"), list) and raw.get("tag") + else "misc", + "score": raw.get("score", 0), + "solved": bool(status.get("solved")) if isinstance(status, dict) else False, + "total_solves": status.get("solves") if isinstance(status, dict) else None, + } + ) + challenges.sort(key=lambda item: (str(item["category"]), item["id"] or 0)) + + data = { + "updated_at": datetime.now(UTC).isoformat(), + "game": { + "id": game.get("id"), + "name": game.get("name", ""), + "start_at": game.get("start_at"), + "end_at": game.get("end_at"), + }, + "challenge_count": len(challenges), + "challenges": challenges, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + yaml.safe_dump(data, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + print(f"wrote {len(challenges)} challenges to {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gzctf_platform_check.py b/scripts/gzctf_platform_check.py new file mode 100644 index 0000000..deca0da --- /dev/null +++ b/scripts/gzctf_platform_check.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import argparse +import os + +from backend.platform.gzctf import GZCTFClient + +DEFAULT_BASE_URL = "" + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check GZCTF login, profile and game state." + ) + parser.add_argument("--base-url", default=os.getenv("IPC_GZ_BASE_URL", DEFAULT_BASE_URL)) + parser.add_argument("--username", default=os.getenv("IPC_GZ_USERNAME", "")) + parser.add_argument("--password", default=os.getenv("IPC_GZ_PASSWORD", "")) + parser.add_argument("--game", type=int, default=2) + args = parser.parse_args() + + with GZCTFClient( + base_url=args.base_url, + username=args.username, + password=args.password, + ) as client: + client.login() + profile = client.get_profile() + print("profile:", profile.get("userName"), profile.get("email")) + + game = client.get_game(args.game) + print( + "game:", + game.get("id"), + game.get("title"), + "status=", game.get("status"), + "team=", game.get("teamName"), + ) + + check = client.get_game_check(args.game) + print("check:", check) + + scoreboard = client.get_scoreboard(args.game) + challenges = scoreboard.get("challenges", {}) + challenge_count = sum(len(items) for items in challenges.values()) + print("scoreboard challenge_count:", challenge_count) + for category, items in challenges.items(): + print(category, [(item.get("id"), item.get("title")) for item in items][:10]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ret2shell_platform_check.py b/scripts/ret2shell_platform_check.py new file mode 100644 index 0000000..5b26b70 --- /dev/null +++ b/scripts/ret2shell_platform_check.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import argparse +import os + +from backend.platform.ret2shell import Ret2ShellClient + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Read-only ret2shell smoke check: login, game, challenges." + ) + parser.add_argument("--base-url", default=os.getenv("IPC_R2S_BASE_URL", "")) + parser.add_argument("--game", type=int, default=None) + parser.add_argument("--username", default=os.getenv("IPC_R2S_USERNAME", "")) + parser.add_argument("--password", default=os.getenv("IPC_R2S_PASSWORD", "")) + parser.add_argument("--token", default=os.getenv("IPC_R2S_TOKEN", "")) + args = parser.parse_args() + + with Ret2ShellClient( + base_url=args.base_url, + game_id=args.game, + username=args.username, + password=args.password, + token=args.token, + ) as client: + print("ping:", client.ping()) + profile = client.get_profile() + print("profile:", profile.get("account"), profile.get("nickname")) + game = client.get_game() + print( + "game:", + game.get("id"), + game.get("name"), + "start=", game.get("start_at"), + "end=", game.get("end_at"), + ) + challenges = client.list_challenges() + print("challenge_count:", len(challenges)) + for raw in challenges: + print( + raw.get("id"), + raw.get("name"), + "score=", raw.get("score"), + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ret2shell_submit.py b/scripts/ret2shell_submit.py new file mode 100644 index 0000000..051060d --- /dev/null +++ b/scripts/ret2shell_submit.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import argparse +import os + +from backend.platform.ret2shell import ( + Ret2ShellClient, + Ret2ShellError, + Ret2ShellPreflightError, + Ret2ShellRateLimitError, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Submit one flag to a ret2shell challenge. Dry-run by default so a " + "mistyped command never spends a platform attempt; pass " + "--no-dry-run to actually submit." + ) + ) + parser.add_argument("--base-url", default=os.getenv("IPC_R2S_BASE_URL", "")) + parser.add_argument("--game", type=int, default=None) + parser.add_argument("--username", default=os.getenv("IPC_R2S_USERNAME", "")) + parser.add_argument("--password", default=os.getenv("IPC_R2S_PASSWORD", "")) + parser.add_argument("--token", default=os.getenv("IPC_R2S_TOKEN", "")) + parser.add_argument("--challenge", type=int, required=True) + parser.add_argument("--flag", required=True) + parser.add_argument( + "--no-dry-run", + action="store_true", + help="Actually POST the flag (the shared quota is 10 submissions / 5 minutes).", + ) + args = parser.parse_args() + + with Ret2ShellClient( + base_url=args.base_url, + game_id=args.game, + username=args.username, + password=args.password, + token=args.token, + ) as client: + status = client.challenge_status(args.challenge) + print("challenge status before submit:", status) + if isinstance(status, dict) and status.get("solved"): + print("already solved; nothing to submit") + return 0 + if not args.no_dry_run: + print(f"dry-run ok: would submit {args.flag!r} to challenge {args.challenge}") + return 0 + try: + submission = client.submit_flag( + args.challenge, args.flag, check_solved=False + ) + except Ret2ShellPreflightError as exc: + print(f"preflight refused: {exc}") + return 1 + except Ret2ShellRateLimitError as exc: + print(f"rate limited: {exc}") + return 1 + except Ret2ShellError as exc: + print(f"submission failed: {exc}") + return 1 + print( + "submission", + submission.get("id"), + "solved=", submission.get("solved"), + "result=", submission.get("result"), + ) + return 0 if submission.get("solved") else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conftest.py b/tests/conftest.py index d9380e3..71e0a8c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,7 +64,12 @@ def pytest_collection_modifyitems(config, items) -> None: ) has_database = bool(os.environ.get("IPC_TEST_DATABASE_URL", "").strip()) for item in items: - if Path(str(item.fspath)).name not in POSTGRES_TEST_MODULES: + # Modules in POSTGRES_TEST_MODULES are entirely DB-backed; individual + # tests elsewhere may opt in with an explicit @pytest.mark.postgres. + if ( + Path(str(item.fspath)).name not in POSTGRES_TEST_MODULES + and item.get_closest_marker("postgres") is None + ): continue item.add_marker("postgres") if not has_database: diff --git a/tests/test_blackboard.py b/tests/test_blackboard.py index fd0e2cf..da3dd2f 100644 --- a/tests/test_blackboard.py +++ b/tests/test_blackboard.py @@ -62,6 +62,26 @@ def test_hyperedge_multiple_sources(db): assert set(intent.from_) == {f1.id, f2.id} +def test_create_fact_deduplicates_reworded_source_finding(db): + with db.connect() as conn: + pid = graph_store.create_project(conn, "T", "o", "g", "pwn") + first = node_store.create_fact( + conn, + pid, + "Nday candidate services/camera/foo.cpp lines 40-48; " + "introducing commit a876352a6fb119ba2e81159ec106359a99b56461.", + ) + second = node_store.create_fact( + conn, + pid, + "Confirmed candidate services/camera/foo.cpp:40-48; " + "introducing commit a876352a6fb119ba2e81159ec106359a99b56461; " + "track mapping pending.", + ) + assert second.id == first.id + assert len(node_store.list_facts(conn, pid)) == 3 + + def test_expire_workers_releases_stale_claims(db): with db.connect() as conn: pid = graph_store.create_project(conn, "T", "o", "g", "misc") diff --git a/tests/test_gzctf.py b/tests/test_gzctf.py new file mode 100644 index 0000000..c5f944f --- /dev/null +++ b/tests/test_gzctf.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import pytest + +from backend.platform.gzctf import ( + GZCTFClient, + GZCTFLoginError, + GZCTFPreflightError, + validate_submission, +) + + +class FakeResponse: + def __init__(self, status_code=200, payload=None): + self.status_code = status_code + self._payload = payload + + def json(self): + return self._payload + + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + + +class FakeSession: + def __init__(self): + self.calls = [] + self.closed = False + + def post(self, url, **kwargs): + self.calls.append(("post", url, kwargs)) + if url.endswith("/api/Account/LogIn"): + if kwargs["json"]["password"] != "correct": + return FakeResponse( + 401, + {"title": "Wrong username or password", "status": 401}, + ) + return FakeResponse(200, None) + if url.endswith("/Challenges/7"): + return FakeResponse(200, {"id": 1234, "isTrackComplete": True}) + return FakeResponse(200, {"ok": True}) + + def get(self, url, **kwargs): + self.calls.append(("get", url, kwargs)) + if url.endswith("/api/Account/Profile"): + return FakeResponse(200, {"userName": "tester"}) + return FakeResponse(200, {"ok": True}) + + def close(self): + self.closed = True + + +def test_gzctf_login_failure_raises(monkeypatch): + monkeypatch.setattr("backend.platform.gzctf.requests.Session", FakeSession) + client = GZCTFClient( + base_url="https://ctf.test", + username="user", + password="bad", + ) + with pytest.raises(GZCTFLoginError, match="Wrong username"): + client.login() + client.close() + + +def test_gzctf_authenticated_flow(monkeypatch): + monkeypatch.setattr("backend.platform.gzctf.requests.Session", FakeSession) + client = GZCTFClient( + base_url="https://ctf.test", + username="user", + password="correct", + ) + profile = client.get_profile() + details = client.get_game_details(2) + check = client.get_game_check(2) + submission = client.submit_flag( + 2, + 7, + "flag{demo}", + level=2, + track_id="track-1", + ) + status = client.get_submission_status(2, 7, submission["id"]) + + assert profile["userName"] == "tester" + assert details == {"ok": True} + assert check == {"ok": True} + assert submission["id"] == 1234 + assert status == {"ok": True} + urls = [call[1] for call in client.session.calls] + assert any(url.endswith("/api/Account/LogIn") for url in urls) + assert any(url.endswith("/api/Game/2/Check") for url in urls) + assert any(url.endswith("/api/Game/2/Challenges/7") for url in urls) + assert any(url.endswith("/api/Game/2/Challenges/7/Status/1234") for url in urls) + submit_call = next( + call + for call in client.session.calls + if call[1].endswith("/api/Game/2/Challenges/7") + ) + assert submit_call[2]["json"] == { + "flag": "flag{demo}", + "level": 2, + "trackId": "track-1", + } + client.close() + + +def _challenge_state(*, multi=True, current_level=1, cooldown=None): + tracks = [ + { + "trackId": "track-1", + "isEnabled": True, + "currentLevel": current_level, + "levels": [ + {"level": 1, "status": "Unsolved"}, + {"level": 2, "status": "Locked"}, + {"level": 3, "status": "Locked"}, + ], + } + ] + if multi: + tracks.append( + { + "trackId": "track-2", + "isEnabled": True, + "currentLevel": current_level, + "levels": [{"level": 1, "status": "Unsolved"}], + } + ) + return {"id": 143, "challengeLock": False, "tracks": tracks, "cooldown": cooldown or {}} + + +def test_gzctf_preflight_requires_track_and_strips_answer(): + with pytest.raises(GZCTFPreflightError, match="multiple enabled tracks"): + validate_submission( + _challenge_state(), + level=1, + answer="flag{demo}", + ) + with pytest.raises(GZCTFPreflightError, match="answer must not be empty"): + validate_submission( + _challenge_state(multi=False), + level=1, + answer=" ", + ) + decision = validate_submission( + _challenge_state(), + level=1, + answer=" flag{demo} ", + track_id="track-1", + evidence={ + "status": "verified", + "answers": {"1": "flag{demo}"}, + }, + ) + assert decision["answer"] == "flag{demo}" + assert decision["track_id"] == "track-1" + + +def test_gzctf_preflight_refuses_locked_and_cooldown_levels(): + with pytest.raises(GZCTFPreflightError, match="locked"): + validate_submission( + _challenge_state(multi=False), + level=2, + answer="flag{demo}", + ) + with pytest.raises(GZCTFPreflightError, match="wrong-answer limit"): + validate_submission( + _challenge_state( + multi=False, + cooldown={"wrongCount": 10, "wrongAnswerLimit": 10}, + ), + level=1, + answer="flag{demo}", + ) + + +def test_gzctf_preflight_accepts_unlocked_status_object(): + challenge = _challenge_state(multi=False) + challenge["challengeLock"] = { + "isLocked": False, + "unlocksAt": None, + "triggerAnswer": None, + } + decision = validate_submission( + challenge, + level=1, + answer="flag{demo}", + evidence={ + "status": "verified", + "answers": {"1": "flag{demo}"}, + }, + ) + assert decision["ok"] is True + + +def test_gzctf_preflight_refuses_locked_status_object(): + challenge = _challenge_state(multi=False) + challenge["challengeLock"] = {"isLocked": True, "reason": "banned"} + with pytest.raises(GZCTFPreflightError, match="currently locked"): + validate_submission( + challenge, + level=1, + answer="services/foo.c:1", + ) diff --git a/tests/test_members.py b/tests/test_members.py index d6b70c5..befbe22 100644 --- a/tests/test_members.py +++ b/tests/test_members.py @@ -489,6 +489,36 @@ def test_member_context_uses_sandbox_visible_attachment_paths(deps): ] +def test_member_context_includes_challenge_description_and_external_id(deps): + db, d, reports, flags = deps + origin = "https://ctf.example/challenges\n\nConnect: nc 10.0.0.1 9999\nFlag format: flag{...}" + with db.connect() as conn: + pid = graph_store.create_project( + conn, "T", origin, "get flag", "pwn", external_id="chal-42" + ) + intent = edge_store.create_intent(conn, pid, ["origin"], "pwn the service", "diamond") + member = create_member(MemberConfig(name="jade", api_format="mock"), d) + + context = member._build_context(pid, intent.id, "pwn", 1, False, False) + + assert context["external_id"] == "chal-42" + assert context["challenge_description"] == origin + assert "nc 10.0.0.1 9999" in context["challenge_description"] + note = next((n for n in context["runtime_notes"] if "challenge_description" in n), "") + assert "ret2shell" in note and "external_id" in note + + +def test_member_context_defaults_without_external_id(deps): + db, d, reports, flags = deps + pid, iid = _project(db) + member = create_member(MemberConfig(name="jade", api_format="mock"), d) + + context = member._build_context(pid, iid, "web", 1, False, False) + + assert context["external_id"] is None + assert context["challenge_description"] == "origin" + + def test_member_report_defaults_to_low_when_unspecified(deps): db, d, reports, flags = deps pid, iid = _project(db) diff --git a/tests/test_ops_agent.py b/tests/test_ops_agent.py index d1c434e..7be0458 100644 --- a/tests/test_ops_agent.py +++ b/tests/test_ops_agent.py @@ -1162,7 +1162,7 @@ def fake_post(url, **kwargs): assert calls[0][0] == "https://api.openai.com/v1/responses" body = calls[0][1]["json"] assert "input" in body and "messages" not in body - assert body["max_output_tokens"] == 4096 + assert body["max_output_tokens"] == 16384 assert body["reasoning"] == {"effort": "none"} assert body["text"]["format"]["type"] == "json_schema" assert "temperature" not in body @@ -1209,7 +1209,7 @@ def fake_post(url, **kwargs): assert calls[0][0].endswith("/v1/responses") assert calls[1][0].endswith("/v1/chat/completions") - assert calls[1][1]["max_completion_tokens"] == 4096 + assert calls[1][1]["max_completion_tokens"] == 16384 assert calls[1][1]["reasoning_effort"] == "none" assert calls[2][0].endswith("/v1/chat/completions") assert sum(url.endswith("/responses") for url, _ in calls) == 1 diff --git a/tests/test_orchestration.py b/tests/test_orchestration.py index 7deb6e7..96ecd09 100644 --- a/tests/test_orchestration.py +++ b/tests/test_orchestration.py @@ -692,3 +692,60 @@ def test_non_retryable_member_failure_is_not_requeued(state): ) assert not any(entry["event"] == "member_task_retry_scheduled" for entry in entries) orch.shutdown() + + +def test_repeated_stalls_defer_original_intent_and_dispatch_alternative(state): + from backend.core.orchestrator import IntentLease, Orchestrator + from backend.members.base_member import SolveResult + + pid = _make_project(state, "web") + with state.db.connect() as conn: + graph_store.set_status(conn, pid, "running") + stalled = edge_store.create_intent( + conn, pid, ["origin"], "repeat a stalled scan", "diamond" + ) + alternative = edge_store.create_intent( + conn, pid, ["origin"], "run the alternate static query", "diamond" + ) + + orch = Orchestrator(state, max_workers=1) + for _ in range(orch._MAX_CONSECUTIVE_STALLS): + # Mirror the real dispatch path: the worker claims the intent with a + # lease, then finishes stalled, and the reaper releases the claim. + with state.db.connect() as conn: + token = edge_store.claim_intent(conn, pid, stalled.id, "aventurine") + assert token + with orch._lock: + orch._intent_leases[(pid, stalled.id)] = IntentLease( + worker="aventurine", owner="aventurine", token=token + ) + future = Future() + future.set_result(SolveResult(status="stalled", steps=3)) + with orch._lock: + orch._task_index[(pid, stalled.id)] = future + orch._reap_finished_futures() + + key = (pid, stalled.id) + assert orch._member_stall_counts[key] == orch._MAX_CONSECUTIVE_STALLS + assert orch._member_retry_blocked(pid, stalled.id) + with state.db.connect() as conn: + assert edge_store.get_intent(conn, pid, stalled.id)["worker"] is None + + launched = [] + + def capture_launch(project_id, member_name, intent_id, category, is_initial): + launched.append((project_id, member_name, intent_id, category, is_initial)) + return True + + orch._launch_member = capture_launch + orch._dispatch_project(pid) + assert launched == [(pid, "aventurine", alternative.id, "web", False)] + + entries = state.logger.read_log("project", pid, None) + assert any( + entry["event"] == "member_intent_deferred" + and entry["intent"] == stalled.id + and entry["reason"] == "repeated_stalls" + for entry in entries + ) + orch.shutdown() diff --git a/tests/test_ret2shell.py b/tests/test_ret2shell.py new file mode 100644 index 0000000..c4bfdfa --- /dev/null +++ b/tests/test_ret2shell.py @@ -0,0 +1,704 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from backend.blackboard import graph_store +from backend.mcp.mcp_client import MCPClient +from backend.platform.mapping import FieldMapping +from backend.platform.ret2shell import ( + Ret2ShellAdapter, + Ret2ShellAuthError, + Ret2ShellClient, + Ret2ShellError, + Ret2ShellPreflightError, + Ret2ShellRateLimitError, + _SubmitRateLimiter, +) +from backend.platform.ret2shell_mcp import WsrxTunnelManager, build_ret2shell_mcp +from backend.server.app import create_app +from tests.helpers import setup_test_auth, write_mock_config + +BASE = "https://r2s.test" + + +class FakeResponse: + def __init__(self, status_code=200, payload=None, text="", headers=None, content=None): + self.status_code = status_code + self._payload = payload + if text: + self.text = text + elif payload is not None: + self.text = json.dumps(payload) + else: + self.text = "" + self.headers = headers or {} + if content is not None: + self._content = content + elif payload is not None: + self._content = json.dumps(payload).encode() + else: + self._content = b"" + + @property + def content(self): + return self._content + + def json(self): + if self._payload is None: + raise ValueError("no JSON payload") + return self._payload + + def iter_content(self, chunk_size): + yield self._content + + +class FakeSession: + def __init__(self, router): + self.router = router + self.calls: list[tuple[str, str, dict]] = [] + + def request(self, method, url, **kwargs): + self.calls.append((method, url, kwargs)) + return self.router(method, url, kwargs) + + def post(self, url, **kwargs): + return self.request("POST", url, **kwargs) + + def get(self, url, **kwargs): + return self.request("GET", url, **kwargs) + + def close(self): + pass + + +def make_client(router, authed=True, **kwargs): + defaults = dict( + base_url=BASE, + game_id=37, + username="user", + password="pass", + poll_attempts=3, + poll_interval=0.0, + sleep=lambda seconds: None, + ) + if authed: + defaults["token"] = "seed-token" + defaults.update(kwargs) + return Ret2ShellClient(session=FakeSession(router), **defaults) + + +def captcha_ok(url): + if url.endswith("/api/account/captcha/cli"): + return FakeResponse( + 200, + {"id": "cap-1", "validator": "pow", "challenge": "2#seed"}, + ) + return None + + +def login_ok(url, token="token-1"): + if cap := captcha_ok(url): + return cap + if url.endswith("/api/account/login"): + return FakeResponse(200, headers={"Set-Token": token}) + return None + + +# ---- auth ---- + + +def test_login_captures_set_token_header_and_uses_bearer(): + def router(method, url, kwargs): + if login := login_ok(url): + return login + if url.endswith("/api/account/profile"): + return FakeResponse(200, {"account": "user"}) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router, authed=False) + profile = client.get_profile() + assert profile["account"] == "user" + profile_call = next(call for call in client.session.calls if call[1].endswith("/api/account/profile")) + assert profile_call[2]["headers"]["Authorization"] == "Bearer token-1" + client.close() + + +def test_login_failure_raises_auth_error(): + def router(method, url, kwargs): + if url.endswith("/api/account/login"): + return FakeResponse(401, text="account or password is wrong") + if login := login_ok(url): + return login + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router, authed=False) + with pytest.raises(Ret2ShellAuthError, match="account or password is wrong"): + client.get_profile() + client.close() + + +def test_set_token_refresh_rolls_token_forward(): + state = {"logins": 0} + + def router(method, url, kwargs): + if cap := captcha_ok(url): + return cap + if url.endswith("/api/account/login"): + state["logins"] += 1 + return FakeResponse(200, headers={"Set-Token": "token-1"}) + if url.endswith("/api/game/37"): + if state["logins"] == 1: + return FakeResponse(200, {"id": 37}, headers={"Set-Token": "token-2"}) + return FakeResponse(200, {"id": 37}) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router, authed=False) + client.get_game() + client.get_game() + auth = client.session.calls[-1][2]["headers"]["Authorization"] + assert auth == "Bearer token-2" + assert state["logins"] == 1 + client.close() + + +def test_expired_token_relogins_once(): + state = {"profile_calls": 0} + + def router(method, url, kwargs): + if cap := captcha_ok(url): + return cap + if url.endswith("/api/account/login"): + return FakeResponse(200, headers={"Set-Token": "token-2"}) + if url.endswith("/api/account/profile"): + state["profile_calls"] += 1 + if state["profile_calls"] == 1: + return FakeResponse(401, text="token expired") + return FakeResponse(200, {"account": "user"}) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router, authed=False) + assert client.get_profile()["account"] == "user" + first_post = next(call for call in client.session.calls if call[0] == "POST") + assert first_post[1].endswith("/api/account/login") # login before any data call + assert client.session.calls[-1][2]["headers"]["Authorization"] == "Bearer token-2" + client.close() + + +# ---- challenges / attachments ---- + + +def test_list_challenges_parses_items_total_tuple(): + def router(method, url, kwargs): + if url.endswith("/api/game/37/challenge"): + return FakeResponse( + 200, + [ + [{"id": 1, "name": "Warmup", "tag": [{"name": "pwn", "primary": True}]}], + 1, + ], + ) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router) + challenges = client.list_challenges() + assert challenges[0]["name"] == "Warmup" + client.close() + + +def test_error_detail_uses_plain_text_body(): + def router(method, url, kwargs): + if url.endswith("/api/game/37/challenge"): + return FakeResponse(412, text="game has not started") + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router) + with pytest.raises(Ret2ShellError, match="game has not started"): + client.list_challenges() + client.close() + + +def test_adapter_maps_categories_and_downloads_files(tmp_path): + files = [{"folder": "static", "file": "handout.zip"}] + + class StubClient: + game_id = 37 + + def list_challenges(self, game_id=None): + return [ + {"id": 1, "name": "Pwn Me", "tag": [{"name": "pwn", "primary": True}], "content": "nc"}, + {"id": 2, "name": "Odd", "tag": [{"name": "weird", "primary": True}], "content": ""}, + {"id": 3, "name": "NoTag", "content": ""}, + ] + + def get_challenge(self, challenge_id, game_id=None): + return {} + + def list_files(self, challenge_id, game_id=None): + return files + + def download_file(self, challenge_id, folder, file, dest_dir, game_id=None, *, max_bytes=None): + target = Path(dest_dir) / file + target.write_bytes(b"zip-bytes") + return target + + adapter = Ret2ShellAdapter(StubClient(), category_map={"weird": "reverse"}) + challenges = adapter.fetch_challenges() + assert [c.category for c in challenges] == ["pwn", "reverse", "misc"] + assert [c.external_id for c in challenges] == ["1", "2", "3"] + assert challenges[0].attachment_urls == [] + + downloaded = adapter.download_attachments(challenges[0], tmp_path) + assert downloaded[0].name == "handout.zip" + assert downloaded[0].read_bytes() == b"zip-bytes" + + +def test_download_file_streams_to_disk(tmp_path): + def router(method, url, kwargs): + if url.endswith("/api/game/37/challenge/5/file") and kwargs.get("params", {}).get("file"): + return FakeResponse(200, content=b"attachment-data") + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router) + target = client.download_file(5, "static", "handout.tar.gz", tmp_path) + assert target.name == "handout.tar.gz" + assert target.read_bytes() == b"attachment-data" + call = client.session.calls[0] + assert call[2]["params"] == {"folder": "static", "file": "handout.tar.gz"} + client.close() + + +# ---- instances ---- + + +def test_instance_lifecycle_and_wait_for_endpoints(): + state = {"polls": 0, "started": False} + + def router(method, url, kwargs): + if url.endswith("/api/game/37/challenge/5/instance") and method == "POST": + state["started"] = True + return FakeResponse(200) + if url.endswith("/api/game/37/instance"): + state["polls"] += 1 + exposed = [{"1.2.3.4:1337": {"protocol": "tcp", "port": 1337}}] if state["polls"] >= 2 else None + return FakeResponse( + 200, + [ + { + "state": "Running", + "challenge_id": 5, + "exposed_ports": exposed, + "renew_count": 0, + } + ], + ) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router) + client.start_instance(5) + assert state["started"] is True + instance = client.wait_for_instance(5, timeout=10, interval=0) + assert instance["exposed_ports"] + client.close() + + +def test_wait_for_instance_times_out(): + def router(method, url, kwargs): + if url.endswith("/api/game/37/instance"): + return FakeResponse(200, [{"state": "Pending", "challenge_id": 5, "exposed_ports": None}]) + raise AssertionError(f"unexpected {method} {url}") + + clock = {"now": 0.0} + client = make_client( + router, + monotonic=lambda: clock["now"], + sleep=lambda seconds: clock.__setitem__("now", clock["now"] + seconds), + ) + with pytest.raises(Ret2ShellError, match="did not become reachable"): + client.wait_for_instance(5, timeout=3, interval=1) + client.close() + + +# ---- submission ---- + + +def _submit_router(submissions): + state = {"polls": 0} + + def router(method, url, kwargs): + if login := login_ok(url): + return login + if url.endswith("/api/game/37/challenge/5/submit"): + if method == "POST": + submissions.append({"id": 900, "solved": None, "result": None}) + return FakeResponse(200, submissions[-1]) + state["polls"] += 1 + submission = submissions[0] + if state["polls"] >= 2 and submission["solved"] is None: + submission["solved"] = True + submission["result"] = "accepted" + return FakeResponse(200, submission) + raise AssertionError(f"unexpected {method} {url}") + + return router + + +def test_submit_flag_polls_async_verdict(): + submissions = [] + client = make_client(_submit_router(submissions)) + result = client.submit_flag(5, "flag{demo}", check_solved=False) + assert result["solved"] is True + assert result["result"] == "accepted" + client.close() + + +def test_submit_flag_preflight_refuses_solved_challenge(): + def router(method, url, kwargs): + if login := login_ok(url): + return login + if url.endswith("/api/game/37/challenge/5/submit") and method == "GET": + return FakeResponse(200, {"solved": True, "solves": 12}) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router) + with pytest.raises(Ret2ShellPreflightError, match="already solved"): + client.submit_flag(5, "flag{demo}") + posts = [call for call in client.session.calls if call[0] == "POST" and call[1].endswith("/submit")] + assert posts == [] + client.close() + + +def test_submit_flag_rejects_empty_flag(): + client = make_client(lambda *args: FakeResponse(200, {})) + with pytest.raises(Ret2ShellPreflightError, match="flag must not be empty"): + client.submit_flag(5, " ") + client.close() + + +def test_submit_flag_maps_http_429_to_rate_limit(): + def router(method, url, kwargs): + if login := login_ok(url): + return login + if url.endswith("/api/game/37/challenge/5/submit"): + if method == "POST": + return FakeResponse(429, text="too many submissions, please calmdown") + return FakeResponse(200, {"solved": False}) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router) + with pytest.raises(Ret2ShellRateLimitError, match="too many submissions"): + client.submit_flag(5, "flag{demo}", check_solved=False) + client.close() + + +def test_submit_flag_poll_timeout_keeps_solved_none(): + submissions = [{"id": 900, "solved": None, "result": None}] + + def router(method, url, kwargs): + if login := login_ok(url): + return login + if url.endswith("/api/game/37/challenge/5/submit"): + if method == "POST": + return FakeResponse(200, submissions[0]) + return FakeResponse(200, submissions[0]) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router, poll_attempts=2) + result = client.submit_flag(5, "flag{demo}", check_solved=False) + assert result["solved"] is None + gets = [c for c in client.session.calls if c[0] == "GET" and c[1].endswith("/submit")] + assert len(gets) == 2 + client.close() + + +def test_rate_limiter_enforces_shared_quota(): + clock = {"now": 0.0} + limiter = _SubmitRateLimiter(monotonic=lambda: clock["now"]) + for _ in range(10): + limiter.reserve() + with pytest.raises(Ret2ShellRateLimitError, match="retry in 300s"): + limiter.reserve() + clock["now"] = 301.0 + limiter.reserve() # window expired + assert limiter.used() == 1 + + +# ---- MCP ---- + + +class StubPlatformClient: + base_url = "https://ctf.r2s.test" + + def __init__(self): + self.calls: list[tuple[str, ...]] = [] + + def find_instance(self, challenge_id, game_id=None): + self.calls.append(("find", challenge_id)) + return None + + def start_instance(self, challenge_id, game_id=None): + self.calls.append(("start", challenge_id)) + + def wait_for_instance(self, challenge_id, game_id=None, **kwargs): + self.calls.append(("wait", challenge_id)) + return { + "state": "Running", + "exposed_ports": [{"ctf.r2s.test:1337": {"protocol": "tcp"}}], + "renew_count": 0, + } + + def renew_instance(self, challenge_id, game_id=None): + self.calls.append(("renew", challenge_id)) + + def destroy_instance(self, challenge_id, game_id=None): + self.calls.append(("destroy", challenge_id)) + + def challenge_status(self, challenge_id, game_id=None): + self.calls.append(("status", challenge_id)) + return {"solved": False, "solves": 3} + + def close(self): + pass + + +def _call_tool(server, name, **arguments): + async def run(): + async with MCPClient.in_process(server) as client: + tools = {tool.name for tool in await client.list_tools()} + assert name in tools + return await client.call_tool(name, arguments) + + return asyncio.run(run()) + + +def test_ret2shell_mcp_exposes_instance_tools(): + stub = StubPlatformClient() + server = build_ret2shell_mcp(stub) + started = _call_tool(server, "instance_start", challenge_id=5) + assert started["endpoints"] + assert ("start", 5) in stub.calls and ("wait", 5) in stub.calls + assert _call_tool(server, "instance_status", challenge_id=5)["running"] is False + assert _call_tool(server, "instance_renew", challenge_id=5)["renewed"] is True + assert _call_tool(server, "instance_stop", challenge_id=5)["stopped"] is True + status = _call_tool(server, "challenge_status", challenge_id=5) + assert status["solved"] is False and status["solves"] == 3 + + +# ---- wsrx tunneling ---- + + +class FakeProcess: + def __init__(self, remote, local_port): + self.remote = remote + self.local_port = local_port + self.returncode = None + self.terminated = False + + def poll(self): + return self.returncode + + def terminate(self): + self.terminated = True + self.returncode = 0 + + def kill(self): + self.returncode = -9 + + def wait(self, timeout=None): + return self.returncode + + +def _fake_manager(spawns): + def spawner(remote, local_port): + process = FakeProcess(remote, local_port) + spawns.append(process) + return process + + return WsrxTunnelManager(spawner=spawner, probe=lambda port: True) + + +class TrafficStubClient(StubPlatformClient): + """Platform shape observed on ret2shell: traffic token + ports.""" + + def _traffic_instance(self): + return { + "state": "Running", + "exposed_ports": None, + "traffic": "tok1430", + "ports": [9999], + "renew_count": 0, + } + + def find_instance(self, challenge_id, game_id=None): + self.calls.append(("find", challenge_id)) + if ("start", challenge_id) not in self.calls: + return None + return self._traffic_instance() + + def wait_for_instance(self, challenge_id, game_id=None, **kwargs): + self.calls.append(("wait", challenge_id)) + return self._traffic_instance() + + +def test_wsrx_manager_spawns_idempotent_and_cleans_up(): + spawns = [] + manager = _fake_manager(spawns) + remote = "wss://ctf.xidian.edu.cn/api/traffic/tok1430?port=9999" + + first = manager.ensure(1430, [remote]) + assert first["endpoints"] == ["ipc-app:21430"] + assert first["started"] == [remote] + assert len(spawns) == 1 + assert spawns[0].remote == remote + assert spawns[0].local_port == 21430 + + # Repeated ensure for the same challenge reuses the live tunnel. + second = manager.ensure(1430, [remote]) + assert second["endpoints"] == ["ipc-app:21430"] + assert second["started"] == [] + assert len(spawns) == 1 + + manager.stop(1430) + assert spawns[0].terminated + assert manager.endpoints(1430) == [] + + +def test_wsrx_manager_restarts_dead_tunnel_and_binds_one_port_each(): + spawns = [] + manager = _fake_manager(spawns) + remotes = [ + "wss://ctf.xidian.edu.cn/api/traffic/tok?port=1337", + "wss://ctf.xidian.edu.cn/api/traffic/tok?port=9999", + ] + result = manager.ensure(7, remotes) + assert result["endpoints"] == ["ipc-app:20007", "ipc-app:20008"] + assert len(spawns) == 2 + + spawns[0].returncode = 1 # process died behind our back + result = manager.ensure(7, remotes) + assert result["endpoints"] == ["ipc-app:20007", "ipc-app:20008"] + assert result["started"] == [remotes[0]] + assert len(spawns) == 3 + # Only the two live tunnels remain tracked. + assert manager.endpoints(7) == ["ipc-app:20007", "ipc-app:20008"] + + +def test_instance_start_tunnels_wsrx_remotes_to_local_port(): + spawns = [] + stub = TrafficStubClient() + server = build_ret2shell_mcp(stub, tunnel_manager=_fake_manager(spawns)) + started = _call_tool(server, "instance_start", challenge_id=1430) + assert started["endpoints"] == ["ipc-app:21430"] + assert started["wsrx_remotes"] == [ + "wss://ctf.r2s.test/api/traffic/tok1430?port=9999" + ] + assert ("start", 1430) in stub.calls + + # Idempotent: a repeated start must not spawn a second tunnel. + again = _call_tool(server, "instance_start", challenge_id=1430) + assert again["endpoints"] == ["ipc-app:21430"] + assert len(spawns) == 1 + + status = _call_tool(server, "instance_status", challenge_id=1430) + assert status["running"] is True + assert status["endpoints"] == ["ipc-app:21430"] + + stopped = _call_tool(server, "instance_stop", challenge_id=1430) + assert stopped["stopped"] is True + assert stopped["tunnels_closed"] == [21430] + assert spawns[0].terminated + assert ("destroy", 1430) in stub.calls + + +def test_instance_start_passes_direct_endpoints_through(): + # GZCTF-style direct host:port exposure needs no wsrx at all. + spawns = [] + stub = StubPlatformClient() + server = build_ret2shell_mcp(stub, tunnel_manager=_fake_manager(spawns)) + started = _call_tool(server, "instance_start", challenge_id=5) + assert started["endpoints"] == [{"ctf.r2s.test:1337": {"protocol": "tcp"}}] + assert "wsrx_remotes" not in started + assert spawns == [] + + +# ---- API import integration ---- + + +@pytest.fixture +def api_client(tmp_path, monkeypatch): + config_dir = write_mock_config(tmp_path / "config") + monkeypatch.setenv("IPC_ROOT", str(tmp_path)) + app = create_app(root=tmp_path) + with TestClient(app) as test_client: + setup_test_auth(test_client) + test_client.app.state.ipc.config_dir = config_dir + test_client.app.state.ipc.reload_config() + yield test_client + + +@pytest.mark.postgres +def test_platform_import_supports_ret2shell_platform(api_client, monkeypatch): + class StubClient: + def __init__(self, *args, **kwargs): + pass + + def list_challenges(self, game_id=None): + return [ + { + "id": 11, + "name": "Remote Calc", + "tag": [{"name": "pwn", "primary": True}], + "content": "nc target 1337", + } + ] + + def get_challenge(self, challenge_id, game_id=None): + return {} + + def list_files(self, challenge_id, game_id=None): + return [{"folder": "static", "file": "calc.zip"}] + + def download_file(self, challenge_id, folder, file, dest_dir, game_id=None, *, max_bytes=None): + target = Path(dest_dir) / file + target.write_bytes(b"zip") + return target + + def close(self): + pass + + monkeypatch.setattr("backend.api.platform.Ret2ShellClient", StubClient) + mapping = FieldMapping(platform="ret2shell", game_id=37) + + preview = api_client.post( + "/api/platform/challenges", json={"mapping": mapping.model_dump()} + ) + assert preview.status_code == 200 + challenges = preview.json()["challenges"] + assert challenges[0]["external_id"] == "11" + assert challenges[0]["category"] == "pwn" + + imported = api_client.post( + "/api/platform/import", + json={"mapping": mapping.model_dump(), "select": ["11"]}, + ) + assert imported.status_code == 201 + item = imported.json()["imported"][0] + assert item["category"] == "pwn" + with api_client.app.state.ipc.db.connect() as conn: + # The challenge description must land in the origin fact so the + # Member context exposes it (connection commands, flag format...). + facts = {f.id: f.description for f in graph_store.project_detail(conn, item["project_id"]).facts} + assert "nc target 1337" in facts.get("origin", "") + graph_store.set_flag(conn, item["project_id"], "flag{r2s}") + graph_store.set_status(conn, item["project_id"], "completed") + assert api_client.get(f"/api/flags/{item['project_id']}").json()["flag"] == "flag{r2s}" + + +def test_field_mapping_requires_list_url_for_http_json(): + with pytest.raises(ValueError, match="list_url is required"): + FieldMapping() + assert FieldMapping(platform="ret2shell").game_id is None