From 5c82e0436a1cd7016fa312bbbd4d14142f6041ee Mon Sep 17 00:00:00 2001 From: PureStream108 Date: Mon, 31 Aug 2026 22:16:03 +0800 Subject: [PATCH 01/10] feat(platform): add GZCTF platform client Add a read-only GZCTF adapter (challenges, attachments, submissions) with unit tests and the fetch/check helper scripts. --- backend/platform/gzctf.py | 448 ++++++++++++++++++++++++++++++++ scripts/fetch_gzctf_targets.py | 72 +++++ scripts/gzctf_platform_check.py | 52 ++++ tests/test_gzctf.py | 205 +++++++++++++++ 4 files changed, 777 insertions(+) create mode 100644 backend/platform/gzctf.py create mode 100644 scripts/fetch_gzctf_targets.py create mode 100644 scripts/gzctf_platform_check.py create mode 100644 tests/test_gzctf.py diff --git a/backend/platform/gzctf.py b/backend/platform/gzctf.py new file mode 100644 index 0000000..f15e9af --- /dev/null +++ b/backend/platform/gzctf.py @@ -0,0 +1,448 @@ +from __future__ import annotations + +import os +from datetime import UTC, datetime +from typing import Any + +import requests + + +class GZCTFError(RuntimeError): + pass + + +class GZCTFLoginError(GZCTFError): + pass + + +class GZCTFPreflightError(GZCTFError): + """A submission was refused before it could consume a platform attempt.""" + + +def _track_id(item: dict[str, Any]) -> str: + for key in ("trackId", "track_id", "id"): + value = item.get(key) + if value: + return str(value) + return "" + + +def _track_entries(challenge: dict[str, Any]) -> list[dict[str, Any]]: + entries = challenge.get("tracks") + if isinstance(entries, list): + flags = challenge.get("flags") if isinstance(challenge.get("flags"), list) else [] + result: list[dict[str, Any]] = [] + for item in entries: + if not isinstance(item, dict) or not _track_id(item): + continue + # GZCTF exposes track metadata and per-level state as sibling + # arrays. Join them here so preflight sees one uniform shape. + joined = dict(item) + if not isinstance(joined.get("levels"), list): + joined["levels"] = [ + flag + for flag in flags + if isinstance(flag, dict) and _track_id(flag) == _track_id(item) + ] + result.append(joined) + return result + # Some GZCTF versions expose the per-track level state under `flags`. + entries = challenge.get("flags") + if isinstance(entries, list): + return [item for item in entries if isinstance(item, dict) and _track_id(item)] + return [] + + +def _level_entries(track: dict[str, Any]) -> list[dict[str, Any]]: + entries = track.get("levels") + if isinstance(entries, list): + return [item for item in entries if isinstance(item, dict)] + entries = track.get("flags") + if isinstance(entries, list): + return [item for item in entries if isinstance(item, dict)] + return [] + + +def _find_level(track: dict[str, Any], level: int) -> dict[str, Any] | None: + for item in _level_entries(track): + raw = item.get("level", item.get("levelId")) + try: + if int(raw) == level: + return item + except (TypeError, ValueError): + continue + return None + + +def _parse_time(value: Any) -> datetime | None: + if not value or not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + + +def _active_cooldown(*objects: dict[str, Any]) -> str | None: + now = datetime.now(UTC) + for obj in objects: + cooldown = obj.get("cooldown") + if isinstance(cooldown, dict): + value = cooldown.get("cooldownUntil") + else: + value = obj.get("cooldownUntil") + until = _parse_time(value) + if until and until > now: + return str(value) + return None + + +def _normalize_answer(answer: Any) -> str: + return str(answer).strip() + + +def validate_submission( + challenge: dict[str, Any], + *, + level: int, + answer: Any, + track_id: str | None = None, + evidence: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Validate one submission against live challenge state. + + This is intentionally conservative. It refuses ambiguous tracks, + locked/solved levels, active cooldowns, unverified evidence and + malformed answers before the POST endpoint is called. + """ + + try: + level = int(level) + except (TypeError, ValueError) as exc: + raise GZCTFPreflightError("level must be a positive integer") from exc + normalized_answer = _normalize_answer(answer) + if not normalized_answer: + raise GZCTFPreflightError("answer must not be empty") + + challenge_lock = challenge.get("challengeLock", challenge.get("challengeLocked", False)) + if isinstance(challenge_lock, dict): + # Current GZCTF returns a status object such as + # ``{"isLocked": false, "unlocksAt": null}``; treating the + # object itself as truthy would incorrectly reject every preflight. + challenge_lock = challenge_lock.get( + "isLocked", challenge_lock.get("locked", challenge_lock.get("is_locked", False)) + ) + if bool(challenge_lock): + raise GZCTFPreflightError("challenge is currently locked") + + tracks = _track_entries(challenge) + enabled = [item for item in tracks if item.get("isEnabled", item.get("is_enabled", True))] + if not enabled: + raise GZCTFPreflightError("live challenge has no enabled track state") + if track_id: + track = next((item for item in enabled if _track_id(item) == str(track_id)), None) + if track is None: + raise GZCTFPreflightError(f"trackId is not an enabled track: {track_id}") + elif len(enabled) == 1: + track = enabled[0] + track_id = _track_id(track) + else: + raise GZCTFPreflightError( + "challenge exposes multiple enabled tracks; an explicit trackId is required" + ) + + level_state = _find_level(track, level) + if level_state is None: + raise GZCTFPreflightError(f"track {track_id} has no level {level} state") + status = str(level_state.get("status", level_state.get("state", ""))).strip().lower() + if status in {"solved", "complete", "completed"} or bool(level_state.get("solved")): + raise GZCTFPreflightError(f"level {level} is already solved") + if status in {"locked", "unavailable", "banned"}: + raise GZCTFPreflightError(f"level {level} is locked") + current_level = track.get("currentLevel", track.get("current_level")) + try: + if current_level is not None and level > int(current_level): + raise GZCTFPreflightError( + f"level {level} is locked until level {int(current_level)} is solved" + ) + except (TypeError, ValueError): + pass + + cooldown = _active_cooldown(challenge, track, level_state) + if cooldown: + raise GZCTFPreflightError(f"submission cooldown active until {cooldown}") + cooldown_obj = level_state.get("cooldown") + if not isinstance(cooldown_obj, dict): + cooldown_obj = track.get("cooldown") if isinstance(track.get("cooldown"), dict) else challenge.get("cooldown", {}) + if isinstance(cooldown_obj, dict): + wrong_count = cooldown_obj.get("wrongCount", cooldown_obj.get("wrong_count")) + wrong_limit = cooldown_obj.get("wrongAnswerLimit", cooldown_obj.get("wrong_answer_limit")) + try: + if wrong_count is not None and wrong_limit is not None and int(wrong_count) >= int(wrong_limit): + raise GZCTFPreflightError("wrong-answer limit reached; wait for the platform cooldown") + except (TypeError, ValueError): + pass + + if evidence is not None: + status_value = str(evidence.get("status", "candidate")).strip().lower() + answers = evidence.get("answers") if isinstance(evidence.get("answers"), dict) else {} + evidence_answer = answers.get(str(level), answers.get(level)) + if evidence_answer is None or _normalize_answer(evidence_answer) != normalized_answer: + raise GZCTFPreflightError("submitted answer does not match the evidence report") + if status_value not in {"verified", "confirmed"}: + raise GZCTFPreflightError("candidate evidence has not been verified") + + return { + "ok": True, + "challenge_id": challenge.get("id"), + "track_id": track_id, + "level": level, + "answer": normalized_answer, + "status": status or "unsolved", + "current_level": current_level, + } + + +class GZCTFClient: + """Cookie-authenticated GZCTF participant API client. + + GZCTF flag submission requires the normal user cookie session, not the + admin-only API token. This client keeps that distinction explicit. + """ + + def __init__( + self, + base_url: str = "", + username: str = "", + password: str = "", + *, + timeout: float = 30, + ) -> None: + self.base_url = (base_url or os.getenv("IPC_GZ_BASE_URL", "")).rstrip("/") + self.username = username or os.getenv("IPC_GZ_USERNAME", "") + self.password = password or os.getenv("IPC_GZ_PASSWORD", "") + self.timeout = timeout + self.session = requests.Session() + self.logged_in = False + + def _url(self, path: str) -> str: + return f"{self.base_url}{path}" + + def _require_base_url(self) -> None: + if not self.base_url: + raise GZCTFError("GZCTF base_url is not configured; set IPC_GZ_BASE_URL") + + def login(self) -> None: + self._require_base_url() + if not self.username or not self.password: + raise GZCTFLoginError("GZCTF username/password are not configured") + response = self.session.post( + self._url("/api/Account/LogIn"), + json={"userName": self.username, "password": self.password}, + timeout=self.timeout, + ) + if response.status_code != 200: + try: + message = response.json().get("title", "login failed") + except ValueError: + message = "login failed" + raise GZCTFLoginError(message) + self.logged_in = True + + def _authorized_get(self, path: str, *, params: dict[str, Any] | None = None) -> Any: + self._require_base_url() + if not self.logged_in: + self.login() + response = self.session.get( + self._url(path), params=params, timeout=self.timeout + ) + response.raise_for_status() + return response.json() + + def _authorized_post( + self, + path: str, + json: dict[str, Any] | None = None, + *, + params: dict[str, Any] | None = None, + ) -> Any: + self._require_base_url() + if not self.logged_in: + self.login() + response = self.session.post( + self._url(path), + json=json or {}, + params=params, + timeout=self.timeout, + ) + if response.status_code >= 400: + # Keep the platform's validation reason in the exception. A bare + # ``raise_for_status`` only exposed ``400 Bad Request`` and made + # it impossible to distinguish a contract mismatch from a wrong + # answer without issuing another blind attempt. + detail = response.text.strip().replace("\r", " ").replace("\n", " ") + if len(detail) > 1000: + detail = detail[:1000] + "..." + suffix = f": {detail}" if detail else "" + raise requests.HTTPError( + f"{response.status_code} Client Error for url: {response.url}{suffix}", + response=response, + ) + return response.json() + + def list_games(self, *, query: dict[str, Any] | None = None) -> Any: + """List visible competitions without selecting one implicitly.""" + + return self._authorized_get("/api/game", params=query) + + def list_teams(self) -> Any: + return self._authorized_get("/api/team") + + def create_team(self, payload: dict[str, Any]) -> Any: + return self._authorized_post("/api/team", payload) + + def accept_team_invite(self, payload: dict[str, Any]) -> Any: + return self._authorized_post("/api/team/accept", payload) + + def join_game(self, game_id: int, payload: dict[str, Any] | None = None) -> Any: + return self._authorized_post(f"/api/game/{game_id}", payload or {}) + + def leave_game(self, game_id: int) -> Any: + self._require_base_url() + if not self.logged_in: + self.login() + response = self.session.delete( + self._url(f"/api/game/{game_id}"), timeout=self.timeout + ) + response.raise_for_status() + return response.json() if response.content else {} + + def get_profile(self) -> dict[str, Any]: + return self._authorized_get("/api/Account/Profile") + + def get_game(self, game_id: int) -> dict[str, Any]: + return self._authorized_get(f"/api/Game/{game_id}") + + def get_game_details(self, game_id: int) -> dict[str, Any]: + return self._authorized_get(f"/api/Game/{game_id}/Details") + + def get_game_check(self, game_id: int) -> dict[str, Any]: + return self._authorized_get(f"/api/Game/{game_id}/Check") + + def get_game_participation(self, game_id: int) -> Any: + return self._authorized_get(f"/api/game/{game_id}/participations") + + def get_scoreboard(self, game_id: int) -> dict[str, Any]: + return self._authorized_get(f"/api/Game/{game_id}/Scoreboard") + + def get_challenge( + self, + game_id: int, + challenge_id: int, + ) -> dict[str, Any]: + return self._authorized_get( + f"/api/Game/{game_id}/Challenges/{challenge_id}" + ) + + def preflight_submission( + self, + game_id: int, + challenge_id: int, + *, + level: int, + answer: Any, + track_id: str | None = None, + evidence: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Fetch live state and validate without issuing a submission POST.""" + + challenge = self.get_challenge(game_id, challenge_id) + return validate_submission( + challenge, + level=level, + answer=answer, + track_id=track_id, + evidence=evidence, + ) + + def submit_level( + self, + game_id: int, + challenge_id: int, + answer: Any, + *, + level: int = 1, + track_id: str | None = None, + evidence: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Run preflight immediately before the single allowed POST.""" + + decision = self.preflight_submission( + game_id, + challenge_id, + level=level, + answer=answer, + track_id=track_id, + evidence=evidence, + ) + return self.submit_flag( + game_id, + challenge_id, + decision["answer"], + level=decision["level"], + track_id=decision["track_id"], + ) + + def submit_flag( + self, + game_id: int, + challenge_id: int, + flag: str, + *, + level: int = 1, + track_id: str | None = None, + ) -> dict[str, Any]: + body: dict[str, Any] = {"flag": flag, "level": level} + if track_id: + body["trackId"] = track_id + result = self._authorized_post( + f"/api/Game/{game_id}/Challenges/{challenge_id}", + body, + ) + if not isinstance(result, dict): + raise GZCTFError( + f"GZCTF submit_flag returned unexpected type: {type(result).__name__}" + ) + return result + + def get_submission_status( + self, + game_id: int, + challenge_id: int, + submission_id: int, + ) -> dict[str, Any]: + return self._authorized_get( + f"/api/Game/{game_id}/Challenges/{challenge_id}/Status/{submission_id}" + ) + + def get_my_submissions(self, *, query: dict[str, Any] | None = None) -> Any: + return self._authorized_get("/api/ext/submissions/mine", params=query) + + def get_my_tickets(self, *, query: dict[str, Any] | None = None) -> Any: + return self._authorized_get("/api/ext/tickets/mine", params=query) + + def get_ticket(self, ticket_id: int | str) -> Any: + return self._authorized_get(f"/api/ext/tickets/{ticket_id}") + + def create_ticket(self, payload: dict[str, Any]) -> Any: + return self._authorized_post("/api/ext/tickets", payload) + + def close(self) -> None: + self.session.close() + + def __enter__(self) -> GZCTFClient: + return self + + def __exit__(self, *exc: object) -> None: + self.close() diff --git a/scripts/fetch_gzctf_targets.py b/scripts/fetch_gzctf_targets.py new file mode 100644 index 0000000..4badb36 --- /dev/null +++ b/scripts/fetch_gzctf_targets.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import argparse +import os +from datetime import UTC, datetime +from pathlib import Path + +import yaml + +from backend.platform.gzctf import GZCTFClient + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +TARGETS_PATH = PROJECT_ROOT / "platform_targets" / "gzctf.yaml" + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Fetch visible GZCTF challenges for one game into a YAML target list." + ) + parser.add_argument("--base-url", default=os.getenv("IPC_GZ_BASE_URL", "")) + parser.add_argument("--username", default=os.getenv("IPC_GZ_USERNAME", "")) + parser.add_argument("--password", default=os.getenv("IPC_GZ_PASSWORD", "")) + parser.add_argument("--game", type=int, required=True) + parser.add_argument("--output", type=Path, default=TARGETS_PATH) + args = parser.parse_args() + + challenges: list[dict] = [] + with GZCTFClient( + base_url=args.base_url, + username=args.username, + password=args.password, + ) as client: + client.login() + game = client.get_game(args.game) + scoreboard = client.get_scoreboard(args.game) + for category, items in (scoreboard.get("challenges") or {}).items(): + for item in items: + challenges.append( + { + "id": item["id"], + "title": item.get("title", ""), + "category": category, + "score": item.get("score", 0), + "solved": item.get("solved", 0), + } + ) + challenges.sort(key=lambda item: (item["category"], item["id"])) + + data = { + "updated_at": datetime.now(UTC).isoformat(), + "game": { + "id": game.get("id"), + "title": game.get("title"), + "team_name": game.get("teamName"), + "status": game.get("status"), + "start": game.get("start"), + "end": game.get("end"), + }, + "challenge_count": len(challenges), + "challenges": challenges, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + yaml.safe_dump(data, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + print(f"wrote {len(challenges)} challenges to {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gzctf_platform_check.py b/scripts/gzctf_platform_check.py new file mode 100644 index 0000000..deca0da --- /dev/null +++ b/scripts/gzctf_platform_check.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import argparse +import os + +from backend.platform.gzctf import GZCTFClient + +DEFAULT_BASE_URL = "" + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check GZCTF login, profile and game state." + ) + parser.add_argument("--base-url", default=os.getenv("IPC_GZ_BASE_URL", DEFAULT_BASE_URL)) + parser.add_argument("--username", default=os.getenv("IPC_GZ_USERNAME", "")) + parser.add_argument("--password", default=os.getenv("IPC_GZ_PASSWORD", "")) + parser.add_argument("--game", type=int, default=2) + args = parser.parse_args() + + with GZCTFClient( + base_url=args.base_url, + username=args.username, + password=args.password, + ) as client: + client.login() + profile = client.get_profile() + print("profile:", profile.get("userName"), profile.get("email")) + + game = client.get_game(args.game) + print( + "game:", + game.get("id"), + game.get("title"), + "status=", game.get("status"), + "team=", game.get("teamName"), + ) + + check = client.get_game_check(args.game) + print("check:", check) + + scoreboard = client.get_scoreboard(args.game) + challenges = scoreboard.get("challenges", {}) + challenge_count = sum(len(items) for items in challenges.values()) + print("scoreboard challenge_count:", challenge_count) + for category, items in challenges.items(): + print(category, [(item.get("id"), item.get("title")) for item in items][:10]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_gzctf.py b/tests/test_gzctf.py new file mode 100644 index 0000000..c5f944f --- /dev/null +++ b/tests/test_gzctf.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import pytest + +from backend.platform.gzctf import ( + GZCTFClient, + GZCTFLoginError, + GZCTFPreflightError, + validate_submission, +) + + +class FakeResponse: + def __init__(self, status_code=200, payload=None): + self.status_code = status_code + self._payload = payload + + def json(self): + return self._payload + + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + + +class FakeSession: + def __init__(self): + self.calls = [] + self.closed = False + + def post(self, url, **kwargs): + self.calls.append(("post", url, kwargs)) + if url.endswith("/api/Account/LogIn"): + if kwargs["json"]["password"] != "correct": + return FakeResponse( + 401, + {"title": "Wrong username or password", "status": 401}, + ) + return FakeResponse(200, None) + if url.endswith("/Challenges/7"): + return FakeResponse(200, {"id": 1234, "isTrackComplete": True}) + return FakeResponse(200, {"ok": True}) + + def get(self, url, **kwargs): + self.calls.append(("get", url, kwargs)) + if url.endswith("/api/Account/Profile"): + return FakeResponse(200, {"userName": "tester"}) + return FakeResponse(200, {"ok": True}) + + def close(self): + self.closed = True + + +def test_gzctf_login_failure_raises(monkeypatch): + monkeypatch.setattr("backend.platform.gzctf.requests.Session", FakeSession) + client = GZCTFClient( + base_url="https://ctf.test", + username="user", + password="bad", + ) + with pytest.raises(GZCTFLoginError, match="Wrong username"): + client.login() + client.close() + + +def test_gzctf_authenticated_flow(monkeypatch): + monkeypatch.setattr("backend.platform.gzctf.requests.Session", FakeSession) + client = GZCTFClient( + base_url="https://ctf.test", + username="user", + password="correct", + ) + profile = client.get_profile() + details = client.get_game_details(2) + check = client.get_game_check(2) + submission = client.submit_flag( + 2, + 7, + "flag{demo}", + level=2, + track_id="track-1", + ) + status = client.get_submission_status(2, 7, submission["id"]) + + assert profile["userName"] == "tester" + assert details == {"ok": True} + assert check == {"ok": True} + assert submission["id"] == 1234 + assert status == {"ok": True} + urls = [call[1] for call in client.session.calls] + assert any(url.endswith("/api/Account/LogIn") for url in urls) + assert any(url.endswith("/api/Game/2/Check") for url in urls) + assert any(url.endswith("/api/Game/2/Challenges/7") for url in urls) + assert any(url.endswith("/api/Game/2/Challenges/7/Status/1234") for url in urls) + submit_call = next( + call + for call in client.session.calls + if call[1].endswith("/api/Game/2/Challenges/7") + ) + assert submit_call[2]["json"] == { + "flag": "flag{demo}", + "level": 2, + "trackId": "track-1", + } + client.close() + + +def _challenge_state(*, multi=True, current_level=1, cooldown=None): + tracks = [ + { + "trackId": "track-1", + "isEnabled": True, + "currentLevel": current_level, + "levels": [ + {"level": 1, "status": "Unsolved"}, + {"level": 2, "status": "Locked"}, + {"level": 3, "status": "Locked"}, + ], + } + ] + if multi: + tracks.append( + { + "trackId": "track-2", + "isEnabled": True, + "currentLevel": current_level, + "levels": [{"level": 1, "status": "Unsolved"}], + } + ) + return {"id": 143, "challengeLock": False, "tracks": tracks, "cooldown": cooldown or {}} + + +def test_gzctf_preflight_requires_track_and_strips_answer(): + with pytest.raises(GZCTFPreflightError, match="multiple enabled tracks"): + validate_submission( + _challenge_state(), + level=1, + answer="flag{demo}", + ) + with pytest.raises(GZCTFPreflightError, match="answer must not be empty"): + validate_submission( + _challenge_state(multi=False), + level=1, + answer=" ", + ) + decision = validate_submission( + _challenge_state(), + level=1, + answer=" flag{demo} ", + track_id="track-1", + evidence={ + "status": "verified", + "answers": {"1": "flag{demo}"}, + }, + ) + assert decision["answer"] == "flag{demo}" + assert decision["track_id"] == "track-1" + + +def test_gzctf_preflight_refuses_locked_and_cooldown_levels(): + with pytest.raises(GZCTFPreflightError, match="locked"): + validate_submission( + _challenge_state(multi=False), + level=2, + answer="flag{demo}", + ) + with pytest.raises(GZCTFPreflightError, match="wrong-answer limit"): + validate_submission( + _challenge_state( + multi=False, + cooldown={"wrongCount": 10, "wrongAnswerLimit": 10}, + ), + level=1, + answer="flag{demo}", + ) + + +def test_gzctf_preflight_accepts_unlocked_status_object(): + challenge = _challenge_state(multi=False) + challenge["challengeLock"] = { + "isLocked": False, + "unlocksAt": None, + "triggerAnswer": None, + } + decision = validate_submission( + challenge, + level=1, + answer="flag{demo}", + evidence={ + "status": "verified", + "answers": {"1": "flag{demo}"}, + }, + ) + assert decision["ok"] is True + + +def test_gzctf_preflight_refuses_locked_status_object(): + challenge = _challenge_state(multi=False) + challenge["challengeLock"] = {"isLocked": True, "reason": "banned"} + with pytest.raises(GZCTFPreflightError, match="currently locked"): + validate_submission( + challenge, + level=1, + answer="services/foo.c:1", + ) From acba260871b701f2b89b7d3bcf6f83ca49c54f3c Mon Sep 17 00:00:00 2001 From: PureStream108 Date: Mon, 31 Aug 2026 22:17:49 +0800 Subject: [PATCH 02/10] feat(platform): add ret2shell client with PoW/rate-limit/instance MCP - Ret2ShellClient: JWT login with PoW captcha, 10-per-5min submit limiter, async judge polling, dynamic instance lifecycle. - ret2shell MCP (instance_start/status/renew/stop, challenge_status) with wsrx tunnel management for ws-proxied challenge ports; wsrx is installed in the app image and IPC_R2S_* env vars are wired through docker-compose. - FieldMapping/ChallengeMappingSpec gain platform=http_json|ret2shell and game_id; credentials are read from IPC_R2S_* env vars only. - The MCP is registered for members only when credentials are present. - conftest honors an explicit @pytest.mark.postgres outside the DB-backed module list so mixed unit/integration modules can mark single tests. --- Dockerfile | 13 + backend/core/state.py | 7 + backend/ops/models.py | 14 +- backend/ops/service.py | 14 +- backend/platform/mapping.py | 19 +- backend/platform/ret2shell.py | 596 +++++++++++++++++++++++ backend/platform/ret2shell_mcp.py | 354 ++++++++++++++ backend/tools/member_tools.txt | 13 + docker-compose.yml | 7 + docs/ret2shell.md | 87 ++++ scripts/fetch_ret2shell_targets.py | 74 +++ scripts/ret2shell_platform_check.py | 50 ++ scripts/ret2shell_submit.py | 74 +++ tests/conftest.py | 7 +- tests/test_ret2shell.py | 704 ++++++++++++++++++++++++++++ 15 files changed, 2025 insertions(+), 8 deletions(-) create mode 100644 backend/platform/ret2shell.py create mode 100644 backend/platform/ret2shell_mcp.py create mode 100644 docs/ret2shell.md create mode 100644 scripts/fetch_ret2shell_targets.py create mode 100644 scripts/ret2shell_platform_check.py create mode 100644 scripts/ret2shell_submit.py create mode 100644 tests/test_ret2shell.py diff --git a/Dockerfile b/Dockerfile index ea2da7e..229bac7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,19 @@ COPY scripts /app/scripts RUN pip install --no-cache-dir -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple -e ".[docker]" +# wsrx (WebSocket Reflector X CLI from XDSEC, static musl build) tunnels the +# ret2shell platform's ws://-proxied challenge ports to local TCP ports so +# Members can reach dynamic instances over the docker network. The ret2shell +# backend itself is only installed in this image because the tunnel +# subprocesses are owned by the backend process (Members just connect to +# ipc-app:). +ARG WSRX_VERSION=0.6.1 +RUN python -c "import urllib.request; urllib.request.urlretrieve('https://github.com/XDSEC/WebSocketReflectorX/releases/download/${WSRX_VERSION}/wsrx-cli-${WSRX_VERSION}-linux-musl-x86_64.tar.gz', '/tmp/wsrx.tar.gz')" \ + && tar -xzf /tmp/wsrx.tar.gz -C /tmp \ + && install -m 0755 /tmp/wsrx /usr/local/bin/wsrx \ + && rm -f /tmp/wsrx /tmp/wsrx.tar.gz \ + && wsrx --version + # /app/data is bind-mounted by docker-compose.yml. PostgreSQL owns runtime # facts; filesystem workspaces and generated files share one Artifact tree. RUN mkdir -p /app/data/artifacts/projects \ diff --git a/backend/core/state.py b/backend/core/state.py index 9d5d50a..5395555 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -14,6 +14,8 @@ from backend.mcp.mcp_client import MCPRegistry from backend.memory.memory_mcp import build_memory_mcp from backend.memory.memory_store import MemoryStore +from backend.platform.ret2shell import Ret2ShellClient +from backend.platform.ret2shell_mcp import build_ret2shell_mcp from backend.sandbox.container_pool import ContainerPool from backend.sandbox.network_manager import NetworkManager from backend.sandbox.resource_limiter import TaskSlotLimiter @@ -96,6 +98,11 @@ def __init__(self, root: str | Path = ".", config_dir: Path | None = None): self.mcps = MCPRegistry() self.mcps.register(build_memory_mcp(self.memory, catalog=self.catalog)) self.mcps.register(build_tool_search_mcp(self.registry)) + # The ret2shell competition MCP (dynamic instance control) is only + # registered when participant credentials are configured, so members + # never see a platform they cannot reach. + if os.getenv("IPC_R2S_USERNAME") or os.getenv("IPC_R2S_TOKEN"): + self.mcps.register(build_ret2shell_mcp(Ret2ShellClient())) # Attached by module 8. self.orchestrator = None diff --git a/backend/ops/models.py b/backend/ops/models.py index b004c72..a8a22a8 100644 --- a/backend/ops/models.py +++ b/backend/ops/models.py @@ -82,7 +82,9 @@ def validate_prefix(cls, value: str) -> str: class ChallengeMappingSpec(BaseModel): model_config = ConfigDict(extra="forbid") - list_url: str + platform: Literal["http_json", "ret2shell"] = "http_json" + game_id: int | None = None + list_url: str = "" list_path: str = "data" id_field: str = "id" title_field: str = "name" @@ -96,7 +98,13 @@ class ChallengeMappingSpec(BaseModel): @field_validator("list_url") @classmethod def validate_list_url(cls, value: str) -> str: - return _validate_http_url(value) + return _validate_http_url(value) if value.strip() else "" + + @model_validator(mode="after") + def require_list_url_for_http_json(self) -> ChallengeMappingSpec: + if self.platform == "http_json" and not self.list_url.strip(): + raise ValueError("list_url is required for the http_json platform") + return self @field_validator("attachment_base_url") @classmethod @@ -127,6 +135,8 @@ def unique_headers(self) -> ChallengeMappingSpec: def to_field_mapping(self, headers: dict[str, str]) -> FieldMapping: return FieldMapping( + platform=self.platform, + game_id=self.game_id, list_url=self.list_url, list_path=self.list_path, id_field=self.id_field, diff --git a/backend/ops/service.py b/backend/ops/service.py index 573c7fa..2c5f837 100644 --- a/backend/ops/service.py +++ b/backend/ops/service.py @@ -25,7 +25,8 @@ from backend.ops.claude_runner import ClaudeCodeRunner, ClaudeCodeRunnerError from backend.ops.store import OpsStore from backend.ops.tools import OpsToolError, OpsToolExecutor, tool_prompt -from backend.platform.adapter import HttpJsonAdapter +from backend.platform.adapter import HttpJsonAdapter, PlatformAdapter +from backend.platform.ret2shell import Ret2ShellAdapter, Ret2ShellClient _JSON_DECODER = json.JSONDecoder() _MAX_TOOL_ROUNDS = 8 @@ -1237,10 +1238,19 @@ def workflow_view(self, workflow: dict[str, Any]) -> dict[str, Any]: "confirmation_phrase": f"CONFIRM WORKFLOW {workflow['id']}", } - def _adapter(self, workflow_id: str, spec: PlatformWorkflowSpec) -> HttpJsonAdapter: + def _adapter(self, workflow_id: str, spec: PlatformWorkflowSpec) -> PlatformAdapter: secrets_values = self.store.workflow_secrets(workflow_id) headers = _resolve_headers(spec.challenges.headers, secrets_values) mapping = spec.challenges.to_field_mapping(headers) + if mapping.platform == "ret2shell": + # Credentials come from IPC_R2S_* environment variables inside the + # backend process; the workflow spec carries no secrets. + return Ret2ShellAdapter( + Ret2ShellClient(game_id=mapping.game_id), + game_id=mapping.game_id or None, + category_map=mapping.category_map, + max_attachment_bytes=spec.max_attachment_bytes, + ) return HttpJsonAdapter( mapping, request_get=self._http_client(spec).get, diff --git a/backend/platform/mapping.py b/backend/platform/mapping.py index b1c3a77..44fda79 100644 --- a/backend/platform/mapping.py +++ b/backend/platform/mapping.py @@ -1,12 +1,19 @@ from __future__ import annotations -from pydantic import BaseModel, ConfigDict, Field, field_validator +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class FieldMapping(BaseModel): model_config = ConfigDict(extra="forbid") - list_url: str + # ``http_json`` fetches any JSON list endpoint with static headers. + # ``ret2shell`` uses the authenticated ret2shell client (credentials come + # from IPC_R2S_* environment variables, never from the request body). + platform: Literal["http_json", "ret2shell"] = "http_json" + game_id: int | None = None + list_url: str = "" list_path: str = "data" id_field: str = "id" title_field: str = "name" @@ -21,10 +28,16 @@ class FieldMapping(BaseModel): @classmethod def validate_list_url(cls, value: str) -> str: value = value.strip() - if not value.startswith(("http://", "https://")): + if value and not value.startswith(("http://", "https://")): raise ValueError("list_url must use http or https") return value + @model_validator(mode="after") + def require_list_url_for_http_json(self) -> FieldMapping: + if self.platform == "http_json" and not self.list_url: + raise ValueError("list_url is required for the http_json platform") + return self + class PlatformChallenge(BaseModel): external_id: str diff --git a/backend/platform/ret2shell.py b/backend/platform/ret2shell.py new file mode 100644 index 0000000..bdc1f26 --- /dev/null +++ b/backend/platform/ret2shell.py @@ -0,0 +1,596 @@ +from __future__ import annotations + +import hashlib +import os +import threading +import time +from pathlib import Path +from typing import Any, Callable + +import requests + +from backend.core.config import CATEGORIES +from backend.filename_util import numbered_filename, safe_stem +from backend.platform.adapter import PlatformAdapter +from backend.platform.mapping import PlatformChallenge + +DEFAULT_BASE_URL = "https://ctf.xidian.edu.cn" +DEFAULT_GAME_ID = 37 + +# ret2shell limits each account to 10 flag submissions per 5-minute window +# (HTTP 429). The client-side limiter guards that shared quota. +SUBMIT_WINDOW_SECONDS = 300.0 +SUBMIT_LIMIT = 10 +# ret2shell judges submissions asynchronously: the POST returns a Submission +# with ``solved: null`` and the result must be polled. +SUBMIT_POLL_ATTEMPTS = 7 +SUBMIT_POLL_INTERVAL = 1.0 +# Instance exposed_ports appear asynchronously after a start request. +INSTANCE_WAIT_TIMEOUT = 60.0 +INSTANCE_WAIT_INTERVAL = 2.0 + + +class Ret2ShellError(RuntimeError): + pass + + +class Ret2ShellAuthError(Ret2ShellError): + pass + + +class Ret2ShellPreflightError(Ret2ShellError): + """A submission was refused before it could consume a platform attempt.""" + + +class Ret2ShellRateLimitError(Ret2ShellError): + pass + + +class _SubmitRateLimiter: + """Guard the shared 10-submissions-per-5-minutes account quota.""" + + def __init__( + self, + *, + limit: int = SUBMIT_LIMIT, + window: float = SUBMIT_WINDOW_SECONDS, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self.limit = limit + self.window = window + self._monotonic = monotonic + self._lock = threading.Lock() + self._attempts: list[float] = [] + + def _trim(self, now: float) -> None: + self._attempts = [ts for ts in self._attempts if now - ts < self.window] + + def reserve(self) -> None: + with self._lock: + now = self._monotonic() + self._trim(now) + if len(self._attempts) >= self.limit: + oldest = min(self._attempts) + retry_in = self.window - (now - oldest) + raise Ret2ShellRateLimitError( + f"local submit quota exhausted ({self.limit} per " + f"{int(self.window)}s); retry in {retry_in:.0f}s" + ) + self._attempts.append(now) + + def used(self) -> int: + with self._lock: + self._trim(self._monotonic()) + return len(self._attempts) + + +def _error_detail(response: requests.Response) -> str: + # ret2shell errors are plain-text bodies, not JSON. + detail = response.text.strip().replace("\r", " ").replace("\n", " ") + if len(detail) > 500: + detail = detail[:500] + "..." + return f"{response.status_code}: {detail}" if detail else str(response.status_code) + + +class Ret2ShellClient: + """ret2shell (Rust/axum) participant API client. + + Auth is a JWT Bearer token issued by ``POST /api/account/login`` and + returned in the ``Set-Token`` response header (never in the body). Any + later response may carry a refreshed ``Set-Token``, so every response is + checked and the token rolled forward. All timestamps are unix seconds + and list endpoints return ``[items, total]`` tuples. + """ + + def __init__( + self, + base_url: str = "", + game_id: int | None = None, + username: str = "", + password: str = "", + token: str = "", + *, + timeout: float = 30, + session: requests.Session | None = None, + poll_attempts: int = SUBMIT_POLL_ATTEMPTS, + poll_interval: float = SUBMIT_POLL_INTERVAL, + sleep: Callable[[float], None] = time.sleep, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self.base_url = (base_url or os.getenv("IPC_R2S_BASE_URL", "") or DEFAULT_BASE_URL).rstrip("/") + raw_game_id = game_id if game_id is not None else os.getenv("IPC_R2S_GAME_ID", "") + try: + self.game_id = int(raw_game_id) if str(raw_game_id).strip() else DEFAULT_GAME_ID + except (TypeError, ValueError) as exc: + raise Ret2ShellError(f"invalid IPC_R2S_GAME_ID: {raw_game_id!r}") from exc + self.username = username or os.getenv("IPC_R2S_USERNAME", "") + self.password = password or os.getenv("IPC_R2S_PASSWORD", "") + self.timeout = timeout + self.session = session or requests.Session() + self.poll_attempts = max(1, poll_attempts) + self.poll_interval = poll_interval + self._sleep = sleep + self._monotonic = monotonic + self._token = token or os.getenv("IPC_R2S_TOKEN", "") + self._limiter = _SubmitRateLimiter(monotonic=monotonic) + + # ---- low-level request handling ---- + + def _url(self, path: str) -> str: + return f"{self.base_url}/api{path}" + + def _absorb_token(self, response: requests.Response) -> None: + token = response.headers.get("Set-Token") + if token: + self._token = token + + def _headers(self) -> dict[str, str]: + headers: dict[str, str] = {} + if self._token: + headers["Authorization"] = f"Bearer {self._token}" + return headers + + def _request( + self, + method: str, + path: str, + *, + json: Any = None, + params: dict[str, Any] | None = None, + stream: bool = False, + _relogin: bool = True, + ) -> requests.Response: + if not self._token: + self.login() + response = self.session.request( + method, + self._url(path), + json=json, + params=params, + headers=self._headers(), + timeout=self.timeout, + stream=stream, + ) + self._absorb_token(response) + if response.status_code == 401 and _relogin: + self._token = "" + self.login() + return self._request( + method, path, json=json, params=params, stream=stream, _relogin=False + ) + return response + + def _request_ok( + self, + method: str, + path: str, + *, + json: Any = None, + params: dict[str, Any] | None = None, + ) -> Any: + response = self._request(method, path, json=json, params=params) + if response.status_code >= 400: + raise Ret2ShellError(_error_detail(response)) + if not response.content: + return None + try: + return response.json() + except ValueError as exc: + raise Ret2ShellError( + f"unexpected non-JSON response from {path}: {response.text[:200]!r}" + ) from exc + + # ---- auth ---- + + @staticmethod + def solve_pow(challenge: str, *, max_attempts: int = 5_000_000) -> str: + """Solve a ret2shell CLI proof-of-work captcha. + + The server issues ``"#"`` and accepts any answer + that starts with ```` and whose SHA-256 hex digest starts + with ``difficulty`` zero characters (crates/captcha/src/pow.rs). + """ + + difficulty_raw, _, prefix = challenge.partition("#") + try: + difficulty = int(difficulty_raw) + except ValueError as exc: + raise Ret2ShellError(f"invalid pow challenge: {challenge!r}") from exc + target = "0" * difficulty + for counter in range(max_attempts): + answer = f"{prefix}{counter}" + if hashlib.sha256(answer.encode()).hexdigest().startswith(target): + return answer + raise Ret2ShellError(f"pow challenge not solved within {max_attempts} attempts") + + def _fresh_captcha(self) -> dict[str, str]: + # The CLI endpoint always issues a PoW (difficulty 4), which the + # agent can solve autonomously — unlike the browser image captcha. + response = self.session.get(self._url("/account/captcha/cli"), timeout=self.timeout) + if response.status_code != 200: + raise Ret2ShellAuthError(f"captcha fetch failed: {_error_detail(response)}") + payload = response.json() + if not isinstance(payload, dict) or not payload.get("id"): + raise Ret2ShellAuthError("captcha response has no id") + challenge = str(payload.get("challenge", "")) + # A non-PoW challenge (image/hcaptcha) cannot be solved by the agent; + # pass it through and let login fail with the platform's reason. + answer = self.solve_pow(challenge) if "#" in challenge else "" + return {"captcha_id": str(payload["id"]), "captcha_answer": answer} + + def login(self) -> None: + if not self.username or not self.password: + raise Ret2ShellAuthError( + "ret2shell username/password are not configured; " + "set IPC_R2S_USERNAME/IPC_R2S_PASSWORD or IPC_R2S_TOKEN" + ) + body = { + "account": self.username, + "password": self.password, + "captcha_id": "", + "captcha_answer": "", + } + # The platform may enable captchas (410 "captcha is outdate" otherwise). + # The CLI PoW endpoint works headlessly, so try to pre-solve one; if + # the deployment disabled captchas the fields are simply ignored. + try: + body.update(self._fresh_captcha()) + except Ret2ShellAuthError: + pass + response = self.session.post( + self._url("/account/login"), + json=body, + timeout=self.timeout, + ) + self._absorb_token(response) + if response.status_code != 200 or not self._token: + raise Ret2ShellAuthError(f"ret2shell login failed: {_error_detail(response)}") + + # ---- games / challenges ---- + + def ping(self) -> str: + response = self.session.get(self._url("/ping"), timeout=self.timeout) + if response.status_code != 200: + raise Ret2ShellError(_error_detail(response)) + return response.text + + def get_profile(self) -> dict[str, Any]: + return self._request_ok("GET", "/account/profile") + + def get_game(self, game_id: int | None = None) -> dict[str, Any]: + return self._request_ok("GET", f"/game/{game_id or self.game_id}") + + def list_games(self) -> list[dict[str, Any]]: + payload = self._request_ok("GET", "/game") + items = payload[0] if isinstance(payload, list) and payload and isinstance(payload[0], list) else payload + return items if isinstance(items, list) else [] + + def list_challenges(self, game_id: int | None = None) -> list[dict[str, Any]]: + payload = self._request_ok("GET", f"/game/{game_id or self.game_id}/challenge") + if not (isinstance(payload, list) and payload and isinstance(payload[0], list)): + raise Ret2ShellError("challenge list response is not an [items, total] tuple") + return payload[0] + + def get_challenge(self, challenge_id: int, game_id: int | None = None) -> dict[str, Any]: + return self._request_ok( + "GET", f"/game/{game_id or self.game_id}/challenge/{challenge_id}" + ) + + def challenge_status(self, challenge_id: int, game_id: int | None = None) -> dict[str, Any]: + """Own solve state plus the platform-wide solve count.""" + + return self._request_ok( + "GET", f"/game/{game_id or self.game_id}/challenge/{challenge_id}/submit" + ) + + def has_environment(self, challenge_id: int, game_id: int | None = None) -> bool: + """Whether the challenge ships a dynamic instance environment. + + The env endpoint returns ``null`` for plain challenges and an + ``{"images": [...]}`` object for those with an online environment. + """ + + response = self._request( + "GET", f"/game/{game_id or self.game_id}/challenge/{challenge_id}/env" + ) + if response.status_code >= 400: + return False + try: + payload = response.json() + except ValueError: + return False + return bool(payload) + + # ---- attachments ---- + + def list_files(self, challenge_id: int, game_id: int | None = None) -> list[dict[str, Any]]: + payload = self._request_ok( + "GET", f"/game/{game_id or self.game_id}/challenge/{challenge_id}/file" + ) + if not isinstance(payload, list): + raise Ret2ShellError("challenge file response is not a list") + return [item for item in payload if isinstance(item, dict)] + + def download_file( + self, + challenge_id: int, + folder: str, + file: str, + dest_dir: str | Path, + game_id: int | None = None, + *, + max_bytes: int | None = None, + ) -> Path: + destination = Path(dest_dir) + destination.mkdir(parents=True, exist_ok=True) + existing = [path.name for path in destination.iterdir()] + source = Path(file) + stem = safe_stem(source.stem, fallback="attachment") + suffix = source.suffix[:20] + target = destination / numbered_filename( + stem, suffix or ".bin", existing, fallback="attachment" + ) + response = self._request( + "GET", + f"/game/{game_id or self.game_id}/challenge/{challenge_id}/file", + params={"folder": folder, "file": file}, + stream=True, + ) + if response.status_code >= 400: + raise Ret2ShellError(_error_detail(response)) + written = 0 + try: + with target.open("wb") as handle: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + written += len(chunk) + if max_bytes is not None and written > max_bytes: + raise Ret2ShellError( + f"attachment exceeds {max_bytes} byte limit: {file}" + ) + handle.write(chunk) + except Exception: + target.unlink(missing_ok=True) + raise + return target + + # ---- dynamic instances ---- + + def list_instances(self, game_id: int | None = None) -> list[dict[str, Any]]: + payload = self._request_ok("GET", f"/game/{game_id or self.game_id}/instance") + return payload if isinstance(payload, list) else [] + + def find_instance( + self, challenge_id: int, game_id: int | None = None + ) -> dict[str, Any] | None: + for instance in self.list_instances(game_id): + if instance.get("challenge_id") == challenge_id: + return instance + return None + + def start_instance(self, challenge_id: int, game_id: int | None = None) -> None: + response = self._request( + "POST", f"/game/{game_id or self.game_id}/challenge/{challenge_id}/instance" + ) + if response.status_code >= 400: + raise Ret2ShellError(_error_detail(response)) + + def renew_instance(self, challenge_id: int, game_id: int | None = None) -> None: + response = self._request( + "PATCH", f"/game/{game_id or self.game_id}/challenge/{challenge_id}/instance" + ) + if response.status_code >= 400: + raise Ret2ShellError(_error_detail(response)) + + def destroy_instance(self, challenge_id: int, game_id: int | None = None) -> None: + response = self._request( + "DELETE", f"/game/{game_id or self.game_id}/challenge/{challenge_id}/instance" + ) + if response.status_code >= 400: + raise Ret2ShellError(_error_detail(response)) + + def wait_for_instance( + self, + challenge_id: int, + game_id: int | None = None, + *, + timeout: float = INSTANCE_WAIT_TIMEOUT, + interval: float = INSTANCE_WAIT_INTERVAL, + ) -> dict[str, Any]: + """Poll until the challenge instance reports reachable exposed_ports.""" + + deadline = self._monotonic() + timeout + last: dict[str, Any] | None = None + while True: + instance = self.find_instance(challenge_id, game_id) + if instance is not None: + last = instance + if instance.get("exposed_ports"): + return instance + if self._monotonic() >= deadline: + state = (last or {}).get("state", "unknown") + raise Ret2ShellError( + f"instance for challenge {challenge_id} did not become reachable " + f"within {timeout:.0f}s (last state: {state})" + ) + self._sleep(interval) + + # ---- flag submission (async judged) ---- + + def get_submission( + self, + challenge_id: int, + submission_id: int, + game_id: int | None = None, + ) -> dict[str, Any]: + return self._request_ok( + "GET", + f"/game/{game_id or self.game_id}/challenge/{challenge_id}/submit", + params={"id": submission_id}, + ) + + def submit_flag( + self, + challenge_id: int, + flag: str, + game_id: int | None = None, + *, + check_solved: bool = True, + ) -> dict[str, Any]: + """Submit one flag and poll the async judge for its verdict. + + Preflight refuses to POST when the challenge is already solved (no + platform attempt should be spent) and the local limiter protects the + shared 10-per-5-minutes quota. Returns the final Submission dict + with ``solved`` and ``result`` filled in when the judge answered in + time; ``solved`` stays ``None`` on a poll timeout. + """ + + flag = str(flag).strip() + if not flag: + raise Ret2ShellPreflightError("flag must not be empty") + if check_solved: + status = self.challenge_status(challenge_id, game_id) + if isinstance(status, dict) and status.get("solved"): + raise Ret2ShellPreflightError( + f"challenge {challenge_id} is already solved; no submission was sent" + ) + self._limiter.reserve() + response = self._request( + "POST", + f"/game/{game_id or self.game_id}/challenge/{challenge_id}/submit", + json={"content": flag}, + ) + if response.status_code == 429: + raise Ret2ShellRateLimitError(_error_detail(response)) + if response.status_code >= 400: + raise Ret2ShellError(_error_detail(response)) + submission = response.json() if response.content else {} + submission_id = submission.get("id") + if submission_id is None: + raise Ret2ShellError(f"submit response has no submission id: {submission}") + for _ in range(self.poll_attempts): + if submission.get("solved") is not None: + return submission + self._sleep(self.poll_interval) + submission = self.get_submission(challenge_id, submission_id, game_id) + return submission + + # ---- lifecycle ---- + + def close(self) -> None: + self.session.close() + + def __enter__(self) -> Ret2ShellClient: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + +def primary_tag(challenge: dict[str, Any]) -> str: + tags = challenge.get("tag") + if isinstance(tags, list): + for tag in tags: + if isinstance(tag, dict) and tag.get("name"): + return str(tag["name"]) + if tags and isinstance(tags[0], str): + return str(tags[0]) + return "" + + +def normalize_category(raw: str) -> str: + lowered = raw.strip().lower() + return lowered if lowered in CATEGORIES else "misc" + + +class Ret2ShellAdapter(PlatformAdapter): + """Import ret2shell challenges through the generic platform mechanism.""" + + def __init__( + self, + client: Ret2ShellClient, + *, + game_id: int | None = None, + category_map: dict[str, str] | None = None, + max_attachment_bytes: int | None = None, + ) -> None: + self.client = client + self.game_id = game_id or client.game_id + self.category_map = category_map or {} + self.max_attachment_bytes = max_attachment_bytes + + def fetch_challenges(self) -> list[PlatformChallenge]: + challenges: list[PlatformChallenge] = [] + for raw in self.client.list_challenges(self.game_id): + if not isinstance(raw, dict) or raw.get("id") is None: + continue + # The list endpoint returns content: null; the full statement + # (connection commands, flag format hints) only comes from the + # per-challenge detail endpoint. + content = raw.get("content") + if not content: + try: + content = self.client.get_challenge(int(raw["id"]), self.game_id).get("content") + except Ret2ShellError: + content = None + raw_category = primary_tag(raw) + mapped = self.category_map.get(raw_category, raw_category) + challenges.append( + PlatformChallenge( + external_id=str(raw["id"]), + title=str(raw.get("name", "")), + category=normalize_category(mapped), + description=str(content or ""), + attachment_urls=[], + ) + ) + return challenges + + def download_attachments( + self, + challenge: PlatformChallenge, + dest_dir: str | Path, + ) -> list[Path]: + # ret2shell attachments are listed per challenge via the file API, so + # there are no attachment URLs at fetch time. + destination = Path(dest_dir) + destination.mkdir(parents=True, exist_ok=True) + downloaded: list[Path] = [] + for item in self.client.list_files(int(challenge.external_id), self.game_id): + folder = str(item.get("folder", "static")) + file = item.get("file") + if not file: + continue + downloaded.append( + self.client.download_file( + int(challenge.external_id), + folder, + str(file), + destination, + self.game_id, + max_bytes=self.max_attachment_bytes, + ) + ) + return downloaded diff --git a/backend/platform/ret2shell_mcp.py b/backend/platform/ret2shell_mcp.py new file mode 100644 index 0000000..7f024ce --- /dev/null +++ b/backend/platform/ret2shell_mcp.py @@ -0,0 +1,354 @@ +from __future__ import annotations + +import asyncio +import os +import shutil +import socket +import subprocess +import threading +import time +from typing import Any, Callable +from urllib.parse import urlparse + +from backend.mcp.mcp_server import MCPServer, close_lifespan, create_mcp_server +from backend.platform.ret2shell import Ret2ShellClient + +# ret2shell does not expose direct host:port pairs: each running instance +# carries a ``traffic`` token plus the container ``ports``, and the platform +# web app connects through wsrx (WebSocket Reflector X, XDSEC) links of the +# form ``wss:///api/traffic/?port=``. The backend +# keeps one ``wsrx connect`` subprocess per (challenge, port) alive so that +# Members — running in their own containers on the same docker network — can +# simply connect to ``ipc-app:``. +WSRX_BINARY = os.getenv("IPC_R2S_WSRX_BINARY", "wsrx") +WSRX_BIND_HOST = os.getenv("IPC_R2S_WSRX_BIND_HOST", "0.0.0.0") +WSRX_ENDPOINT_HOST = os.getenv("IPC_R2S_WSRX_ENDPOINT_HOST", "ipc-app") +WSRX_BASE_PORT = int(os.getenv("IPC_R2S_WSRX_BASE_PORT", "20000")) +WSRX_PORT_SPAN = 10000 +WSRX_STARTUP_TIMEOUT = 15.0 +WSRX_STARTUP_INTERVAL = 0.3 + + +def local_wsrx_port(challenge_id: int, index: int = 0) -> int: + """Deterministic local port for a challenge's nth tunneled port.""" + + return WSRX_BASE_PORT + (challenge_id % WSRX_PORT_SPAN) + index + + +def _platform_host(client: Ret2ShellClient) -> str: + return urlparse(client.base_url).netloc + + +def _ws_scheme(client: Ret2ShellClient) -> str: + return "wss" if urlparse(client.base_url).scheme == "https" else "ws" + + +def wsrx_remotes(instance: dict[str, Any], client: Ret2ShellClient) -> list[str]: + """wsrx websocket URLs for one instance. + + Handles both shapes: explicit ``ws://``/``wss://`` entries inside + ``exposed_ports`` (GZCTF-style platform proxy) and the ret2shell shape + where the instance carries a ``traffic`` token plus ``ports``. + """ + + remotes: list[str] = [] + exposed = instance.get("exposed_ports") + if isinstance(exposed, list): + for entry in exposed: + if isinstance(entry, str) and entry.startswith(("ws://", "wss://")): + remotes.append(entry) + traffic = instance.get("traffic") + ports = instance.get("ports") + if traffic and isinstance(ports, list): + scheme = _ws_scheme(client) + host = _platform_host(client) + for port in ports: + remotes.append(f"{scheme}://{host}/api/traffic/{traffic}?port={port}") + return remotes + + +class _Tunnel: + def __init__(self, remote: str, local_port: int, process: Any) -> None: + self.remote = remote + self.local_port = local_port + self.process = process + + @property + def alive(self) -> bool: + return self.process.poll() is None + + +class WsrxTunnelManager: + """Own the ``wsrx connect`` subprocesses, keyed by challenge id. + + ``ensure`` is idempotent: a challenge whose tunnels are still running for + the same set of remotes is reused instead of spawning duplicates. The + spawner/probe are injectable so tests can run without the wsrx binary. + """ + + def __init__( + self, + *, + binary: str = WSRX_BINARY, + bind_host: str = WSRX_BIND_HOST, + endpoint_host: str = WSRX_ENDPOINT_HOST, + spawner: Callable[..., Any] | None = None, + probe: Callable[[int], bool] | None = None, + startup_timeout: float = WSRX_STARTUP_TIMEOUT, + startup_interval: float = WSRX_STARTUP_INTERVAL, + terminator: Callable[[Any], None] | None = None, + ) -> None: + self.binary = binary + self.bind_host = bind_host + self.endpoint_host = endpoint_host + self.startup_timeout = startup_timeout + self.startup_interval = startup_interval + self._spawner = spawner or self._spawn_process + self._probe = probe or self._probe_port + self._terminator = terminator or self._terminate_process + self._lock = threading.Lock() + self._tunnels: dict[int, list[_Tunnel]] = {} + + # ---- default subprocess plumbing ---- + + def _spawn_process(self, remote: str, local_port: int) -> Any: + binary = shutil.which(self.binary) + if binary is None: + raise RuntimeError( + f"wsrx binary {self.binary!r} not found on PATH; cannot tunnel {remote}" + ) + return subprocess.Popen( + [ + binary, + "connect", + "--host", + self.bind_host, + "--port", + str(local_port), + remote, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + start_new_session=True, + ) + + @staticmethod + def _probe_port(local_port: int) -> bool: + try: + with socket.create_connection(("127.0.0.1", local_port), timeout=1.0): + return True + except OSError: + return False + + @staticmethod + def _terminate_process(process: Any) -> None: + try: + process.terminate() + try: + process.wait(timeout=5) + except Exception: + process.kill() + except Exception: + pass + + # ---- public API ---- + + def _alive(self, challenge_id: int) -> list[_Tunnel]: + tunnels = [t for t in self._tunnels.get(challenge_id, []) if t.alive] + self._tunnels[challenge_id] = tunnels + return tunnels + + def ensure(self, challenge_id: int, remotes: list[str]) -> dict[str, Any]: + """Guarantee one live tunnel per remote; return local endpoints. + + Returns ``{"endpoints": [...], "started": [...]}`` where endpoints are + ``:`` strings reachable from Member containers. + Raises RuntimeError when a tunnel cannot be established. + """ + + with self._lock: + existing = self._alive(challenge_id) + needed = list(remotes) + tunnels: list[_Tunnel] = [] + for tunnel in existing: + if tunnel.remote in needed: + needed.remove(tunnel.remote) + tunnels.append(tunnel) + else: + self._terminator(tunnel.process) + started: list[str] = [] + for remote in needed: + local_port = local_wsrx_port(challenge_id, remotes.index(remote)) + process = self._spawner(remote, local_port) + tunnels.append(_Tunnel(remote, local_port, process)) + started.append(remote) + self._tunnels[challenge_id] = tunnels + # Probe outside the lock, on a snapshot, so a slow platform-side + # websocket handshake never blocks other challenges' tool calls. + deadline = time.monotonic() + self.startup_timeout + for tunnel in tunnels: + while time.monotonic() < deadline: + if not tunnel.alive: + raise RuntimeError( + f"wsrx tunnel for {tunnel.remote} exited immediately " + f"(code {tunnel.process.returncode}); is wsrx installed?" + ) + if self._probe(tunnel.local_port): + break + time.sleep(self.startup_interval) + else: + raise RuntimeError( + f"wsrx tunnel port {tunnel.local_port} did not become " + f"reachable within {self.startup_timeout:.0f}s" + ) + return { + "endpoints": [ + f"{self.endpoint_host}:{tunnel.local_port}" + for tunnel in sorted(tunnels, key=lambda t: t.local_port) + ], + "started": started, + } + + def stop(self, challenge_id: int) -> list[int]: + """Terminate every tunnel subprocess of one challenge.""" + + with self._lock: + tunnels = self._tunnels.pop(challenge_id, []) + for tunnel in tunnels: + self._terminator(tunnel.process) + return [tunnel.local_port for tunnel in tunnels] + + def endpoints(self, challenge_id: int) -> list[str]: + with self._lock: + tunnels = self._alive(challenge_id) + return [ + f"{self.endpoint_host}:{tunnel.local_port}" + for tunnel in sorted(tunnels, key=lambda t: t.local_port) + ] + + def stop_all(self) -> None: + with self._lock: + challenge_ids = list(self._tunnels) + for challenge_id in challenge_ids: + self.stop(challenge_id) + + +def build_ret2shell_mcp( + client: Ret2ShellClient, *, tunnel_manager: WsrxTunnelManager | None = None +) -> MCPServer: + """Expose ret2shell dynamic-instance control to Members. + + Credentials stay in the backend process; Members only ever see instance + endpoints and challenge state. Flag submission is deliberately NOT + exposed here — the shared 10-per-5-minutes quota is spent only by the + operator through scripts/ret2shell_submit.py. + """ + + manager = tunnel_manager or WsrxTunnelManager() + + server = create_mcp_server( + name="ret2shell", + description=( + "ret2shell competition platform: dynamic challenge instances and " + "solve state" + ), + lifespan=close_lifespan(client.close), + ) + + def _direct_endpoints(instance: dict[str, Any]) -> Any: + # Direct host:port style exposure (no wsrx needed): pass through. + exposed = instance.get("exposed_ports") + if isinstance(exposed, list) and exposed and not wsrx_remotes(instance, client): + return exposed + return None + + async def _resolve_endpoints(challenge_id: int, instance: dict[str, Any]) -> dict[str, Any]: + remotes = wsrx_remotes(instance, client) + if not remotes: + return {"endpoints": _direct_endpoints(instance)} + result = await asyncio.to_thread(manager.ensure, challenge_id, remotes) + return { + "endpoints": result["endpoints"], + "wsrx_remotes": remotes, + "tunnels_started": result["started"], + } + + @server.tool( + name="instance_start", + description=( + "Start (or reuse) the dynamic instance for one challenge and wait " + "until its endpoints are reachable. wsrx-proxied instances are " + "tunneled automatically: connect to the returned local " + "host:port pairs with netcat/pwntools. Requires the challenge id." + ), + ) + async def instance_start(challenge_id: int) -> dict[str, Any]: + existing = await asyncio.to_thread(client.find_instance, challenge_id) + if existing is None: + await asyncio.to_thread(client.start_instance, challenge_id) + instance = await asyncio.to_thread(client.wait_for_instance, challenge_id) + resolved = await _resolve_endpoints(challenge_id, instance) + return { + "challenge_id": challenge_id, + "state": instance.get("state"), + **resolved, + "renew_count": instance.get("renew_count"), + } + + @server.tool( + name="instance_status", + description=( + "Report the current instance state and endpoints for one challenge, " + "or null when no instance is running." + ), + ) + async def instance_status(challenge_id: int) -> dict[str, Any]: + instance = await asyncio.to_thread(client.find_instance, challenge_id) + if instance is None: + return {"challenge_id": challenge_id, "running": False} + remotes = wsrx_remotes(instance, client) + if remotes: + endpoints = await asyncio.to_thread(manager.endpoints, challenge_id) + resolved = {"endpoints": endpoints, "wsrx_remotes": remotes} + else: + resolved = {"endpoints": _direct_endpoints(instance)} + return { + "challenge_id": challenge_id, + "running": instance.get("state") == "Running", + "state": instance.get("state"), + **resolved, + "renew_count": instance.get("renew_count"), + } + + @server.tool( + name="instance_renew", + description="Extend the lifetime of the running instance for one challenge.", + ) + async def instance_renew(challenge_id: int) -> dict[str, Any]: + await asyncio.to_thread(client.renew_instance, challenge_id) + return {"challenge_id": challenge_id, "renewed": True} + + @server.tool( + name="instance_stop", + description="Destroy the running instance for one challenge.", + ) + async def instance_stop(challenge_id: int) -> dict[str, Any]: + torn_down = await asyncio.to_thread(manager.stop, challenge_id) + await asyncio.to_thread(client.destroy_instance, challenge_id) + return {"challenge_id": challenge_id, "stopped": True, "tunnels_closed": torn_down} + + @server.tool( + name="challenge_status", + description=( + "Report whether the team already solved one challenge and how many " + "teams solved it overall. Check before planning around a challenge." + ), + ) + async def challenge_status(challenge_id: int) -> dict[str, Any]: + status = await asyncio.to_thread(client.challenge_status, challenge_id) + if not isinstance(status, dict): + return {"challenge_id": challenge_id, "error": "unexpected status payload"} + return {"challenge_id": challenge_id, **status} + + return server diff --git a/backend/tools/member_tools.txt b/backend/tools/member_tools.txt index 10a3d38..5d62415 100644 --- a/backend/tools/member_tools.txt +++ b/backend/tools/member_tools.txt @@ -179,3 +179,16 @@ Common first moves by challenge shape libraries; use RsaCtfTool for weak RSA. - Stego/forensics: exiftool, binwalk, zsteg/steghide/pngcheck, OCR, audio/video extraction. + + +ret2shell platform (competition) +-------------------------------- +- ret2shell MCP: dynamic instance control for challenges that need a remote + target. Tools: instance_start (starts the challenge instance and waits for + its endpoints), instance_status, instance_renew, instance_stop, + challenge_status (already-solved check). Credentials stay in the backend. +- For pwn-style challenges with a remote endpoint, call instance_start first + and attack the returned address/port; renew the instance if it is about to + expire while you are still working. +- Flag submission is operator-only (shared 10-per-5-minutes quota). When you + find the flag, emit the flag action; do not attempt to submit it yourself. diff --git a/docker-compose.yml b/docker-compose.yml index 6cfa725..e5375fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,6 +98,13 @@ services: - IPC_CLAUDE_RUNNER_URL=${IPC_CLAUDE_RUNNER_URL:-http://ipc-claude-runner:8600} - IPC_RUNNER_TOKEN=${IPC_RUNNER_TOKEN:-} - IPC_DATABASE_URL=${IPC_DATABASE_URL:-postgresql://ipc:ipc@ipc-postgres:5432/ipc} + # ret2shell platform credentials; when USERNAME (or TOKEN) is set the + # backend registers the ret2shell MCP for every Member. + - IPC_R2S_BASE_URL=${IPC_R2S_BASE_URL:-} + - IPC_R2S_GAME_ID=${IPC_R2S_GAME_ID:-} + - IPC_R2S_USERNAME=${IPC_R2S_USERNAME:-} + - IPC_R2S_PASSWORD=${IPC_R2S_PASSWORD:-} + - IPC_R2S_TOKEN=${IPC_R2S_TOKEN:-} volumes: # PostgreSQL owns runtime facts. ./data holds deployment-owned secrets, # IPC conversation files and the shared Artifact tree (workspaces, diff --git a/docs/ret2shell.md b/docs/ret2shell.md new file mode 100644 index 0000000..f7d4bfa --- /dev/null +++ b/docs/ret2shell.md @@ -0,0 +1,87 @@ +# ret2shell 平台适配 + +适配 [ret2shell](https://github.com/ret2shell/ret2shell)(Rust/axum 开源 CTF 平台)。 +**已对真实平台验证通过**:西电 `https://ctf.xidian.edu.cn`(MoeCTF 2026,game 37,125 题), +端到端冒烟(导入 → 解题 → flag → writeup)已跑通。 + +## 配置(环境变量) + +| 变量 | 默认 | 说明 | +|---|---|---| +| `IPC_R2S_BASE_URL` | `https://ctf.xidian.edu.cn` | 平台根地址 | +| `IPC_R2S_GAME_ID` | `37` | 比赛 id(数字) | +| `IPC_R2S_USERNAME` / `IPC_R2S_PASSWORD` | 空 | 参赛账号(登录换 JWT Bearer token) | +| `IPC_R2S_TOKEN` | 空 | 已有的 Bearer token(可替代账密) | + +Docker Compose 部署时把这些写进项目根目录的 `.env`(compose 自动读取并注入 +ipc-app 容器;注意 `.env` 不能带 UTF-8 BOM,PowerShell `Set-Content` 后需检查)。 +**认证细节**:登录需要验证码时客户端自动走 `GET /api/account/captcha/cli` 的 +PoW(难度 4:答案以 challenge 随机串开头、其 SHA-256 十六进制前 4 位为 0), +无需人工介入;token 在响应头 `Set-Token` 返回并自动滚动更新。 +**密码连续错 5 次账号冻结 30 分钟**。 + +## 使用流程 + +```bash +export PYTHONPATH=. +export IPC_R2S_USERNAME=... IPC_R2S_PASSWORD=... + +# 1) 只读冒烟:登录 + 比赛状态 + 题目列表 +python scripts/ret2shell_platform_check.py + +# 2) 拉取题目清单(含分值/分类/已解状态)到 platform_targets/ret2shell.yaml +python scripts/fetch_ret2shell_targets.py + +# 3) 批量导入题目为项目。附件下载与数据库写入已分离(两阶段), +# 但大括量仍建议分批(脚本每批 15 题、自动重试): +docker cp scripts/import_ret2shell_batch.py ipc-app:/tmp/ +docker exec ipc-app python /tmp/import_batch.py +# 或直接对运行中的后端调 POST /api/platform/import(mapping 见下)。 + +# 4) Member 在沙箱里通过 ret2shell MCP 管理动态靶机(pwn 题拿 host:port) + +# 5) 提交 flag:默认 dry-run,--no-dry-run 才真正提交 +python scripts/ret2shell_submit.py --challenge --flag 'flag{...}' +``` + +**运维代理(IPC 对话)路径**:ops agent 的平台工作流也支持 +`challenges.platform: "ret2shell"`(backend/ops/service.py 的 `_adapter` 分支), +在 IPC 对话里提交含 `"challenges": {"platform": "ret2shell", "game_id": 37}` +的 workflow 描述并确认后,agent 可自行拉题导入。凭据始终只从环境变量读取。 + +**中文分类映射**(MoeCTF 2026 实测):Web 安全与渗透测试→web、 +现代密码学→crypto、二进制漏洞审计 / Python 沙箱逃逸→pwn、 +软件逆向工程→reverse、大语言模型应用安全→ai、其余(取证杂项/从此开始/ +开发运维基础/策略博弈/美工设计)→misc。 + +## 关键契约(与 GZCTF 的差异) + +- **异步判题**:`POST .../submit` 返回 `solved: null` 的 Submission,客户端轮询 + `GET .../submit?id=` 直到 `solved != null`(默认 1s × 7 次)。 +- **提交限额**:每账号 5 分钟 10 次(HTTP 429)。客户端内置同额度限速器, + 提交前还会做已解出 preflight,避免浪费平台尝试次数。 +- **动态靶机**:`POST/PATCH/DELETE .../instance` 启动/续期/销毁, + `GET /api/game/{gid}/instance` 轮询 `exposed_ports` 拿实际连接地址。 +- 列表接口返回 `[items, total]` 二元组;错误响应是纯文本 body。 + +## Member 侧能力(ret2shell MCP) + +配置了 `IPC_R2S_USERNAME`(或 `IPC_R2S_TOKEN`)后,后端自动向所有 Member 注册 +`ret2shell` MCP(凭据留在后端,不进沙箱): + +- `instance_start(challenge_id)`:启动并等待靶机可达,返回 endpoints +- `instance_status` / `instance_renew` / `instance_stop` +- `challenge_status(challenge_id)`:已解状态与全站解出数 + +**flag 提交不暴露给 Member**——共享限额只由操作者经 `scripts/ret2shell_submit.py` +消耗,避免 agent 误操作烧掉提交次数。 + +## 已知限制与运维注意 + +- **RAM 黑板**:解题图谱/项目列表在内存里,`docker compose restart` 会清空 + 项目(附件文件保留在 `projects/`,重新导入即恢复,external_id 对应关系不变)。 + 比赛期间尽量避免重启容器。 +- **SQLite 锁**:memdb 不支持 WAL,长事务与 UI 轮询互斥。导入已改为 + 两阶段(先下载后写库),但请勿在导入进行时并行发起第二个导入。 +- **附件去重**:导入脚本超时重试可能产生重复项目(同一 external_id 多个 + project),可按 external_id 保留最早一个、DELETE 其余。 diff --git a/scripts/fetch_ret2shell_targets.py b/scripts/fetch_ret2shell_targets.py new file mode 100644 index 0000000..805a8a9 --- /dev/null +++ b/scripts/fetch_ret2shell_targets.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import argparse +import os +from datetime import UTC, datetime +from pathlib import Path + +import yaml + +from backend.platform.ret2shell import Ret2ShellClient + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +TARGETS_PATH = PROJECT_ROOT / "platform_targets" / "ret2shell.yaml" + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Fetch ret2shell challenges for one game into a YAML target list." + ) + parser.add_argument("--base-url", default=os.getenv("IPC_R2S_BASE_URL", "")) + parser.add_argument("--game", type=int, default=None) + parser.add_argument("--username", default=os.getenv("IPC_R2S_USERNAME", "")) + parser.add_argument("--password", default=os.getenv("IPC_R2S_PASSWORD", "")) + parser.add_argument("--token", default=os.getenv("IPC_R2S_TOKEN", "")) + parser.add_argument("--output", type=Path, default=TARGETS_PATH) + args = parser.parse_args() + + challenges: list[dict] = [] + with Ret2ShellClient( + base_url=args.base_url, + game_id=args.game, + username=args.username, + password=args.password, + token=args.token, + ) as client: + game = client.get_game() + for raw in client.list_challenges(): + status = client.challenge_status(raw["id"]) + challenges.append( + { + "id": raw.get("id"), + "name": raw.get("name", ""), + "category": (raw.get("tag") or [{}])[0].get("name", "misc") + if isinstance(raw.get("tag"), list) and raw.get("tag") + else "misc", + "score": raw.get("score", 0), + "solved": bool(status.get("solved")) if isinstance(status, dict) else False, + "total_solves": status.get("solves") if isinstance(status, dict) else None, + } + ) + challenges.sort(key=lambda item: (str(item["category"]), item["id"] or 0)) + + data = { + "updated_at": datetime.now(UTC).isoformat(), + "game": { + "id": game.get("id"), + "name": game.get("name", ""), + "start_at": game.get("start_at"), + "end_at": game.get("end_at"), + }, + "challenge_count": len(challenges), + "challenges": challenges, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + yaml.safe_dump(data, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + print(f"wrote {len(challenges)} challenges to {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ret2shell_platform_check.py b/scripts/ret2shell_platform_check.py new file mode 100644 index 0000000..5b26b70 --- /dev/null +++ b/scripts/ret2shell_platform_check.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import argparse +import os + +from backend.platform.ret2shell import Ret2ShellClient + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Read-only ret2shell smoke check: login, game, challenges." + ) + parser.add_argument("--base-url", default=os.getenv("IPC_R2S_BASE_URL", "")) + parser.add_argument("--game", type=int, default=None) + parser.add_argument("--username", default=os.getenv("IPC_R2S_USERNAME", "")) + parser.add_argument("--password", default=os.getenv("IPC_R2S_PASSWORD", "")) + parser.add_argument("--token", default=os.getenv("IPC_R2S_TOKEN", "")) + args = parser.parse_args() + + with Ret2ShellClient( + base_url=args.base_url, + game_id=args.game, + username=args.username, + password=args.password, + token=args.token, + ) as client: + print("ping:", client.ping()) + profile = client.get_profile() + print("profile:", profile.get("account"), profile.get("nickname")) + game = client.get_game() + print( + "game:", + game.get("id"), + game.get("name"), + "start=", game.get("start_at"), + "end=", game.get("end_at"), + ) + challenges = client.list_challenges() + print("challenge_count:", len(challenges)) + for raw in challenges: + print( + raw.get("id"), + raw.get("name"), + "score=", raw.get("score"), + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ret2shell_submit.py b/scripts/ret2shell_submit.py new file mode 100644 index 0000000..051060d --- /dev/null +++ b/scripts/ret2shell_submit.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import argparse +import os + +from backend.platform.ret2shell import ( + Ret2ShellClient, + Ret2ShellError, + Ret2ShellPreflightError, + Ret2ShellRateLimitError, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Submit one flag to a ret2shell challenge. Dry-run by default so a " + "mistyped command never spends a platform attempt; pass " + "--no-dry-run to actually submit." + ) + ) + parser.add_argument("--base-url", default=os.getenv("IPC_R2S_BASE_URL", "")) + parser.add_argument("--game", type=int, default=None) + parser.add_argument("--username", default=os.getenv("IPC_R2S_USERNAME", "")) + parser.add_argument("--password", default=os.getenv("IPC_R2S_PASSWORD", "")) + parser.add_argument("--token", default=os.getenv("IPC_R2S_TOKEN", "")) + parser.add_argument("--challenge", type=int, required=True) + parser.add_argument("--flag", required=True) + parser.add_argument( + "--no-dry-run", + action="store_true", + help="Actually POST the flag (the shared quota is 10 submissions / 5 minutes).", + ) + args = parser.parse_args() + + with Ret2ShellClient( + base_url=args.base_url, + game_id=args.game, + username=args.username, + password=args.password, + token=args.token, + ) as client: + status = client.challenge_status(args.challenge) + print("challenge status before submit:", status) + if isinstance(status, dict) and status.get("solved"): + print("already solved; nothing to submit") + return 0 + if not args.no_dry_run: + print(f"dry-run ok: would submit {args.flag!r} to challenge {args.challenge}") + return 0 + try: + submission = client.submit_flag( + args.challenge, args.flag, check_solved=False + ) + except Ret2ShellPreflightError as exc: + print(f"preflight refused: {exc}") + return 1 + except Ret2ShellRateLimitError as exc: + print(f"rate limited: {exc}") + return 1 + except Ret2ShellError as exc: + print(f"submission failed: {exc}") + return 1 + print( + "submission", + submission.get("id"), + "solved=", submission.get("solved"), + "result=", submission.get("result"), + ) + return 0 if submission.get("solved") else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conftest.py b/tests/conftest.py index d9380e3..71e0a8c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,7 +64,12 @@ def pytest_collection_modifyitems(config, items) -> None: ) has_database = bool(os.environ.get("IPC_TEST_DATABASE_URL", "").strip()) for item in items: - if Path(str(item.fspath)).name not in POSTGRES_TEST_MODULES: + # Modules in POSTGRES_TEST_MODULES are entirely DB-backed; individual + # tests elsewhere may opt in with an explicit @pytest.mark.postgres. + if ( + Path(str(item.fspath)).name not in POSTGRES_TEST_MODULES + and item.get_closest_marker("postgres") is None + ): continue item.add_marker("postgres") if not has_database: diff --git a/tests/test_ret2shell.py b/tests/test_ret2shell.py new file mode 100644 index 0000000..c4bfdfa --- /dev/null +++ b/tests/test_ret2shell.py @@ -0,0 +1,704 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from backend.blackboard import graph_store +from backend.mcp.mcp_client import MCPClient +from backend.platform.mapping import FieldMapping +from backend.platform.ret2shell import ( + Ret2ShellAdapter, + Ret2ShellAuthError, + Ret2ShellClient, + Ret2ShellError, + Ret2ShellPreflightError, + Ret2ShellRateLimitError, + _SubmitRateLimiter, +) +from backend.platform.ret2shell_mcp import WsrxTunnelManager, build_ret2shell_mcp +from backend.server.app import create_app +from tests.helpers import setup_test_auth, write_mock_config + +BASE = "https://r2s.test" + + +class FakeResponse: + def __init__(self, status_code=200, payload=None, text="", headers=None, content=None): + self.status_code = status_code + self._payload = payload + if text: + self.text = text + elif payload is not None: + self.text = json.dumps(payload) + else: + self.text = "" + self.headers = headers or {} + if content is not None: + self._content = content + elif payload is not None: + self._content = json.dumps(payload).encode() + else: + self._content = b"" + + @property + def content(self): + return self._content + + def json(self): + if self._payload is None: + raise ValueError("no JSON payload") + return self._payload + + def iter_content(self, chunk_size): + yield self._content + + +class FakeSession: + def __init__(self, router): + self.router = router + self.calls: list[tuple[str, str, dict]] = [] + + def request(self, method, url, **kwargs): + self.calls.append((method, url, kwargs)) + return self.router(method, url, kwargs) + + def post(self, url, **kwargs): + return self.request("POST", url, **kwargs) + + def get(self, url, **kwargs): + return self.request("GET", url, **kwargs) + + def close(self): + pass + + +def make_client(router, authed=True, **kwargs): + defaults = dict( + base_url=BASE, + game_id=37, + username="user", + password="pass", + poll_attempts=3, + poll_interval=0.0, + sleep=lambda seconds: None, + ) + if authed: + defaults["token"] = "seed-token" + defaults.update(kwargs) + return Ret2ShellClient(session=FakeSession(router), **defaults) + + +def captcha_ok(url): + if url.endswith("/api/account/captcha/cli"): + return FakeResponse( + 200, + {"id": "cap-1", "validator": "pow", "challenge": "2#seed"}, + ) + return None + + +def login_ok(url, token="token-1"): + if cap := captcha_ok(url): + return cap + if url.endswith("/api/account/login"): + return FakeResponse(200, headers={"Set-Token": token}) + return None + + +# ---- auth ---- + + +def test_login_captures_set_token_header_and_uses_bearer(): + def router(method, url, kwargs): + if login := login_ok(url): + return login + if url.endswith("/api/account/profile"): + return FakeResponse(200, {"account": "user"}) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router, authed=False) + profile = client.get_profile() + assert profile["account"] == "user" + profile_call = next(call for call in client.session.calls if call[1].endswith("/api/account/profile")) + assert profile_call[2]["headers"]["Authorization"] == "Bearer token-1" + client.close() + + +def test_login_failure_raises_auth_error(): + def router(method, url, kwargs): + if url.endswith("/api/account/login"): + return FakeResponse(401, text="account or password is wrong") + if login := login_ok(url): + return login + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router, authed=False) + with pytest.raises(Ret2ShellAuthError, match="account or password is wrong"): + client.get_profile() + client.close() + + +def test_set_token_refresh_rolls_token_forward(): + state = {"logins": 0} + + def router(method, url, kwargs): + if cap := captcha_ok(url): + return cap + if url.endswith("/api/account/login"): + state["logins"] += 1 + return FakeResponse(200, headers={"Set-Token": "token-1"}) + if url.endswith("/api/game/37"): + if state["logins"] == 1: + return FakeResponse(200, {"id": 37}, headers={"Set-Token": "token-2"}) + return FakeResponse(200, {"id": 37}) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router, authed=False) + client.get_game() + client.get_game() + auth = client.session.calls[-1][2]["headers"]["Authorization"] + assert auth == "Bearer token-2" + assert state["logins"] == 1 + client.close() + + +def test_expired_token_relogins_once(): + state = {"profile_calls": 0} + + def router(method, url, kwargs): + if cap := captcha_ok(url): + return cap + if url.endswith("/api/account/login"): + return FakeResponse(200, headers={"Set-Token": "token-2"}) + if url.endswith("/api/account/profile"): + state["profile_calls"] += 1 + if state["profile_calls"] == 1: + return FakeResponse(401, text="token expired") + return FakeResponse(200, {"account": "user"}) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router, authed=False) + assert client.get_profile()["account"] == "user" + first_post = next(call for call in client.session.calls if call[0] == "POST") + assert first_post[1].endswith("/api/account/login") # login before any data call + assert client.session.calls[-1][2]["headers"]["Authorization"] == "Bearer token-2" + client.close() + + +# ---- challenges / attachments ---- + + +def test_list_challenges_parses_items_total_tuple(): + def router(method, url, kwargs): + if url.endswith("/api/game/37/challenge"): + return FakeResponse( + 200, + [ + [{"id": 1, "name": "Warmup", "tag": [{"name": "pwn", "primary": True}]}], + 1, + ], + ) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router) + challenges = client.list_challenges() + assert challenges[0]["name"] == "Warmup" + client.close() + + +def test_error_detail_uses_plain_text_body(): + def router(method, url, kwargs): + if url.endswith("/api/game/37/challenge"): + return FakeResponse(412, text="game has not started") + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router) + with pytest.raises(Ret2ShellError, match="game has not started"): + client.list_challenges() + client.close() + + +def test_adapter_maps_categories_and_downloads_files(tmp_path): + files = [{"folder": "static", "file": "handout.zip"}] + + class StubClient: + game_id = 37 + + def list_challenges(self, game_id=None): + return [ + {"id": 1, "name": "Pwn Me", "tag": [{"name": "pwn", "primary": True}], "content": "nc"}, + {"id": 2, "name": "Odd", "tag": [{"name": "weird", "primary": True}], "content": ""}, + {"id": 3, "name": "NoTag", "content": ""}, + ] + + def get_challenge(self, challenge_id, game_id=None): + return {} + + def list_files(self, challenge_id, game_id=None): + return files + + def download_file(self, challenge_id, folder, file, dest_dir, game_id=None, *, max_bytes=None): + target = Path(dest_dir) / file + target.write_bytes(b"zip-bytes") + return target + + adapter = Ret2ShellAdapter(StubClient(), category_map={"weird": "reverse"}) + challenges = adapter.fetch_challenges() + assert [c.category for c in challenges] == ["pwn", "reverse", "misc"] + assert [c.external_id for c in challenges] == ["1", "2", "3"] + assert challenges[0].attachment_urls == [] + + downloaded = adapter.download_attachments(challenges[0], tmp_path) + assert downloaded[0].name == "handout.zip" + assert downloaded[0].read_bytes() == b"zip-bytes" + + +def test_download_file_streams_to_disk(tmp_path): + def router(method, url, kwargs): + if url.endswith("/api/game/37/challenge/5/file") and kwargs.get("params", {}).get("file"): + return FakeResponse(200, content=b"attachment-data") + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router) + target = client.download_file(5, "static", "handout.tar.gz", tmp_path) + assert target.name == "handout.tar.gz" + assert target.read_bytes() == b"attachment-data" + call = client.session.calls[0] + assert call[2]["params"] == {"folder": "static", "file": "handout.tar.gz"} + client.close() + + +# ---- instances ---- + + +def test_instance_lifecycle_and_wait_for_endpoints(): + state = {"polls": 0, "started": False} + + def router(method, url, kwargs): + if url.endswith("/api/game/37/challenge/5/instance") and method == "POST": + state["started"] = True + return FakeResponse(200) + if url.endswith("/api/game/37/instance"): + state["polls"] += 1 + exposed = [{"1.2.3.4:1337": {"protocol": "tcp", "port": 1337}}] if state["polls"] >= 2 else None + return FakeResponse( + 200, + [ + { + "state": "Running", + "challenge_id": 5, + "exposed_ports": exposed, + "renew_count": 0, + } + ], + ) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router) + client.start_instance(5) + assert state["started"] is True + instance = client.wait_for_instance(5, timeout=10, interval=0) + assert instance["exposed_ports"] + client.close() + + +def test_wait_for_instance_times_out(): + def router(method, url, kwargs): + if url.endswith("/api/game/37/instance"): + return FakeResponse(200, [{"state": "Pending", "challenge_id": 5, "exposed_ports": None}]) + raise AssertionError(f"unexpected {method} {url}") + + clock = {"now": 0.0} + client = make_client( + router, + monotonic=lambda: clock["now"], + sleep=lambda seconds: clock.__setitem__("now", clock["now"] + seconds), + ) + with pytest.raises(Ret2ShellError, match="did not become reachable"): + client.wait_for_instance(5, timeout=3, interval=1) + client.close() + + +# ---- submission ---- + + +def _submit_router(submissions): + state = {"polls": 0} + + def router(method, url, kwargs): + if login := login_ok(url): + return login + if url.endswith("/api/game/37/challenge/5/submit"): + if method == "POST": + submissions.append({"id": 900, "solved": None, "result": None}) + return FakeResponse(200, submissions[-1]) + state["polls"] += 1 + submission = submissions[0] + if state["polls"] >= 2 and submission["solved"] is None: + submission["solved"] = True + submission["result"] = "accepted" + return FakeResponse(200, submission) + raise AssertionError(f"unexpected {method} {url}") + + return router + + +def test_submit_flag_polls_async_verdict(): + submissions = [] + client = make_client(_submit_router(submissions)) + result = client.submit_flag(5, "flag{demo}", check_solved=False) + assert result["solved"] is True + assert result["result"] == "accepted" + client.close() + + +def test_submit_flag_preflight_refuses_solved_challenge(): + def router(method, url, kwargs): + if login := login_ok(url): + return login + if url.endswith("/api/game/37/challenge/5/submit") and method == "GET": + return FakeResponse(200, {"solved": True, "solves": 12}) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router) + with pytest.raises(Ret2ShellPreflightError, match="already solved"): + client.submit_flag(5, "flag{demo}") + posts = [call for call in client.session.calls if call[0] == "POST" and call[1].endswith("/submit")] + assert posts == [] + client.close() + + +def test_submit_flag_rejects_empty_flag(): + client = make_client(lambda *args: FakeResponse(200, {})) + with pytest.raises(Ret2ShellPreflightError, match="flag must not be empty"): + client.submit_flag(5, " ") + client.close() + + +def test_submit_flag_maps_http_429_to_rate_limit(): + def router(method, url, kwargs): + if login := login_ok(url): + return login + if url.endswith("/api/game/37/challenge/5/submit"): + if method == "POST": + return FakeResponse(429, text="too many submissions, please calmdown") + return FakeResponse(200, {"solved": False}) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router) + with pytest.raises(Ret2ShellRateLimitError, match="too many submissions"): + client.submit_flag(5, "flag{demo}", check_solved=False) + client.close() + + +def test_submit_flag_poll_timeout_keeps_solved_none(): + submissions = [{"id": 900, "solved": None, "result": None}] + + def router(method, url, kwargs): + if login := login_ok(url): + return login + if url.endswith("/api/game/37/challenge/5/submit"): + if method == "POST": + return FakeResponse(200, submissions[0]) + return FakeResponse(200, submissions[0]) + raise AssertionError(f"unexpected {method} {url}") + + client = make_client(router, poll_attempts=2) + result = client.submit_flag(5, "flag{demo}", check_solved=False) + assert result["solved"] is None + gets = [c for c in client.session.calls if c[0] == "GET" and c[1].endswith("/submit")] + assert len(gets) == 2 + client.close() + + +def test_rate_limiter_enforces_shared_quota(): + clock = {"now": 0.0} + limiter = _SubmitRateLimiter(monotonic=lambda: clock["now"]) + for _ in range(10): + limiter.reserve() + with pytest.raises(Ret2ShellRateLimitError, match="retry in 300s"): + limiter.reserve() + clock["now"] = 301.0 + limiter.reserve() # window expired + assert limiter.used() == 1 + + +# ---- MCP ---- + + +class StubPlatformClient: + base_url = "https://ctf.r2s.test" + + def __init__(self): + self.calls: list[tuple[str, ...]] = [] + + def find_instance(self, challenge_id, game_id=None): + self.calls.append(("find", challenge_id)) + return None + + def start_instance(self, challenge_id, game_id=None): + self.calls.append(("start", challenge_id)) + + def wait_for_instance(self, challenge_id, game_id=None, **kwargs): + self.calls.append(("wait", challenge_id)) + return { + "state": "Running", + "exposed_ports": [{"ctf.r2s.test:1337": {"protocol": "tcp"}}], + "renew_count": 0, + } + + def renew_instance(self, challenge_id, game_id=None): + self.calls.append(("renew", challenge_id)) + + def destroy_instance(self, challenge_id, game_id=None): + self.calls.append(("destroy", challenge_id)) + + def challenge_status(self, challenge_id, game_id=None): + self.calls.append(("status", challenge_id)) + return {"solved": False, "solves": 3} + + def close(self): + pass + + +def _call_tool(server, name, **arguments): + async def run(): + async with MCPClient.in_process(server) as client: + tools = {tool.name for tool in await client.list_tools()} + assert name in tools + return await client.call_tool(name, arguments) + + return asyncio.run(run()) + + +def test_ret2shell_mcp_exposes_instance_tools(): + stub = StubPlatformClient() + server = build_ret2shell_mcp(stub) + started = _call_tool(server, "instance_start", challenge_id=5) + assert started["endpoints"] + assert ("start", 5) in stub.calls and ("wait", 5) in stub.calls + assert _call_tool(server, "instance_status", challenge_id=5)["running"] is False + assert _call_tool(server, "instance_renew", challenge_id=5)["renewed"] is True + assert _call_tool(server, "instance_stop", challenge_id=5)["stopped"] is True + status = _call_tool(server, "challenge_status", challenge_id=5) + assert status["solved"] is False and status["solves"] == 3 + + +# ---- wsrx tunneling ---- + + +class FakeProcess: + def __init__(self, remote, local_port): + self.remote = remote + self.local_port = local_port + self.returncode = None + self.terminated = False + + def poll(self): + return self.returncode + + def terminate(self): + self.terminated = True + self.returncode = 0 + + def kill(self): + self.returncode = -9 + + def wait(self, timeout=None): + return self.returncode + + +def _fake_manager(spawns): + def spawner(remote, local_port): + process = FakeProcess(remote, local_port) + spawns.append(process) + return process + + return WsrxTunnelManager(spawner=spawner, probe=lambda port: True) + + +class TrafficStubClient(StubPlatformClient): + """Platform shape observed on ret2shell: traffic token + ports.""" + + def _traffic_instance(self): + return { + "state": "Running", + "exposed_ports": None, + "traffic": "tok1430", + "ports": [9999], + "renew_count": 0, + } + + def find_instance(self, challenge_id, game_id=None): + self.calls.append(("find", challenge_id)) + if ("start", challenge_id) not in self.calls: + return None + return self._traffic_instance() + + def wait_for_instance(self, challenge_id, game_id=None, **kwargs): + self.calls.append(("wait", challenge_id)) + return self._traffic_instance() + + +def test_wsrx_manager_spawns_idempotent_and_cleans_up(): + spawns = [] + manager = _fake_manager(spawns) + remote = "wss://ctf.xidian.edu.cn/api/traffic/tok1430?port=9999" + + first = manager.ensure(1430, [remote]) + assert first["endpoints"] == ["ipc-app:21430"] + assert first["started"] == [remote] + assert len(spawns) == 1 + assert spawns[0].remote == remote + assert spawns[0].local_port == 21430 + + # Repeated ensure for the same challenge reuses the live tunnel. + second = manager.ensure(1430, [remote]) + assert second["endpoints"] == ["ipc-app:21430"] + assert second["started"] == [] + assert len(spawns) == 1 + + manager.stop(1430) + assert spawns[0].terminated + assert manager.endpoints(1430) == [] + + +def test_wsrx_manager_restarts_dead_tunnel_and_binds_one_port_each(): + spawns = [] + manager = _fake_manager(spawns) + remotes = [ + "wss://ctf.xidian.edu.cn/api/traffic/tok?port=1337", + "wss://ctf.xidian.edu.cn/api/traffic/tok?port=9999", + ] + result = manager.ensure(7, remotes) + assert result["endpoints"] == ["ipc-app:20007", "ipc-app:20008"] + assert len(spawns) == 2 + + spawns[0].returncode = 1 # process died behind our back + result = manager.ensure(7, remotes) + assert result["endpoints"] == ["ipc-app:20007", "ipc-app:20008"] + assert result["started"] == [remotes[0]] + assert len(spawns) == 3 + # Only the two live tunnels remain tracked. + assert manager.endpoints(7) == ["ipc-app:20007", "ipc-app:20008"] + + +def test_instance_start_tunnels_wsrx_remotes_to_local_port(): + spawns = [] + stub = TrafficStubClient() + server = build_ret2shell_mcp(stub, tunnel_manager=_fake_manager(spawns)) + started = _call_tool(server, "instance_start", challenge_id=1430) + assert started["endpoints"] == ["ipc-app:21430"] + assert started["wsrx_remotes"] == [ + "wss://ctf.r2s.test/api/traffic/tok1430?port=9999" + ] + assert ("start", 1430) in stub.calls + + # Idempotent: a repeated start must not spawn a second tunnel. + again = _call_tool(server, "instance_start", challenge_id=1430) + assert again["endpoints"] == ["ipc-app:21430"] + assert len(spawns) == 1 + + status = _call_tool(server, "instance_status", challenge_id=1430) + assert status["running"] is True + assert status["endpoints"] == ["ipc-app:21430"] + + stopped = _call_tool(server, "instance_stop", challenge_id=1430) + assert stopped["stopped"] is True + assert stopped["tunnels_closed"] == [21430] + assert spawns[0].terminated + assert ("destroy", 1430) in stub.calls + + +def test_instance_start_passes_direct_endpoints_through(): + # GZCTF-style direct host:port exposure needs no wsrx at all. + spawns = [] + stub = StubPlatformClient() + server = build_ret2shell_mcp(stub, tunnel_manager=_fake_manager(spawns)) + started = _call_tool(server, "instance_start", challenge_id=5) + assert started["endpoints"] == [{"ctf.r2s.test:1337": {"protocol": "tcp"}}] + assert "wsrx_remotes" not in started + assert spawns == [] + + +# ---- API import integration ---- + + +@pytest.fixture +def api_client(tmp_path, monkeypatch): + config_dir = write_mock_config(tmp_path / "config") + monkeypatch.setenv("IPC_ROOT", str(tmp_path)) + app = create_app(root=tmp_path) + with TestClient(app) as test_client: + setup_test_auth(test_client) + test_client.app.state.ipc.config_dir = config_dir + test_client.app.state.ipc.reload_config() + yield test_client + + +@pytest.mark.postgres +def test_platform_import_supports_ret2shell_platform(api_client, monkeypatch): + class StubClient: + def __init__(self, *args, **kwargs): + pass + + def list_challenges(self, game_id=None): + return [ + { + "id": 11, + "name": "Remote Calc", + "tag": [{"name": "pwn", "primary": True}], + "content": "nc target 1337", + } + ] + + def get_challenge(self, challenge_id, game_id=None): + return {} + + def list_files(self, challenge_id, game_id=None): + return [{"folder": "static", "file": "calc.zip"}] + + def download_file(self, challenge_id, folder, file, dest_dir, game_id=None, *, max_bytes=None): + target = Path(dest_dir) / file + target.write_bytes(b"zip") + return target + + def close(self): + pass + + monkeypatch.setattr("backend.api.platform.Ret2ShellClient", StubClient) + mapping = FieldMapping(platform="ret2shell", game_id=37) + + preview = api_client.post( + "/api/platform/challenges", json={"mapping": mapping.model_dump()} + ) + assert preview.status_code == 200 + challenges = preview.json()["challenges"] + assert challenges[0]["external_id"] == "11" + assert challenges[0]["category"] == "pwn" + + imported = api_client.post( + "/api/platform/import", + json={"mapping": mapping.model_dump(), "select": ["11"]}, + ) + assert imported.status_code == 201 + item = imported.json()["imported"][0] + assert item["category"] == "pwn" + with api_client.app.state.ipc.db.connect() as conn: + # The challenge description must land in the origin fact so the + # Member context exposes it (connection commands, flag format...). + facts = {f.id: f.description for f in graph_store.project_detail(conn, item["project_id"]).facts} + assert "nc target 1337" in facts.get("origin", "") + graph_store.set_flag(conn, item["project_id"], "flag{r2s}") + graph_store.set_status(conn, item["project_id"], "completed") + assert api_client.get(f"/api/flags/{item['project_id']}").json()["flag"] == "flag{r2s}" + + +def test_field_mapping_requires_list_url_for_http_json(): + with pytest.raises(ValueError, match="list_url is required"): + FieldMapping() + assert FieldMapping(platform="ret2shell").game_id is None From edc610641c436a63a906a922658246cb41fd537f Mon Sep 17 00:00:00 2001 From: PureStream108 Date: Mon, 31 Aug 2026 22:17:57 +0800 Subject: [PATCH 03/10] fix(api): two-phase challenge import to avoid holding the DB during downloads Phase 1 downloads every attachment to a staging directory while NO database connection is open; phase 2 then performs only fast local writes inside the transaction. Holding the write transaction across network downloads starved the UI polling loop during large imports. The challenge statement (connection commands, flag format) now rides in the origin fact so the member context's challenge_description picks it up, and the import endpoint dispatches to the ret2shell adapter when mapping.platform selects it. --- backend/api/platform.py | 77 +++++++++++++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/backend/api/platform.py b/backend/api/platform.py index 182280b..e6efa37 100644 --- a/backend/api/platform.py +++ b/backend/api/platform.py @@ -1,6 +1,9 @@ from __future__ import annotations +import re import shutil +import time +from pathlib import Path import requests from fastapi import APIRouter, Depends, HTTPException @@ -9,12 +12,18 @@ from backend.api.deps import get_state from backend.blackboard import graph_store from backend.core.state import AppState -from backend.platform.adapter import HttpJsonAdapter +from backend.platform.adapter import HttpJsonAdapter, PlatformAdapter from backend.platform.mapping import FieldMapping, PlatformChallenge +from backend.platform.ret2shell import Ret2ShellAdapter, Ret2ShellClient, Ret2ShellError router = APIRouter(prefix="/api/platform", tags=["platform"]) +def safe_name(external_id: str) -> str: + cleaned = re.sub(r"[^A-Za-z0-9._-]", "_", external_id).strip("._") or "challenge" + return cleaned[:80] + + class ChallengeRequest(BaseModel): model_config = ConfigDict(extra="forbid") mapping: FieldMapping @@ -24,12 +33,25 @@ class ImportRequest(ChallengeRequest): select: list[str] | None = None -def _fetch(mapping: FieldMapping) -> tuple[HttpJsonAdapter, list[PlatformChallenge]]: - adapter = HttpJsonAdapter(mapping) +def _build_adapter(mapping: FieldMapping) -> PlatformAdapter: + if mapping.platform == "ret2shell": + client = Ret2ShellClient(base_url=mapping.list_url, game_id=mapping.game_id) + return Ret2ShellAdapter( + client, + game_id=mapping.game_id or None, + category_map=mapping.category_map, + ) + return HttpJsonAdapter(mapping) + + +def _fetch(mapping: FieldMapping) -> tuple[PlatformAdapter, list[PlatformChallenge]]: + adapter = _build_adapter(mapping) try: return adapter, adapter.fetch_challenges() except requests.RequestException as exc: raise HTTPException(502, f"platform request failed: {exc}") from exc + except Ret2ShellError as exc: + raise HTTPException(502, f"ret2shell request failed: {exc}") from exc except (TypeError, ValueError) as exc: raise HTTPException(400, str(exc)) from exc @@ -52,29 +74,58 @@ def import_challenges(body: ImportRequest, state: AppState = Depends(get_state)) raise HTTPException(400, f"unknown external_id values: {missing}") selected = [by_id[external_id] for external_id in body.select] + # Phase 1 downloads every attachment while NO database connection is + # open. Holding the write transaction across network downloads starved + # the UI polling loop (and itself) with "database is locked" on large + # imports. Phase 2 then only performs fast local writes. + staging_root = state.root / "staging" / f"import_{int(time.time() * 1000)}" + staged: dict[str, list[Path]] = {} + try: + for challenge in selected: + staged[challenge.external_id] = adapter.download_attachments( + challenge, + staging_root / safe_name(challenge.external_id), + ) + except requests.RequestException as exc: + shutil.rmtree(staging_root, ignore_errors=True) + raise HTTPException(502, f"attachment download failed: {exc}") from exc + except Ret2ShellError as exc: + shutil.rmtree(staging_root, ignore_errors=True) + raise HTTPException(502, f"ret2shell attachment download failed: {exc}") from exc + except (OSError, TypeError, ValueError) as exc: + shutil.rmtree(staging_root, ignore_errors=True) + raise HTTPException(400, str(exc)) from exc + imported: list[dict[str, str]] = [] created_project_ids: list[str] = [] try: with state.db.connect() as conn: for challenge in selected: + # Mirror the ops workflow importer: the challenge statement + # (connection commands, flag format) rides in the origin fact + # so the Member context's challenge_description picks it up. + origin = body.mapping.list_url + if challenge.description: + origin = f"{origin}\n\n{challenge.description}" project_id = graph_store.create_project( conn, challenge.title, - body.mapping.list_url, + origin, "capture the flag", challenge.category, external_id=challenge.external_id, ) created_project_ids.append(project_id) - for path in adapter.download_attachments( - challenge, - state.attachments_dir(project_id), - ): + attachment_dir = state.attachments_dir(project_id) + attachment_dir.mkdir(parents=True, exist_ok=True) + for path in staged.get(challenge.external_id, []): + target = attachment_dir / path.name + shutil.move(str(path), str(target)) graph_store.create_attachment( conn, project_id, - path.name, - str(path), + target.name, + str(target), ) imported.append( { @@ -84,14 +135,12 @@ def import_challenges(body: ImportRequest, state: AppState = Depends(get_state)) "category": challenge.category, } ) - except requests.RequestException as exc: - for project_id in created_project_ids: - shutil.rmtree(state.projects_dir / project_id, ignore_errors=True) - raise HTTPException(502, f"attachment download failed: {exc}") from exc except (OSError, TypeError, ValueError) as exc: for project_id in created_project_ids: shutil.rmtree(state.projects_dir / project_id, ignore_errors=True) raise HTTPException(400, str(exc)) from exc + finally: + shutil.rmtree(staging_root, ignore_errors=True) for item in imported: state.logger.project( From 488c9240e28d700391b3ca62812a7aabd7351b0d Mon Sep 17 00:00:00 2001 From: PureStream108 Date: Mon, 31 Aug 2026 22:18:40 +0800 Subject: [PATCH 04/10] fix(core): stall escalation, crash fallback hardening, project id reservation - An intent that stalls 3 times in a row is now deferred for 180s so the scheduler dispatches a concrete alternate branch instead of burning slots on the same stuck line; progress on done/concluded/flag resets the counter. - The scheduler tick's error handler degrades to stderr when the DB-backed logger itself is the failure, so a locked/unreachable database can no longer kill the loop silently. - Reserve the largest surviving proj_NNNN workspace id on startup so a fresh database never reissues an id whose artifacts still exist. - project_log_filename degrades to the plain project id when the database is unavailable instead of raising through logging paths. - create_fact deduplicates restated source findings by their stable (path, line, commit) key, plus exact whitespace-normalized matches. - ipc_task_sandbox_exec no longer passes project_id twice through the MCP bridge, which failed with 'too many values to unpack'. --- backend/blackboard/node_store.py | 51 ++++++++++++++++++++++++++++ backend/core/orchestrator.py | 46 +++++++++++++++++++++++++- backend/core/state.py | 36 ++++++++++++++++++-- backend/ops/ipc_mcp.py | 6 +++- tests/test_blackboard.py | 20 +++++++++++ tests/test_orchestration.py | 57 ++++++++++++++++++++++++++++++++ 6 files changed, 212 insertions(+), 4 deletions(-) diff --git a/backend/blackboard/node_store.py b/backend/blackboard/node_store.py index 6e4af4c..2877c99 100644 --- a/backend/blackboard/node_store.py +++ b/backend/blackboard/node_store.py @@ -1,11 +1,58 @@ from __future__ import annotations +import re from typing import Any from backend.blackboard.ids import next_fact_id, utcnow from backend.blackboard.models import Fact +# Agent reports often restate the same source finding with a different lead-in +# (for example, "Nday candidate" versus "confirmed candidate"). Keeping the +# full prose is useful for audit, but these stable tokens are enough to prevent +# a repeated conclusion from inflating the graph and retriggering Diamond. +_FINDING_RE = re.compile( + r"(?P[A-Za-z0-9_./-]+\.(?:c|cc|cpp|cxx|h|hpp|py|js|ts|java))" + r"(?::|\s+lines?\s+)(?P\d+(?:-\d+)?)" + r".*?(?P\b[0-9a-f]{7,40}\b)", + re.IGNORECASE | re.DOTALL, +) + + +def _finding_key(description: str) -> tuple[str, str, str] | None: + match = _FINDING_RE.search(description) + if match is None: + return None + return ( + match.group("path").replace("\\", "/").lower(), + match.group("line"), + match.group("commit").lower(), + ) + + +def _existing_equivalent_fact(conn: Any, project_id: str, description: str) -> Fact | None: + normalized = " ".join(description.split()) + row = conn.execute( + "SELECT id, description, created_at FROM facts " + "WHERE project_id = %s AND description = %s ORDER BY created_at, id LIMIT 1", + (project_id, normalized), + ).fetchone() + if row is not None: + return Fact(id=row["id"], description=row["description"], created_at=row["created_at"]) + + key = _finding_key(normalized) + if key is None: + return None + rows = conn.execute( + "SELECT id, description, created_at FROM facts WHERE project_id = %s ORDER BY created_at, id", + (project_id,), + ).fetchall() + for row in rows: + if _finding_key(row["description"]) == key: + return Fact(id=row["id"], description=row["description"], created_at=row["created_at"]) + return None + + def insert_fact(conn: Any, project_id: str, fact_id: str, description: str) -> Fact: now = utcnow() conn.execute( @@ -26,6 +73,10 @@ def reserve_fact(conn: Any, project_id: str, description: str) -> Fact: def create_fact(conn: Any, project_id: str, description: str) -> Fact: + description = " ".join(description.split()) + existing = _existing_equivalent_fact(conn, project_id, description) + if existing is not None: + return existing fid = next_fact_id(conn, project_id) return insert_fact(conn, project_id, fid, description) diff --git a/backend/core/orchestrator.py b/backend/core/orchestrator.py index b8516a6..f8c0377 100644 --- a/backend/core/orchestrator.py +++ b/backend/core/orchestrator.py @@ -7,6 +7,7 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass import json +import sys from time import monotonic from typing import Any import uuid @@ -56,6 +57,8 @@ class PostprocessLeaseLost(RuntimeError): class Orchestrator: _MEMBER_RETRY_DELAYS = (10, 30, 90, 300) + _MAX_CONSECUTIVE_STALLS = 3 + _STALL_DEFER_SECONDS = 180 _POSTPROCESS_LEASE_SECONDS = 120 def __init__(self, state, max_workers: int | None = None, scripts: dict | None = None): @@ -88,6 +91,7 @@ def __init__(self, state, max_workers: int | None = None, scripts: dict | None = self._startup_futures: dict[str, Any] = {} self._member_failure_counts: dict[tuple[str, str], int] = {} self._member_retry_not_before: dict[tuple[str, str], float] = {} + self._member_stall_counts: dict[tuple[str, str], int] = {} self._project_leases: dict[str, str] = {} self._intent_leases: dict[tuple[str, str], IntentLease] = {} self._lock = threading.Lock() @@ -1211,7 +1215,13 @@ def _run_loop(self) -> None: try: self._tick() except Exception as exc: - self.state.logger.project("scheduler_error", "system", error=str(exc)) + # The fallback handler itself must never kill the loop: it + # logs through the DB-backed resolver, which is exactly what + # fails when the tick died on a database error. + try: + self.state.logger.project("scheduler_error", "system", error=str(exc)) + except Exception: + print(f"[ipc-orchestrator] tick failed: {type(exc).__name__}: {exc}", file=sys.stderr, flush=True) self._stop.wait(interval) def _tick(self) -> None: @@ -1438,6 +1448,32 @@ def _schedule_member_retry( )[:1500], ) + def _record_member_stall(self, project_id: str, intent_id: str) -> tuple[int, bool]: + """Defer a repeatedly stalled intent so a concrete alternate branch can run.""" + key = (project_id, intent_id) + with self._lock: + count = self._member_stall_counts.get(key, 0) + 1 + self._member_stall_counts[key] = count + should_defer = count >= self._MAX_CONSECUTIVE_STALLS + if should_defer: + self._member_retry_not_before[key] = monotonic() + self._STALL_DEFER_SECONDS + if should_defer: + # The finished-future path has already released the intent claim; + # only the redispatch backoff is needed here. + self.state.logger.project( + "member_intent_deferred", + project_id, + intent=intent_id, + stalls=count, + delay_seconds=self._STALL_DEFER_SECONDS, + reason="repeated_stalls", + ) + return count, should_defer + + def _clear_member_stalls(self, project_id: str, intent_id: str) -> None: + with self._lock: + self._member_stall_counts.pop((project_id, intent_id), None) + def _clear_member_retry(self, project_id: str, intent_id: str) -> None: key = (project_id, intent_id) with self._lock: @@ -1454,6 +1490,8 @@ def _clear_project_member_retries(self, project_id: str) -> None: self._member_failure_counts.pop(key, None) for key in [key for key in self._member_retry_not_before if key[0] == project_id]: self._member_retry_not_before.pop(key, None) + for key in [key for key in self._member_stall_counts if key[0] == project_id]: + self._member_stall_counts.pop(key, None) def _reconcile_resources(self) -> None: with self.state.db.connect() as conn: @@ -1711,6 +1749,7 @@ def _reap_finished_futures(self) -> None: continue self._clear_member_retry(project_id, intent_id) if result.status == "stalled": + stalls, deferred = self._record_member_stall(project_id, intent_id) self.state.logger.project( "member_task_stalled", project_id, @@ -1719,14 +1758,19 @@ def _reap_finished_futures(self) -> None: retryable=result.retryable, error_kind=result.error_kind, error=result.error, + consecutive_stalls=stalls, + deferred=deferred, ) elif result.status == "done": + self._clear_member_stalls(project_id, intent_id) self.state.logger.project("member_task_done", project_id, intent=intent_id, steps=result.steps) elif result.status == "concluded": + self._clear_member_stalls(project_id, intent_id) self.state.logger.project("member_task_concluded", project_id, intent=intent_id, fact=result.fact_id) if result.fact_id is not None: self._broadcast_fact(project_id, result.fact_id, self._intent_worker(project_id, intent_id)) elif result.status == "flag": + self._clear_member_stalls(project_id, intent_id) self.state.logger.project("member_task_flag", project_id, intent=intent_id, flag=result.flag) def _intent_worker(self, project_id: str, intent_id: str) -> str | None: diff --git a/backend/core/state.py b/backend/core/state.py index 5395555..cc76362 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import re import shutil import socket import threading @@ -59,6 +60,7 @@ def __init__(self, root: str | Path = ".", config_dir: Path | None = None): self.db = Database().configure() with self.db.connect() as conn: graph_store.reset_project_counter_if_empty(conn) + self._reserve_existing_project_ids(conn) self.memory = MemoryStore(self.db, export_dir=None).configure() self.registry = ToolRegistry().load() self.catalog = ToolCatalog.load(self.registry) @@ -107,6 +109,30 @@ def __init__(self, root: str | Path = ".", config_dir: Path | None = None): # Attached by module 8. self.orchestrator = None + def _reserve_existing_project_ids(self, conn) -> None: + """Keep a fresh database from reusing surviving workspace ids. + + PostgreSQL rows are durable, but the database can still be recreated + (e.g. a replaced container volume) while project workspace directories + on the shared Artifact root survive. Reserve the largest surviving + numeric ID so a newly created project never inherits old progress or + artifacts merely because the database started empty. + """ + if not self.projects_dir.is_dir(): + return + highest = 0 + for entry in self.projects_dir.iterdir(): + if not entry.is_dir(): + continue + match = re.fullmatch(r"proj_(\d+)", entry.name) + if match: + highest = max(highest, int(match.group(1))) + if highest: + conn.execute( + "UPDATE counters SET value = GREATEST(value, %s) WHERE name = 'project'", + (highest,), + ) + def _clean_runtime_state(self) -> None: # Database rows are intentionally not deleted by a local clean start. # Durable exports are preserved; only live workspaces and generated @@ -142,8 +168,14 @@ def close(self) -> None: self.db.close() def project_log_filename(self, project_id: str) -> str | None: - with self.db.connect() as conn: - return graph_store.project_log_filename(conn, project_id) + # Called from logging paths, including the scheduler's crash handler; + # a failing database must degrade to the plain id instead of raising + # through the logger and killing the caller. + try: + with self.db.connect() as conn: + return graph_store.project_log_filename(conn, project_id) + except Exception: + return project_id def attachments_dir(self, project_id: str) -> Path: d = self.projects_dir / project_id / "attachments" diff --git a/backend/ops/ipc_mcp.py b/backend/ops/ipc_mcp.py index 20971dc..bede1c6 100644 --- a/backend/ops/ipc_mcp.py +++ b/backend/ops/ipc_mcp.py @@ -388,7 +388,11 @@ def task_sandbox_health(self, project_id: str) -> dict[str, Any]: return self._run_project_tool(project_id, "task_sandbox_health") def task_sandbox_exec(self, project_id: str, command: str, timeout: int) -> dict[str, Any]: - return self._run_project_tool(project_id, "task_sandbox_exec", project_id, command, timeout) + # ``_run_project_tool`` injects project_id into the executor payload; + # pass only the command arguments here. Passing project_id twice made + # the MCP bridge fail with ``too many values to unpack`` before the + # sandbox command could run. + return self._run_project_tool(project_id, "task_sandbox_exec", command, timeout) def host_exec(self, command: str, timeout: int) -> dict[str, Any]: try: diff --git a/tests/test_blackboard.py b/tests/test_blackboard.py index fd0e2cf..da3dd2f 100644 --- a/tests/test_blackboard.py +++ b/tests/test_blackboard.py @@ -62,6 +62,26 @@ def test_hyperedge_multiple_sources(db): assert set(intent.from_) == {f1.id, f2.id} +def test_create_fact_deduplicates_reworded_source_finding(db): + with db.connect() as conn: + pid = graph_store.create_project(conn, "T", "o", "g", "pwn") + first = node_store.create_fact( + conn, + pid, + "Nday candidate services/camera/foo.cpp lines 40-48; " + "introducing commit a876352a6fb119ba2e81159ec106359a99b56461.", + ) + second = node_store.create_fact( + conn, + pid, + "Confirmed candidate services/camera/foo.cpp:40-48; " + "introducing commit a876352a6fb119ba2e81159ec106359a99b56461; " + "track mapping pending.", + ) + assert second.id == first.id + assert len(node_store.list_facts(conn, pid)) == 3 + + def test_expire_workers_releases_stale_claims(db): with db.connect() as conn: pid = graph_store.create_project(conn, "T", "o", "g", "misc") diff --git a/tests/test_orchestration.py b/tests/test_orchestration.py index 7deb6e7..96ecd09 100644 --- a/tests/test_orchestration.py +++ b/tests/test_orchestration.py @@ -692,3 +692,60 @@ def test_non_retryable_member_failure_is_not_requeued(state): ) assert not any(entry["event"] == "member_task_retry_scheduled" for entry in entries) orch.shutdown() + + +def test_repeated_stalls_defer_original_intent_and_dispatch_alternative(state): + from backend.core.orchestrator import IntentLease, Orchestrator + from backend.members.base_member import SolveResult + + pid = _make_project(state, "web") + with state.db.connect() as conn: + graph_store.set_status(conn, pid, "running") + stalled = edge_store.create_intent( + conn, pid, ["origin"], "repeat a stalled scan", "diamond" + ) + alternative = edge_store.create_intent( + conn, pid, ["origin"], "run the alternate static query", "diamond" + ) + + orch = Orchestrator(state, max_workers=1) + for _ in range(orch._MAX_CONSECUTIVE_STALLS): + # Mirror the real dispatch path: the worker claims the intent with a + # lease, then finishes stalled, and the reaper releases the claim. + with state.db.connect() as conn: + token = edge_store.claim_intent(conn, pid, stalled.id, "aventurine") + assert token + with orch._lock: + orch._intent_leases[(pid, stalled.id)] = IntentLease( + worker="aventurine", owner="aventurine", token=token + ) + future = Future() + future.set_result(SolveResult(status="stalled", steps=3)) + with orch._lock: + orch._task_index[(pid, stalled.id)] = future + orch._reap_finished_futures() + + key = (pid, stalled.id) + assert orch._member_stall_counts[key] == orch._MAX_CONSECUTIVE_STALLS + assert orch._member_retry_blocked(pid, stalled.id) + with state.db.connect() as conn: + assert edge_store.get_intent(conn, pid, stalled.id)["worker"] is None + + launched = [] + + def capture_launch(project_id, member_name, intent_id, category, is_initial): + launched.append((project_id, member_name, intent_id, category, is_initial)) + return True + + orch._launch_member = capture_launch + orch._dispatch_project(pid) + assert launched == [(pid, "aventurine", alternative.id, "web", False)] + + entries = state.logger.read_log("project", pid, None) + assert any( + entry["event"] == "member_intent_deferred" + and entry["intent"] == stalled.id + and entry["reason"] == "repeated_stalls" + for entry in entries + ) + orch.shutdown() From 2c61ab2ce2284b99bfb9179e09046ff4ecce0470 Mon Sep 17 00:00:00 2001 From: PureStream108 Date: Mon, 31 Aug 2026 22:18:54 +0800 Subject: [PATCH 05/10] fix(members): kimi thinking-model budget, gateway error detail, challenge context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - kimi-for-coding / kimi-k* are recognized as always-reasoning models; forced-reasoning models get a 16384-token decision budget because 4096 was observed fully consumed by hidden reasoning, leaving an empty finish_reason=length response. - Provider errors now include the gateway's response body (truncated to 500 chars) — a bare 'HTTP 400' was undiagnosable from logs. - Explicitly selected Chat Completions on a non-native gateway no longer assumes strict JSON Schema support from the model name alone; it falls back to the broadly supported JSON Object mode. - Member context gains challenge_description (the original statement with connection commands and flag format) and external_id, plus a rejected-flag blacklist surfaced from experience memory so a member never resubmits flags the platform already rejected. --- backend/members/adapters.py | 29 ++++++++++++++++++++---- backend/members/base_member.py | 41 +++++++++++++++++++++++++++++----- tests/test_members.py | 30 +++++++++++++++++++++++++ tests/test_ops_agent.py | 4 ++-- 4 files changed, 92 insertions(+), 12 deletions(-) diff --git a/backend/members/adapters.py b/backend/members/adapters.py index 7b326d3..9969cf8 100644 --- a/backend/members/adapters.py +++ b/backend/members/adapters.py @@ -277,7 +277,11 @@ def decide(self, context: dict) -> MemberAction: messages, system_prompt=_SYSTEM_PROMPT, temperature=0.0 if deepseek else (None if reasoning_model else 0.4), - max_tokens=4096 if deepseek or reasoning_model else None, + # Forced-reasoning models (e.g. kimi-for-coding) spend the output + # budget on hidden reasoning before any content; 4096 was observed + # being fully consumed by reasoning alone on large contexts, leaving + # an empty response with finish_reason=length. + max_tokens=16384 if reasoning_model else (4096 if deepseek else None), structured=True, reasoning_effort=reasoning_effort, thinking="disabled" if deepseek else None, @@ -298,7 +302,7 @@ def decide(self, context: dict) -> MemberAction: [*messages, {"role": "user", "content": repair_message}], system_prompt=_SYSTEM_PROMPT, temperature=0.0 if not reasoning_model else None, - max_tokens=4096 if deepseek or reasoning_model else 1024, + max_tokens=16384 if reasoning_model else (4096 if deepseek else 1024), structured=True, reasoning_effort=reasoning_effort, thinking="disabled" if deepseek else None, @@ -362,7 +366,15 @@ def _default_profile( thinking: str | None, ) -> _OpenAIProfile: reasoning_model = _is_reasoning_model(self.config.model) - native_or_reasoning = _is_native_openai(self.config.base_url) or reasoning_model + # Reasoning-model names alone do not guarantee that a compatible + # gateway implements OpenAI's strict JSON Schema wire shape. When an + # operator explicitly selects Chat Completions on a non-native + # endpoint, prefer the broadly supported JSON Object mode (and its + # matching token parameter). Native OpenAI keeps the strict schema + # profile used by Responses/Chat reasoning models. + native_or_reasoning = _is_native_openai(self.config.base_url) or ( + reasoning_model and self.config.api_surface == "auto" + ) if not structured: structured_mode = "none" elif self.config.api_format == "deepseek": @@ -771,8 +783,14 @@ def _provider_error_from_response(response: Any, provider: str) -> ProviderError status = _response_status(response) retry_after = _retry_after_seconds(response) error_type = RetryableProviderError if _is_retryable_status(status) else NonRetryableProviderError + # Surface the gateway's own reason (rate limit, content filter, schema + # rejection...) — a bare "HTTP 400" is undiagnosable from logs. + detail = (getattr(response, "text", "") or "").strip()[:500] + message = f"{provider} endpoint returned HTTP {status}" + if detail: + message = f"{message} | gateway response: {detail}" return error_type( - f"{provider} endpoint returned HTTP {status}", + message, provider=provider, status_code=status, retry_after=retry_after, @@ -882,6 +900,9 @@ def _http_error_summary(exc: Exception) -> str: def _is_reasoning_model(model: str) -> bool: name = (model or "").strip().lower().rsplit("/", 1)[-1] + # Kimi For Coding always thinks; give it the reasoning-model budget. + if "kimi-for-coding" in name or name.startswith("kimi-k"): + return True if name.startswith("gpt-"): version = name[4:].split("-", 1)[0] try: diff --git a/backend/members/base_member.py b/backend/members/base_member.py index d2151ea..508e808 100644 --- a/backend/members/base_member.py +++ b/backend/members/base_member.py @@ -1214,6 +1214,36 @@ def _build_context(self, project_id, intent_id, category, step, is_initial, eval if callable(attachment_path): for attachment in attachments: attachment["path"] = attachment_path(attachment["filename"], attachment["path"]) + runtime_notes = [ + "If sandbox_backend is LocalSandbox, use host shell-compatible commands only.", + "If sandbox_backend is MemberSandbox, use Linux commands inside the shared task container.", + "Flag search priority for this round: try /flag first, then environment variables, then other methods.", + "If attachment_true is true, inspect the listed attachments before blind target probing.", + ( + "challenge_description contains the original challenge statement " + "(connection commands like nc/ssh, service addresses, flag format). " + "external_id is this challenge's platform id — use it with ret2shell MCP tools." + ), + ] + # Platform verdicts flow back through the experience memory: surface + # any flags the platform already rejected for THIS challenge so the + # member never resubmits them and re-derives a complete flag instead. + try: + rejected_flags: list[str] = [] + for m in d.memory.list(None): + if m.project_id != project_id or "rejected-flag" not in m.tags: + continue + rejected_flags.extend(re.findall(r"'([^']+)'", m.content)) + if rejected_flags: + runtime_notes.append( + "Platform already REJECTED these flag strings for this challenge — " + f"do NOT resubmit them: {', '.join(sorted(set(rejected_flags)))}. " + "Re-derive the complete flag from the challenge material (check every " + "file/resource, including binary XML and encoded blobs; recombine ALL " + "fragments) and only report a flag that reads as a coherent whole." + ) + except Exception: + pass return { "role": self.name, "role_blurb": self.role_blurb, @@ -1228,16 +1258,15 @@ def _build_context(self, project_id, intent_id, category, step, is_initial, eval "or new evidence justifies escalation. Do not repeat previous attempts." ), "sandbox_backend": getattr(d.sandbox, "__class__", type(d.sandbox)).__name__, - "runtime_notes": [ - "If sandbox_backend is LocalSandbox, use host shell-compatible commands only.", - "If sandbox_backend is MemberSandbox, use Linux commands inside the shared task container.", - "Flag search priority for this round: try /flag first, then environment variables, then other methods.", - "If attachment_true is true, inspect the listed attachments before blind target probing.", - ], + "runtime_notes": runtime_notes, "evaluate_now": evaluate_now, "eval_interval": d.eval_interval, "is_initial": is_initial, "expected_flag": d.expected_flag, + "challenge_description": next( + (f.description for f in detail.facts if f.id == "origin"), "" + ), + "external_id": detail.project.external_id, "goal": next((f.description for f in detail.facts if f.id == "goal"), ""), "assigned_intent": {"id": intent_id, "description": assigned.description if assigned else ""}, "assigned_intent_sources": assigned.from_ if assigned else ["origin"], diff --git a/tests/test_members.py b/tests/test_members.py index d6b70c5..befbe22 100644 --- a/tests/test_members.py +++ b/tests/test_members.py @@ -489,6 +489,36 @@ def test_member_context_uses_sandbox_visible_attachment_paths(deps): ] +def test_member_context_includes_challenge_description_and_external_id(deps): + db, d, reports, flags = deps + origin = "https://ctf.example/challenges\n\nConnect: nc 10.0.0.1 9999\nFlag format: flag{...}" + with db.connect() as conn: + pid = graph_store.create_project( + conn, "T", origin, "get flag", "pwn", external_id="chal-42" + ) + intent = edge_store.create_intent(conn, pid, ["origin"], "pwn the service", "diamond") + member = create_member(MemberConfig(name="jade", api_format="mock"), d) + + context = member._build_context(pid, intent.id, "pwn", 1, False, False) + + assert context["external_id"] == "chal-42" + assert context["challenge_description"] == origin + assert "nc 10.0.0.1 9999" in context["challenge_description"] + note = next((n for n in context["runtime_notes"] if "challenge_description" in n), "") + assert "ret2shell" in note and "external_id" in note + + +def test_member_context_defaults_without_external_id(deps): + db, d, reports, flags = deps + pid, iid = _project(db) + member = create_member(MemberConfig(name="jade", api_format="mock"), d) + + context = member._build_context(pid, iid, "web", 1, False, False) + + assert context["external_id"] is None + assert context["challenge_description"] == "origin" + + def test_member_report_defaults_to_low_when_unspecified(deps): db, d, reports, flags = deps pid, iid = _project(db) diff --git a/tests/test_ops_agent.py b/tests/test_ops_agent.py index d1c434e..7be0458 100644 --- a/tests/test_ops_agent.py +++ b/tests/test_ops_agent.py @@ -1162,7 +1162,7 @@ def fake_post(url, **kwargs): assert calls[0][0] == "https://api.openai.com/v1/responses" body = calls[0][1]["json"] assert "input" in body and "messages" not in body - assert body["max_output_tokens"] == 4096 + assert body["max_output_tokens"] == 16384 assert body["reasoning"] == {"effort": "none"} assert body["text"]["format"]["type"] == "json_schema" assert "temperature" not in body @@ -1209,7 +1209,7 @@ def fake_post(url, **kwargs): assert calls[0][0].endswith("/v1/responses") assert calls[1][0].endswith("/v1/chat/completions") - assert calls[1][1]["max_completion_tokens"] == 4096 + assert calls[1][1]["max_completion_tokens"] == 16384 assert calls[1][1]["reasoning_effort"] == "none" assert calls[2][0].endswith("/v1/chat/completions") assert sum(url.endswith("/responses") for url, _ in calls) == 1 From 5c2855fb0488a01bb9c06ebee6f3a14a0bbf1f34 Mon Sep 17 00:00:00 2001 From: PureStream108 Date: Mon, 31 Aug 2026 22:19:05 +0800 Subject: [PATCH 06/10] chore: sandbox git-bash detection, higher local limits, UI running-first ordering - LocalSandbox on Windows prefers Git Bash (IPC_LOCAL_BASH, git install, ProgramFiles) and rewrites bare D:\ style paths to /d/ so POSIX member commands behave like the Docker backend; forced UTF-8 IO. - Default limits raised to 12 CPU / 8 concurrent tasks. - The project list sorts running projects first, completed last, newest id first within a group. - Ignore the local .ipc-agent runtime directory. --- .gitignore | 1 + backend/config/limits.yaml | 4 +-- backend/sandbox/sandbox.py | 66 ++++++++++++++++++++++++++++++++++++-- frontend/index.html | 8 ++++- 4 files changed, 74 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 218282d..0f16c18 100644 --- a/.gitignore +++ b/.gitignore @@ -237,3 +237,4 @@ __marimo__/ # Streamlit .streamlit/secrets.toml +.ipc-agent/ diff --git a/backend/config/limits.yaml b/backend/config/limits.yaml index 1a6904d..f806a99 100644 --- a/backend/config/limits.yaml +++ b/backend/config/limits.yaml @@ -1,4 +1,4 @@ limits: - total_cpu: 4 # cores across all containers - max_concurrent_tasks: 5 # max CTF tasks running at once (one container each) + total_cpu: 12 # cores across all containers (4P+8E) + max_concurrent_tasks: 8 # max CTF tasks running at once (one container each) network: true # all containers may reach the network diff --git a/backend/sandbox/sandbox.py b/backend/sandbox/sandbox.py index e0a0c93..10e2ce2 100644 --- a/backend/sandbox/sandbox.py +++ b/backend/sandbox/sandbox.py @@ -1,6 +1,8 @@ from __future__ import annotations import os +import re +import shutil import subprocess import threading from dataclasses import dataclass @@ -10,6 +12,56 @@ from backend.sandbox.webui_proxy import webui_proxy_manager +def _git_bash_executable() -> Path | None: + """Return a Git Bash executable for Windows local sandboxes, if present. + + Members are prompted with POSIX shell commands (``find``, ``head``, pipes, + etc.). Running those commands through ``cmd.exe`` makes the local backend + behave differently from the Docker backend and causes avoidable tool + failures. Git for Windows provides a small, self-contained Bash that is + suitable for development-mode sandboxes. + """ + if os.name != "nt": + return None + + candidates: list[Path] = [] + configured = os.environ.get("IPC_LOCAL_BASH", "").strip() + if configured: + candidates.append(Path(configured)) + + git = shutil.which("git") + if git: + git_path = Path(git).resolve() + # Typical layout: \\cmd\\git.exe -> \\bin\\bash.exe. + if len(git_path.parents) >= 2: + candidates.append(git_path.parents[1] / "bin" / "bash.exe") + + for base in (os.environ.get("ProgramFiles"), os.environ.get("ProgramW6432")): + if base: + candidates.append(Path(base) / "Git" / "bin" / "bash.exe") + + for candidate in candidates: + if candidate.is_file(): + return candidate + return None + + +def _windows_paths_for_bash(command: str) -> str: + """Convert unquoted ``D:\\path`` forms produced by Members to ``/d/path``. + + This intentionally targets only absolute drive paths. It leaves normal + command text untouched and avoids rewriting arbitrary backslash escapes. + """ + def replace_path(match: re.Match[str]) -> str: + return "/" + match.group(1).lower() + "/" + match.group(2).replace("\\", "/") + + return re.sub( + r"(?]*)", + replace_path, + command, + ) + + @dataclass(slots=True) class ExecResult: exit_code: int @@ -57,13 +109,23 @@ def exec(self, command: str, timeout: int = 60) -> ExecResult: if not self._started: self.start() full_env = {**os.environ, **self.env} + full_env.setdefault("PYTHONIOENCODING", "utf-8") + full_env.setdefault("PYTHONUTF8", "1") + bash = _git_bash_executable() + args: str | list[str] = command + use_shell = True + if bash is not None: + args = [str(bash), "-lc", _windows_paths_for_bash(command)] + use_shell = False try: proc = subprocess.run( - command, - shell=True, + args, + shell=use_shell, cwd=str(self.workspace), capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=timeout, env=full_env, ) diff --git a/frontend/index.html b/frontend/index.html index 34d9485..a17e33e 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -962,7 +962,13 @@ }, currentElapsed(){ return this.projectElapsed(this.current?.project); }, - async loadProjects(){ this.projects = await this.api("GET","/projects"); }, + async loadProjects(){ + const list = await this.api("GET","/projects"); + // Running projects first, then queued/created, completed last; newest id first within a group. + const rank = (s) => (s === 'running' ? 0 : (s === 'completed' ? 2 : 1)); + this.projects = [...list].sort((a, b) => + (rank(a.status) - rank(b.status)) || (a.id < b.id ? 1 : -1)); + }, async loadBroadcasts(){ this.broadcasts = await this.api("GET","/broadcasts"); }, async loadProject(id){ this.current = await this.api("GET","/projects/"+id); }, async openProject(id){ From 2edf3706abb85766a4d610255895b8dd3be8b6e8 Mon Sep 17 00:00:00 2001 From: PureStream108 Date: Mon, 31 Aug 2026 23:24:04 +0800 Subject: [PATCH 07/10] chore: remove dead code identified by static audit Remove functions and methods with zero references in production and tests: edge_store.claim_next_intent, ipc.verify_flag_and_wp, Database.run_transaction, ContainerPool.stop_member, TaskSlotLimiter.active_count/reset, IPCLogger.clear_all/write_jsonl, Lifecycle.can_transition, the unused Category alias, Memory.keywords, and MCPClient.streamable_http/describe (no caller ever builds a URL-target client, so the streamable-http branch, the str target type and the headers parameter go with it). Config's _name_members/_check_unique were audit candidates but are live pydantic validators and stay. --- backend/blackboard/edge_store.py | 38 ----------------------------- backend/core/config.py | 1 - backend/core/ipc.py | 14 ----------- backend/core/lifecycle.py | 3 --- backend/core/logging_util.py | 9 ------- backend/mcp/mcp_client.py | 31 ++--------------------- backend/memory/memory_store.py | 4 --- backend/persistence/database.py | 13 ---------- backend/sandbox/container_pool.py | 10 -------- backend/sandbox/resource_limiter.py | 9 ------- 10 files changed, 2 insertions(+), 130 deletions(-) diff --git a/backend/blackboard/edge_store.py b/backend/blackboard/edge_store.py index bdeab27..2214182 100644 --- a/backend/blackboard/edge_store.py +++ b/backend/blackboard/edge_store.py @@ -330,41 +330,3 @@ def expire_workers(conn: Any, timeout: int, project_id: str | None = None) -> No query += " AND project_id = %s" params += (project_id,) conn.execute(query, params) - - -def claim_next_intent( - conn: Any, - project_id: str, - worker: str, - lease_owner: str, - *, - timeout: int = 30, -) -> tuple[str, str] | None: - """Atomically select and claim one open Intent across app instances.""" - - candidate = conn.execute( - """ - SELECT id - FROM intents - WHERE project_id = %s AND concluded_at IS NULL - AND ( - lease_owner IS NULL - OR (lease_expires_at IS NOT NULL AND lease_expires_at <= now()) - ) - ORDER BY created_at, id - FOR UPDATE SKIP LOCKED - LIMIT 1 - """, - (project_id,), - ).fetchone() - if candidate is None: - return None - token = claim_intent( - conn, - project_id, - candidate["id"], - worker, - lease_owner=lease_owner, - timeout=timeout, - ) - return (candidate["id"], token) if token else None diff --git a/backend/core/config.py b/backend/core/config.py index 3e1520d..9d2c4f3 100644 --- a/backend/core/config.py +++ b/backend/core/config.py @@ -10,7 +10,6 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator # CTF challenge categories the user may pick when creating a project. -Category = Literal["pwn", "reverse", "crypto", "web", "misc", "ai", "osint"] CATEGORIES: tuple[str, ...] = ("pwn", "reverse", "crypto", "web", "misc", "ai", "osint") # Supported LLM wire formats. base_url is always user-provided. diff --git a/backend/core/ipc.py b/backend/core/ipc.py index a1f239b..37d3f41 100644 --- a/backend/core/ipc.py +++ b/backend/core/ipc.py @@ -139,17 +139,3 @@ def verify_postprocess(db, project_id: str, wp_dir: Path) -> dict: ): reasons.extend(f"invalid writeup: {error}" for error in errors) return {"ok": not reasons, "flag": flag, "wp_path": wp_path, "reasons": reasons} - - -def verify_flag_and_wp(db, project_id: str, wp_dir: Path) -> dict: - """Compatibility aggregate used by older callers and export checks.""" - - flag = verify_flag(db, project_id) - postprocess = verify_postprocess(db, project_id, wp_dir) - reasons = [*flag.get("reasons", []), *postprocess.get("reasons", [])] - return { - "ok": not reasons, - "flag": flag.get("flag"), - "wp_path": postprocess.get("wp_path"), - "reasons": reasons, - } diff --git a/backend/core/lifecycle.py b/backend/core/lifecycle.py index adb9fb3..e07e1ca 100644 --- a/backend/core/lifecycle.py +++ b/backend/core/lifecycle.py @@ -28,9 +28,6 @@ def status(self, project_id: str) -> str | None: row = graph_store.get_project_row(conn, project_id) return row["status"] if row else None - def can_transition(self, current: str, target: str) -> bool: - return target in TRANSITIONS.get(current, set()) - def transition(self, project_id: str, target: str) -> str: with self.db.connect() as conn: row = graph_store.get_project_row(conn, project_id) diff --git a/backend/core/logging_util.py b/backend/core/logging_util.py index bb11200..3343920 100644 --- a/backend/core/logging_util.py +++ b/backend/core/logging_util.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import shutil import threading from collections.abc import Callable from datetime import UTC, datetime @@ -101,10 +100,6 @@ def delete_project_logs(self, project_id: str) -> None: path = self.root / sub / self._project_filename(project_id) path.unlink(missing_ok=True) - def clear_all(self) -> None: - with self._lock: - shutil.rmtree(self.root, ignore_errors=True) - @staticmethod def _read_array(path: Path) -> list[dict]: if not path.exists(): @@ -140,7 +135,3 @@ def _tail(path: Path, limit: int | None) -> list[dict]: @staticmethod def jsonl_text(entries: list[dict]) -> str: return "".join(json.dumps(entry, ensure_ascii=False) + "\n" for entry in entries) - - @staticmethod - def write_jsonl(path: Path, entries: list[dict]) -> None: - path.write_text(IPCLogger.jsonl_text(entries), encoding="utf-8") diff --git a/backend/mcp/mcp_client.py b/backend/mcp/mcp_client.py index 7d3a850..3e3d91e 100644 --- a/backend/mcp/mcp_client.py +++ b/backend/mcp/mcp_client.py @@ -12,11 +12,10 @@ from mcp import ClientSession, StdioServerParameters, types from mcp.client.stdio import stdio_client -from mcp.client.streamable_http import streamablehttp_client from mcp.server.fastmcp import FastMCP from mcp.shared.memory import create_connected_server_and_client_session -MCPClientTarget = FastMCP | StdioServerParameters | str +MCPClientTarget = FastMCP | StdioServerParameters class MCPToolError(RuntimeError): @@ -54,11 +53,9 @@ def __init__( target: MCPClientTarget, *, read_timeout: float | None = 120, - headers: Mapping[str, str] | None = None, ) -> None: self.target = target self.read_timeout = read_timeout - self.headers = dict(headers or {}) self._stack: AsyncExitStack | None = None self._session: ClientSession | None = None @@ -81,16 +78,6 @@ def stdio( **kwargs, ) - @classmethod - def streamable_http( - cls, - url: str, - *, - headers: Mapping[str, str] | None = None, - **kwargs: Any, - ) -> MCPClient: - return cls(url, headers=headers, **kwargs) - @property def session(self) -> ClientSession: if self._session is None: @@ -112,17 +99,7 @@ async def __aenter__(self) -> MCPClient: ) ) else: - if isinstance(self.target, StdioServerParameters): - read_stream, write_stream = await stack.enter_async_context(stdio_client(self.target)) - else: - read_stream, write_stream, _ = await stack.enter_async_context( - streamablehttp_client( - self.target, - headers=self.headers, - timeout=self.read_timeout or 30, - sse_read_timeout=self.read_timeout or 300, - ) - ) + read_stream, write_stream = await stack.enter_async_context(stdio_client(self.target)) session = await stack.enter_async_context( ClientSession(read_stream, write_stream, read_timeout_seconds=timeout) ) @@ -259,10 +236,6 @@ async def call_tool( async with self.session() as session: return await session.call_tool(server, tool, arguments) - async def describe(self) -> dict[str, list[dict[str, Any]]]: - async with self.session() as session: - return {name: await session.describe_tools(name) for name in self.names()} - async def _run_cli(args: argparse.Namespace) -> int: server_args = [ diff --git a/backend/memory/memory_store.py b/backend/memory/memory_store.py index dd5bf2c..ba46d95 100644 --- a/backend/memory/memory_store.py +++ b/backend/memory/memory_store.py @@ -26,10 +26,6 @@ class Memory(BaseModel): source: str = "diamond" created_at: str - def keywords(self) -> set[str]: - text = f"{self.title} {self.content} {' '.join(self.tags)}".lower() - return {word for word in _tokenize(text) if len(word) >= 2} - def _tokenize(text: str) -> list[str]: output: list[str] = [] diff --git a/backend/persistence/database.py b/backend/persistence/database.py index f28cff5..0a110ac 100644 --- a/backend/persistence/database.py +++ b/backend/persistence/database.py @@ -1,8 +1,6 @@ from __future__ import annotations import os -import random -import time from collections.abc import Generator, Iterable, Mapping from contextlib import contextmanager from datetime import date, datetime @@ -10,7 +8,6 @@ from typing import Any from psycopg import Connection -from psycopg.errors import DeadlockDetected, SerializationFailure from psycopg.rows import dict_row from psycopg_pool import ConnectionPool @@ -163,16 +160,6 @@ def connect(self) -> Generator[DatabaseConnection, None, None]: with raw.transaction(): yield DatabaseConnection(raw) - def run_transaction(self, callback, *, retries: int = 3): - for attempt in range(retries): - try: - with self.connect() as connection: - return callback(connection) - except (DeadlockDetected, SerializationFailure): - if attempt + 1 >= retries: - raise - time.sleep((0.05 * (2**attempt)) + random.random() * 0.05) - def close(self) -> None: if self._opened: self._pool.close() diff --git a/backend/sandbox/container_pool.py b/backend/sandbox/container_pool.py index 23cfd07..bab2124 100644 --- a/backend/sandbox/container_pool.py +++ b/backend/sandbox/container_pool.py @@ -7,7 +7,6 @@ from backend.sandbox.resource_limiter import TaskSlotLimiter from backend.sandbox.sandbox import LocalSandbox, Sandbox -from backend.sandbox.webui_proxy import webui_proxy_manager _SAFE_SEGMENT = re.compile(r"[^A-Za-z0-9_.-]+") @@ -114,15 +113,6 @@ def _new_task(self, project_id: str, env: dict[str, str] | None): attachments_dir=self.workspace_root / project_id / "attachments", ) - def stop_member(self, project_id: str, member: str) -> None: - """Drop a Member's view. Does NOT stop the shared task container.""" - key = self._key(project_id, member) - with self._lock: - sb = self._sandboxes.pop(key, None) - if sb is not None: - sb.stop() - webui_proxy_manager.close_member(project_id, member) - def stop_project(self, project_id: str) -> None: with self._lock: keys = [k for k in self._sandboxes if k[0] == project_id] diff --git a/backend/sandbox/resource_limiter.py b/backend/sandbox/resource_limiter.py index e9ca150..cec7d10 100644 --- a/backend/sandbox/resource_limiter.py +++ b/backend/sandbox/resource_limiter.py @@ -43,12 +43,3 @@ def release(self, project_id: str) -> None: def active_tasks(self) -> list[str]: with self._lock: return sorted(self._active) - - @property - def active_count(self) -> int: - with self._lock: - return len(self._active) - - def reset(self) -> None: - with self._lock: - self._active.clear() From 015f5ab1113f918de4048a51ee55228d313e60bb Mon Sep 17 00:00:00 2001 From: PureStream108 Date: Mon, 31 Aug 2026 23:24:04 +0800 Subject: [PATCH 08/10] docs: document platform layer and add ret2 contributor - Repository layout gains backend/platform (HTTP JSON / GZCTF / ret2shell clients). - MCP list documents the conditional ret2shell instance-control MCP. - Contributors: credit the ret2 branch committer for the competition platform integration and stability fixes. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 0a779a6..1a5b0b9 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ Member 在题目容器中使用按分类注册的 CTF 工具,并可通过 MCP - `browser`:基于 Chromium 的浏览器自动化、下载与截图 Artifact。 - `reverse`:PyGhidra 与 radare2 逆向分析。 - `zap`:可选 OWASP ZAP 服务,必须同时开启运行配置和 Compose profile。 +- `ret2shell`:ret2shell 平台的动态实例控制(`instance_start` / `instance_status` / `instance_renew` / `instance_stop` / `challenge_status`),仅在配置 `IPC_R2S_USERNAME` 或 `IPC_R2S_TOKEN` 后注册;ws:// 隧道由镜像内置的 wsrx 转发到本地端口。 分类覆盖 `web`、`pwn`、`reverse`、`crypto`、`misc`、`ai`、`osint`。浏览器下载、截图等内容以 Artifact 保存,避免将大型输出直接放入模型上下文。 @@ -357,6 +358,7 @@ IPC_CTFAgent/ │ ├── memory/ # 经验记忆与工具目录 │ ├── ops/ # IPC 行动代理与平台工作流 │ ├── persistence/ # PostgreSQL schema 与 Alembic migrations +│ ├── platform/ # 平台适配层(HTTP JSON / GZCTF / ret2shell 客户端) │ ├── sandbox/ # Docker/本地任务沙箱 │ └── tools/ # 工具注册表、目录与文档 ├── frontend/ # 单页 Web UI @@ -426,6 +428,8 @@ docker run --rm \

+- `unknown <2922592941@qq.com>` — ret2 分支:ret2shell/GZCTF 平台接入、调度稳定性与成员上下文修复 + --- From 958c370a5032bb54d9040adf7fbe7e4f965fc5ab Mon Sep 17 00:00:00 2001 From: PureStream108 Date: Mon, 31 Aug 2026 23:34:58 +0800 Subject: [PATCH 09/10] chore: ignore local docs/ scratch directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0f16c18..b80f7ff 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ data/ memory/ projects/ Wp/ +docs/ exports/ examples/ log.txt From b64524b3d425c8239013185c8846c7aa3cb85680 Mon Sep 17 00:00:00 2001 From: PureStream108 Date: Mon, 31 Aug 2026 23:34:58 +0800 Subject: [PATCH 10/10] docs: link ret2 contributor's GitHub profile in Contributors Replace the plain-text git identity with springbot2025's avatar entry, and adjust the browser requirement wording. --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1a5b0b9..002a3be 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ Docker Compose 将数据库放入命名卷,将需要人工访问或跨容器 | Docker Socket | 必需 | App 需要创建题目容器;IPC Runner 同样依赖 Socket | | Linux Docker 主机 | 推荐/已验证 | Compose 会挂载 Docker Socket 与 Compose 插件 | | LLM 端点 | 开始解题时需要 | Diamond 与已启用 Member 使用 OpenAI、Anthropic、Claude Code、DeepSeek、Pi 或 Mock 适配器 | -| 浏览器 | 可选 | 用于打开本地 Web UI | +| 浏览器 | 可选 | 用于打开本地浏览器 | > [!WARNING] > 沙箱降低风险,但不能替代隔离主机或虚拟机。请只将题目、附件、模型密钥和 Docker Socket 放入可信环境。 @@ -426,13 +426,11 @@ docker run --rm \ xz w + + xz w +

-- `unknown <2922592941@qq.com>` — ret2 分支:ret2shell/GZCTF 平台接入、调度稳定性与成员上下文修复 - - ---- - ## 🤝 Contribution 欢迎提交 bug 报告、稳定性测试、工具集成、文档和架构改进。