Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ data/
memory/
projects/
Wp/
docs/
exports/
examples/
log.txt
Expand Down Expand Up @@ -237,3 +238,4 @@ __marimo__/

# Streamlit
.streamlit/secrets.toml
.ipc-agent/
13 changes: 13 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,19 @@ COPY scripts /app/scripts

RUN pip install --no-cache-dir -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple -e ".[docker]"

# wsrx (WebSocket Reflector X CLI from XDSEC, static musl build) tunnels the
# ret2shell platform's ws://-proxied challenge ports to local TCP ports so
# Members can reach dynamic instances over the docker network. The ret2shell
# backend itself is only installed in this image because the tunnel
# subprocesses are owned by the backend process (Members just connect to
# ipc-app:<port>).
ARG WSRX_VERSION=0.6.1
RUN python -c "import urllib.request; urllib.request.urlretrieve('https://github.com/XDSEC/WebSocketReflectorX/releases/download/${WSRX_VERSION}/wsrx-cli-${WSRX_VERSION}-linux-musl-x86_64.tar.gz', '/tmp/wsrx.tar.gz')" \
&& tar -xzf /tmp/wsrx.tar.gz -C /tmp \
&& install -m 0755 /tmp/wsrx /usr/local/bin/wsrx \
&& rm -f /tmp/wsrx /tmp/wsrx.tar.gz \
&& wsrx --version

# /app/data is bind-mounted by docker-compose.yml. PostgreSQL owns runtime
# facts; filesystem workspaces and generated files share one Artifact tree.
RUN mkdir -p /app/data/artifacts/projects \
Expand Down
10 changes: 6 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ Member 在题目容器中使用按分类注册的 CTF 工具,并可通过 MCP
- `browser`:基于 Chromium 的浏览器自动化、下载与截图 Artifact。
- `reverse`:PyGhidra 与 radare2 逆向分析。
- `zap`:可选 OWASP ZAP 服务,必须同时开启运行配置和 Compose profile。
- `ret2shell`:ret2shell 平台的动态实例控制(`instance_start` / `instance_status` / `instance_renew` / `instance_stop` / `challenge_status`),仅在配置 `IPC_R2S_USERNAME` 或 `IPC_R2S_TOKEN` 后注册;ws:// 隧道由镜像内置的 wsrx 转发到本地端口。

分类覆盖 `web`、`pwn`、`reverse`、`crypto`、`misc`、`ai`、`osint`。浏览器下载、截图等内容以 Artifact 保存,避免将大型输出直接放入模型上下文。

Expand Down Expand Up @@ -141,7 +142,7 @@ Docker Compose 将数据库放入命名卷,将需要人工访问或跨容器
| Docker Socket | 必需 | App 需要创建题目容器;IPC Runner 同样依赖 Socket |
| Linux Docker 主机 | 推荐/已验证 | Compose 会挂载 Docker Socket 与 Compose 插件 |
| LLM 端点 | 开始解题时需要 | Diamond 与已启用 Member 使用 OpenAI、Anthropic、Claude Code、DeepSeek、Pi 或 Mock 适配器 |
| 浏览器 | 可选 | 用于打开本地 Web UI |
| 浏览器 | 可选 | 用于打开本地浏览器 |

> [!WARNING]
> 沙箱降低风险,但不能替代隔离主机或虚拟机。请只将题目、附件、模型密钥和 Docker Socket 放入可信环境。
Expand Down Expand Up @@ -357,6 +358,7 @@ IPC_CTFAgent/
│ ├── memory/ # 经验记忆与工具目录
│ ├── ops/ # IPC 行动代理与平台工作流
│ ├── persistence/ # PostgreSQL schema 与 Alembic migrations
│ ├── platform/ # 平台适配层(HTTP JSON / GZCTF / ret2shell 客户端)
│ ├── sandbox/ # Docker/本地任务沙箱
│ └── tools/ # 工具注册表、目录与文档
├── frontend/ # 单页 Web UI
Expand Down Expand Up @@ -424,11 +426,11 @@ docker run --rm \
<a href="https://github.com/ecxwxz">
<img src="https://github.com/ecxwxz.png?size=80" width="80" height="80" style="border-radius: 50%;" alt="xz w" title="xz w" />
</a>
<a href="https://github.com/springbot2025">
<img src="https://github.com/springbot2025.png?size=80" width="80" height="80" style="border-radius: 50%;" alt="xz w" title="xz w" />
</a>
</p>


---

## 🤝 Contribution

欢迎提交 bug 报告、稳定性测试、工具集成、文档和架构改进。
Expand Down
77 changes: 63 additions & 14 deletions backend/api/platform.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
from __future__ import annotations

import re
import shutil
import time
from pathlib import Path

import requests
from fastapi import APIRouter, Depends, HTTPException
Expand All @@ -9,12 +12,18 @@
from backend.api.deps import get_state
from backend.blackboard import graph_store
from backend.core.state import AppState
from backend.platform.adapter import HttpJsonAdapter
from backend.platform.adapter import HttpJsonAdapter, PlatformAdapter
from backend.platform.mapping import FieldMapping, PlatformChallenge
from backend.platform.ret2shell import Ret2ShellAdapter, Ret2ShellClient, Ret2ShellError

router = APIRouter(prefix="/api/platform", tags=["platform"])


def safe_name(external_id: str) -> str:
cleaned = re.sub(r"[^A-Za-z0-9._-]", "_", external_id).strip("._") or "challenge"
return cleaned[:80]


class ChallengeRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
mapping: FieldMapping
Expand All @@ -24,12 +33,25 @@ class ImportRequest(ChallengeRequest):
select: list[str] | None = None


def _fetch(mapping: FieldMapping) -> tuple[HttpJsonAdapter, list[PlatformChallenge]]:
adapter = HttpJsonAdapter(mapping)
def _build_adapter(mapping: FieldMapping) -> PlatformAdapter:
if mapping.platform == "ret2shell":
client = Ret2ShellClient(base_url=mapping.list_url, game_id=mapping.game_id)
return Ret2ShellAdapter(
client,
game_id=mapping.game_id or None,
category_map=mapping.category_map,
)
return HttpJsonAdapter(mapping)


def _fetch(mapping: FieldMapping) -> tuple[PlatformAdapter, list[PlatformChallenge]]:
adapter = _build_adapter(mapping)
try:
return adapter, adapter.fetch_challenges()
except requests.RequestException as exc:
raise HTTPException(502, f"platform request failed: {exc}") from exc
except Ret2ShellError as exc:
raise HTTPException(502, f"ret2shell request failed: {exc}") from exc
except (TypeError, ValueError) as exc:
raise HTTPException(400, str(exc)) from exc

Expand All @@ -52,29 +74,58 @@ def import_challenges(body: ImportRequest, state: AppState = Depends(get_state))
raise HTTPException(400, f"unknown external_id values: {missing}")
selected = [by_id[external_id] for external_id in body.select]

# Phase 1 downloads every attachment while NO database connection is
# open. Holding the write transaction across network downloads starved
# the UI polling loop (and itself) with "database is locked" on large
# imports. Phase 2 then only performs fast local writes.
staging_root = state.root / "staging" / f"import_{int(time.time() * 1000)}"
staged: dict[str, list[Path]] = {}
try:
for challenge in selected:
staged[challenge.external_id] = adapter.download_attachments(
challenge,
staging_root / safe_name(challenge.external_id),
)
except requests.RequestException as exc:
shutil.rmtree(staging_root, ignore_errors=True)
raise HTTPException(502, f"attachment download failed: {exc}") from exc
except Ret2ShellError as exc:
shutil.rmtree(staging_root, ignore_errors=True)
raise HTTPException(502, f"ret2shell attachment download failed: {exc}") from exc
except (OSError, TypeError, ValueError) as exc:
shutil.rmtree(staging_root, ignore_errors=True)
raise HTTPException(400, str(exc)) from exc

imported: list[dict[str, str]] = []
created_project_ids: list[str] = []
try:
with state.db.connect() as conn:
for challenge in selected:
# Mirror the ops workflow importer: the challenge statement
# (connection commands, flag format) rides in the origin fact
# so the Member context's challenge_description picks it up.
origin = body.mapping.list_url
if challenge.description:
origin = f"{origin}\n\n{challenge.description}"
project_id = graph_store.create_project(
conn,
challenge.title,
body.mapping.list_url,
origin,
"capture the flag",
challenge.category,
external_id=challenge.external_id,
)
created_project_ids.append(project_id)
for path in adapter.download_attachments(
challenge,
state.attachments_dir(project_id),
):
attachment_dir = state.attachments_dir(project_id)
attachment_dir.mkdir(parents=True, exist_ok=True)
for path in staged.get(challenge.external_id, []):
target = attachment_dir / path.name
shutil.move(str(path), str(target))
graph_store.create_attachment(
conn,
project_id,
path.name,
str(path),
target.name,
str(target),
)
imported.append(
{
Expand All @@ -84,14 +135,12 @@ def import_challenges(body: ImportRequest, state: AppState = Depends(get_state))
"category": challenge.category,
}
)
except requests.RequestException as exc:
for project_id in created_project_ids:
shutil.rmtree(state.projects_dir / project_id, ignore_errors=True)
raise HTTPException(502, f"attachment download failed: {exc}") from exc
except (OSError, TypeError, ValueError) as exc:
for project_id in created_project_ids:
shutil.rmtree(state.projects_dir / project_id, ignore_errors=True)
raise HTTPException(400, str(exc)) from exc
finally:
shutil.rmtree(staging_root, ignore_errors=True)

for item in imported:
state.logger.project(
Expand Down
38 changes: 0 additions & 38 deletions backend/blackboard/edge_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -330,41 +330,3 @@ def expire_workers(conn: Any, timeout: int, project_id: str | None = None) -> No
query += " AND project_id = %s"
params += (project_id,)
conn.execute(query, params)


def claim_next_intent(
conn: Any,
project_id: str,
worker: str,
lease_owner: str,
*,
timeout: int = 30,
) -> tuple[str, str] | None:
"""Atomically select and claim one open Intent across app instances."""

candidate = conn.execute(
"""
SELECT id
FROM intents
WHERE project_id = %s AND concluded_at IS NULL
AND (
lease_owner IS NULL
OR (lease_expires_at IS NOT NULL AND lease_expires_at <= now())
)
ORDER BY created_at, id
FOR UPDATE SKIP LOCKED
LIMIT 1
""",
(project_id,),
).fetchone()
if candidate is None:
return None
token = claim_intent(
conn,
project_id,
candidate["id"],
worker,
lease_owner=lease_owner,
timeout=timeout,
)
return (candidate["id"], token) if token else None
51 changes: 51 additions & 0 deletions backend/blackboard/node_store.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,58 @@
from __future__ import annotations

import re
from typing import Any

from backend.blackboard.ids import next_fact_id, utcnow
from backend.blackboard.models import Fact


# Agent reports often restate the same source finding with a different lead-in
# (for example, "Nday candidate" versus "confirmed candidate"). Keeping the
# full prose is useful for audit, but these stable tokens are enough to prevent
# a repeated conclusion from inflating the graph and retriggering Diamond.
_FINDING_RE = re.compile(
r"(?P<path>[A-Za-z0-9_./-]+\.(?:c|cc|cpp|cxx|h|hpp|py|js|ts|java))"
r"(?::|\s+lines?\s+)(?P<line>\d+(?:-\d+)?)"
r".*?(?P<commit>\b[0-9a-f]{7,40}\b)",
re.IGNORECASE | re.DOTALL,
)


def _finding_key(description: str) -> tuple[str, str, str] | None:
match = _FINDING_RE.search(description)
if match is None:
return None
return (
match.group("path").replace("\\", "/").lower(),
match.group("line"),
match.group("commit").lower(),
)


def _existing_equivalent_fact(conn: Any, project_id: str, description: str) -> Fact | None:
normalized = " ".join(description.split())
row = conn.execute(
"SELECT id, description, created_at FROM facts "
"WHERE project_id = %s AND description = %s ORDER BY created_at, id LIMIT 1",
(project_id, normalized),
).fetchone()
if row is not None:
return Fact(id=row["id"], description=row["description"], created_at=row["created_at"])

key = _finding_key(normalized)
if key is None:
return None
rows = conn.execute(
"SELECT id, description, created_at FROM facts WHERE project_id = %s ORDER BY created_at, id",
(project_id,),
).fetchall()
for row in rows:
if _finding_key(row["description"]) == key:
return Fact(id=row["id"], description=row["description"], created_at=row["created_at"])
return None


def insert_fact(conn: Any, project_id: str, fact_id: str, description: str) -> Fact:
now = utcnow()
conn.execute(
Expand All @@ -26,6 +73,10 @@ def reserve_fact(conn: Any, project_id: str, description: str) -> Fact:


def create_fact(conn: Any, project_id: str, description: str) -> Fact:
description = " ".join(description.split())
existing = _existing_equivalent_fact(conn, project_id, description)
if existing is not None:
return existing
fid = next_fact_id(conn, project_id)
return insert_fact(conn, project_id, fid, description)

Expand Down
4 changes: 2 additions & 2 deletions backend/config/limits.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
limits:
total_cpu: 4 # cores across all containers
max_concurrent_tasks: 5 # max CTF tasks running at once (one container each)
total_cpu: 12 # cores across all containers (4P+8E)
max_concurrent_tasks: 8 # max CTF tasks running at once (one container each)
network: true # all containers may reach the network
1 change: 0 additions & 1 deletion backend/core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator

# CTF challenge categories the user may pick when creating a project.
Category = Literal["pwn", "reverse", "crypto", "web", "misc", "ai", "osint"]
CATEGORIES: tuple[str, ...] = ("pwn", "reverse", "crypto", "web", "misc", "ai", "osint")

# Supported LLM wire formats. base_url is always user-provided.
Expand Down
14 changes: 0 additions & 14 deletions backend/core/ipc.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,17 +139,3 @@ def verify_postprocess(db, project_id: str, wp_dir: Path) -> dict:
):
reasons.extend(f"invalid writeup: {error}" for error in errors)
return {"ok": not reasons, "flag": flag, "wp_path": wp_path, "reasons": reasons}


def verify_flag_and_wp(db, project_id: str, wp_dir: Path) -> dict:
"""Compatibility aggregate used by older callers and export checks."""

flag = verify_flag(db, project_id)
postprocess = verify_postprocess(db, project_id, wp_dir)
reasons = [*flag.get("reasons", []), *postprocess.get("reasons", [])]
return {
"ok": not reasons,
"flag": flag.get("flag"),
"wp_path": postprocess.get("wp_path"),
"reasons": reasons,
}
3 changes: 0 additions & 3 deletions backend/core/lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,6 @@ def status(self, project_id: str) -> str | None:
row = graph_store.get_project_row(conn, project_id)
return row["status"] if row else None

def can_transition(self, current: str, target: str) -> bool:
return target in TRANSITIONS.get(current, set())

def transition(self, project_id: str, target: str) -> str:
with self.db.connect() as conn:
row = graph_store.get_project_row(conn, project_id)
Expand Down
9 changes: 0 additions & 9 deletions backend/core/logging_util.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from __future__ import annotations

import json
import shutil
import threading
from collections.abc import Callable
from datetime import UTC, datetime
Expand Down Expand Up @@ -101,10 +100,6 @@ def delete_project_logs(self, project_id: str) -> None:
path = self.root / sub / self._project_filename(project_id)
path.unlink(missing_ok=True)

def clear_all(self) -> None:
with self._lock:
shutil.rmtree(self.root, ignore_errors=True)

@staticmethod
def _read_array(path: Path) -> list[dict]:
if not path.exists():
Expand Down Expand Up @@ -140,7 +135,3 @@ def _tail(path: Path, limit: int | None) -> list[dict]:
@staticmethod
def jsonl_text(entries: list[dict]) -> str:
return "".join(json.dumps(entry, ensure_ascii=False) + "\n" for entry in entries)

@staticmethod
def write_jsonl(path: Path, entries: list[dict]) -> None:
path.write_text(IPCLogger.jsonl_text(entries), encoding="utf-8")
Loading