diff --git a/.github/workflows/http-proxy-pool.yml b/.github/workflows/http-proxy-pool.yml new file mode 100644 index 000000000..fe4d5798a --- /dev/null +++ b/.github/workflows/http-proxy-pool.yml @@ -0,0 +1,22 @@ +name: HTTP proxy pool +on: + push: + paths: ['tools/http-framework-test.yaml', 'scripts/test_http_proxy_pool.py', 'requirements.txt', '.github/workflows/http-proxy-pool.yml'] + pull_request: + paths: ['tools/http-framework-test.yaml', 'scripts/test_http_proxy_pool.py', 'requirements.txt', '.github/workflows/http-proxy-pool.yml'] +permissions: + contents: read +jobs: + proxy-pool: + strategy: + fail-fast: false + matrix: + httpx: ['0.27.0', '0.28.1'] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: python -m pip install 'httpx[http2,socks]==${{ matrix.httpx }}' PyYAML + - run: python -m unittest discover -s scripts -p 'test_http_proxy_pool.py' -v diff --git a/CHANGELOG.md b/CHANGELOG.md index 507404494..dc35a00ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Added +- 系统设置新增代理二级菜单，提供固定出口、代理池、最短切换间隔及凭证脱敏；配置仅注入后续 HTTP 工具进程，不影响 Kali 其他工具和模型 API。`http-framework-test` 新增可选代理池：支持静态/结构化代理列表与供应商接口、粘性/轮询策略、供应商故障静态回退、失败冷却及有界切换；仅自动重放 GET/HEAD/OPTIONS，遵守 429 Retry-After，切换失败时返回响应或错误且不回退直连。附中英文指南与本地 HTTP/HTTPS/SOCKS5 回归测试（#357）。 + - 支持通过 `agent.task_timeout_minutes` 配置单代理、多代理、工作流及异步任务执行超时：未配置默认 600 分钟，`0` 表示不限时，负数拒绝加载；续跑保留重新计时行为，单次工具超时仍独立生效。 - 新增单代理与多代理异步对话 API：接收后返回任务回执，客户端断线不停止执行；支持按用户隔离的幂等提交、持久状态与结果查询、显式取消以及服务重启后的中断状态。同步接口保持兼容。 diff --git a/config.example.yaml b/config.example.yaml index 92db51b2b..9ec59fd54 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -608,3 +608,15 @@ project: # 把目录变成跟踪该仓库的 git 工作树，此后按正常安装一键更新。 # update: # repo: https://github.com/AIPentest/CyberStrikeAI.git # 一个仓库地址；留空/不写 = 官方仓库 + +# HTTP 测试工具的受管出口；不影响 Kali 其他工具、外部 MCP 或大模型 API。 +# 固定 URL 是否固定公网 IP 取决于供应商。运行中的工具使用启动时快照。 +proxy: + mode: manual # manual=沿用工具参数，fixed=固定代理，pool=显式启用代理池 + proxy_url: "" # fixed 模式必填，支持 http(s)/socks5://host:port + proxy_pool: "" # pool 模式：每行一个代理 URL；与供应商至少配置一项 + proxy_source: "" # 可选供应商 HTTP(S) 地址 + strategy: sticky # sticky 或 round_robin + max_attempts: 3 + cooldown: 60 + min_switch_interval: 60 # 间隔内即使失败也不提前切换；0 表示不限制 diff --git a/docs/README.md b/docs/README.md index 69137466b..6498427ba 100644 --- a/docs/README.md +++ b/docs/README.md @@ -32,6 +32,7 @@ CyberStrikeAI documentation is organized by user journey. Start with deployment, - [资产管理](zh-CN/asset-management.md) - [项目任务续接](zh-CN/project-resume.md) - [Token 用量分析](zh-CN/token-usage-monitoring.md) +- [HTTP 测试工具代理池](zh-CN/http-proxy-pool.md) - [知识库](zh-CN/knowledge-base.md) - [RBAC 权限管理](zh-CN/rbac.md) - [机器人接入](zh-CN/robot.md) @@ -76,6 +77,7 @@ CyberStrikeAI documentation is organized by user journey. Start with deployment, - [Asset Management](en-US/asset-management.md) - [Resuming Project Tasks](en-US/project-resume.md) - [Token Usage Analysis](en-US/token-usage-monitoring.md) +- [HTTP Test Tool Proxy Pool](en-US/http-proxy-pool.md) - [Knowledge Base](en-US/knowledge-base.md) - [RBAC Administration](en-US/rbac.md) - [Robot / Chatbot](en-US/robot.md) diff --git a/docs/en-US/README.md b/docs/en-US/README.md index 237d12eb4..79883569a 100644 --- a/docs/en-US/README.md +++ b/docs/en-US/README.md @@ -20,6 +20,7 @@ ## Feature guides - [Resuming Project Tasks](project-resume.md) · [Token Usage Analysis](token-usage-monitoring.md) +- [HTTP Test Tool Proxy Pool](http-proxy-pool.md) - [Asset Management](asset-management.md) · [Knowledge Base](knowledge-base.md) · [Robot / Chatbot](robot.md) · [Vision](VISION.md) - [WebShell](webshell.md) · [C2](c2.md) · [MCP Federation](mcp-federation.md) diff --git a/docs/en-US/http-proxy-pool.md b/docs/en-US/http-proxy-pool.md new file mode 100644 index 000000000..d4295749b --- /dev/null +++ b/docs/en-US/http-proxy-pool.md @@ -0,0 +1,99 @@ +# HTTP test tool proxy pool + +[中文](../zh-CN/http-proxy-pool.md) | [Documentation home](README.md) + +`http-framework-test` accepts a user-supplied proxy list and/or provider endpoint. It switches proxies within a bounded attempt budget on transport failures or configured HTTP status codes, and reuses a successful proxy. Settings → Proxy settings can manage egress for this HTTP tool; the default remains per-call arguments. + +## Usage + +After updating the tool file, apply configuration in settings or restart the service to reload its parameters. The proxy submenu also supports persistent settings that apply to subsequent invocations without restarting active processes. + +Example MCP tool arguments: + +```json +{ + "url": "https://your-authorized-target.example/health", + "method": "GET", + "proxy_pool": "[\"http://127.0.0.1:8081\",\"http://127.0.0.1:8082\"]", + "proxy_max_attempts": 3, + "proxy_cooldown": "60", + "proxy_fail_status": "403,429", + "response_max_bytes": 8192 +} +``` + +`proxy_pool` also accepts newline-separated endpoints. Supported forms are `http://host:port`, `https://host:port`, `socks5://host:port`, and bare `host:port` (HTTP). An explicit port is required. IPv6 brackets and URL-encoded `user:password@host:port` credentials are supported. SOCKS support is included in the `httpx[http2,socks]` requirement. + +To fetch proxies automatically, add or use `proxy_source` alone: + +```json +{ + "url": "https://your-authorized-target.example/health", + "proxy_source": "https://your-proxy-provider.example/api/proxies" +} +``` + +The endpoint must return UTF-8 newline-separated endpoints, a JSON array of endpoint strings, or an object containing a `proxies` or `data` array: + +```json +{"proxies": ["http://127.0.0.1:8081", "127.0.0.1:8082"]} +``` + +Entries may also be objects such as `{"host":"127.0.0.1","port":8081,"scheme":"http","username":"user","password":"pass"}` (`ip` aliases `host`) or `{"url":"http://127.0.0.1:8081"}`. Scheme defaults to HTTP, credentials are optional and automatically URL-encoded. Other provider formats require an adapter. No public free-proxy sources are built in. The source is fetched at startup and at most once more per request/redirect hop when available candidates run out. New endpoints are deduplicated and merged without clearing cooldowns. Fetches have 10-second connect/read timeouts, do not follow redirects, accept at most 1 MiB, and the pool holds at most 256 endpoints. An initial fetch failure warns and uses static endpoints when available; source-only calls fail. A refresh failure preserves the last HTTP failure response, or reports an error if no response exists. + +## Behavior + +| Parameter | Default | Meaning | +| --- | --- | --- | +| `proxy_pool` | Empty | JSON string array or newline-separated list | +| `proxy_strategy` | `"sticky"` | `sticky` reuses successful proxies; `round_robin` rotates and skips cooldowns | +| `proxy_source` | Empty | User-configured HTTP(S) provider endpoint, optionally combined with a list | +| `proxy_max_attempts` | `3` | 1–10 distinct proxy attempts per request/redirect hop | +| `proxy_cooldown` | `"60"` | 0–86400 seconds before a failed proxy is eligible again | +| `proxy_fail_status` | `"403,429"` | Comma-separated status codes from 400–599; `"none"` disables status-based switching | + +- Successful proxies stay selected by default. With `proxy_strategy="round_robin"`, each request/redirect hop rotates through eligible proxies. Failed proxies enter cooldown. Each proxy is tried at most once per request/redirect hop. +- Only GET, HEAD and OPTIONS are replayed automatically. POST, PUT, DELETE and other methods retain the first result. An explicit `repeat` still requests additional executions. +- Transport failures before response headers can trigger switching. Failures while reading a response body are reported without replaying a partial response. +- A 429 with a valid future `Retry-After` stops automatic replay and reports that the caller should retry later. Further requests to that origin within the same tool invocation are blocked until the wait expires; waits are recorded for at most 24 hours. +- If the final attempt returns an HTTP failure response, it is preserved with a nonzero exit code. A network failure or a pool with all endpoints cooling down reports an error. There is no direct fallback or extra direct DNS/TCP/TLS target probe. +- Pool diagnostics show attempt numbers, proxy numbers and failure types. They omit proxy credentials and provider URLs. The platform's existing argument traces, audit records and logs still apply; treat arguments containing credentials as sensitive configuration. + +## Scope and design + +[#357](https://github.com/AIPentest/CyberStrikeAI/issues/357) requests automatic proxy acquisition and switching after a ban, without specifying providers, detection or tool scope. This implementation starts with `http-framework-test`, where both HTTP and decrypted HTTPS responses can be inspected. A standalone forwarding proxy cannot read HTTPS responses inside a CONNECT tunnel. See the [HTTPX proxy](https://www.python-httpx.org/advanced/proxies/) and [transport](https://www.python-httpx.org/advanced/transports/) documentation. + +A 403 can be an application authorization failure; a 429 can apply to an account or other scope. Status matching is a configurable policy, not proof of an IP ban. CAPTCHA or HTTP 200 block pages remain unchanged. Replay and waiting rules are informed by [RFC 9110 §9.2.2](https://www.rfc-editor.org/rfc/rfc9110.html#section-9.2.2) and [RFC 6585 §4](https://www.rfc-editor.org/rfc/rfc6585.html#section-4). + +Pool state lasts for one tool invocation. Other scanners, `exec`, Eino `execute`, external MCP servers and LLM API connections are outside this pool; the settings submenu manages only this HTTP tool. Provider fetching uses a separate client with TLS verification and no target cookies, target Authorization, environment proxies or inherited `allow_insecure`. Use HTTPS for production provider endpoints. + +## Troubleshooting and source + +| Message | Action | +| --- | --- | +| `Proxy pool exhausted` | Check endpoint connectivity, wait for cooldown, or replace the list/provider results | +| `failure policy still matched` | Inspect the preserved response and distinguish IP restrictions from normal authorization/rate limits; adjust `proxy_fail_status` if needed | +| `Retry-After is still active` | Wait for the target's requested delay before another call | +| `Could not fetch proxy source` | Check provider connectivity, authentication and TLS; target `allow_insecure` does not affect provider connections | +| `SOCKS support requires ...` | Install `httpx[socks]` in the tool's Python environment or update its requirements | + +The implementation and MCP parameter definitions are in [tools/http-framework-test.yaml](../../tools/http-framework-test.yaml); regression scenarios are in [scripts/test_http_proxy_pool.py](../../scripts/test_http_proxy_pool.py). + +## Local verification + +```sh +python3 -m pip install 'httpx[http2,socks]>=0.27.0' PyYAML +python3 -m unittest discover -s scripts -p 'test_http_proxy_pool.py' -v +``` + +Tests execute the code read from the tool YAML against local HTTP proxies, CONNECT tunnels and TLS servers. No external targets or real providers are contacted. The HTTPS fixture requires `openssl`. + +## Managed settings and stable egress + +Settings → Proxy settings offers manual arguments, a fixed proxy, or a pool. Fixed mode keeps the configured endpoint across calls and does not fail over. Pool mode offers sticky or round-robin selection with a minimum switch interval (60 seconds in the settings form). The interval also prevents early failover: return the failure response/error instead. Timing is invocation-local; use fixed mode and a provider with fixed egress for stability across calls. A fixed URL alone cannot guarantee the provider's public IP. + +`PUT /api/config` persists the `proxy` object and applies it to future HTTP tool invocations. GET masks `proxy_url`, `proxy_pool`, and `proxy_source`; preserve `********` to keep a saved value or send an empty string to clear it. The settings list accepts one URL per line; provider responses may still use structured entries. See `config.example.yaml`. Managed values override model-supplied proxy arguments through a process-local environment snapshot, never global HTTP_PROXY or command arguments. This is not a network sandbox for arbitrary shell commands. + +Design references: [Zyte sessions](https://docs.zyte.com/zyte-api/usage/features.html#sessions) motivate stable IP/cookie conditions; [HTTPX clients](https://www.python-httpx.org/advanced/clients/) motivate connection reuse; [Nuclei](https://docs.projectdiscovery.io/opensource/nuclei/running) exposes distinct proxy/rate controls; [Nmap](https://nmap.org/book/man-bypass-firewalls-ids.html) documents limits on which scan phases support proxies. Other Kali tools retain native routing, proxy and rate settings. No output-based ban guessing or process restarts are introduced. + +LLM API transports, streams and retry/backoff remain separate from the scan pool. Account/token rate limits are handled by existing provider retry policies. Stable public-IP requirements depend on the provider; this feature never rotates model API egress. diff --git a/docs/zh-CN/README.md b/docs/zh-CN/README.md index 09e51bc72..a15c1af64 100644 --- a/docs/zh-CN/README.md +++ b/docs/zh-CN/README.md @@ -20,6 +20,7 @@ ## 功能指南 - [项目任务续接](project-resume.md) · [Token 用量分析](token-usage-monitoring.md) +- [HTTP 测试工具代理池](http-proxy-pool.md) - [资产管理](asset-management.md) · [知识库](knowledge-base.md) · [机器人接入](robot.md) · [视觉分析](VISION.md) - [WebShell](webshell.md) · [C2](c2.md) · [MCP 联邦](mcp-federation.md) diff --git a/docs/zh-CN/http-proxy-pool.md b/docs/zh-CN/http-proxy-pool.md new file mode 100644 index 000000000..50c5e702a --- /dev/null +++ b/docs/zh-CN/http-proxy-pool.md @@ -0,0 +1,116 @@ +# HTTP 测试工具代理池 + +[English](../en-US/http-proxy-pool.md) | [文档首页](README.md) + +`http-framework-test` 支持用户提供的代理列表及代理供应商接口。连接失败或响应命中指定状态码时，在次数上限内切换代理；成功代理继续复用连接。默认沿用工具参数。也可在「系统设置 → 代理设置」统一管理此 HTTP 工具的出口。 + +## 使用 + +更新工具文件后，在系统设置中应用配置或重启服务以重新加载工具参数。也可以在代理二级菜单保存配置，后续调用立即生效；运行中进程保留启动时快照。 + +调用 `http-framework-test` 时传入以下参数（MCP 工具参数示例）： + +```json +{ + "url": "https://your-authorized-target.example/health", + "method": "GET", + "proxy_pool": "[\"http://127.0.0.1:8081\",\"http://127.0.0.1:8082\"]", + "proxy_max_attempts": 3, + "proxy_cooldown": "60", + "proxy_fail_status": "403,429", + "response_max_bytes": 8192 +} +``` + +`proxy_pool` 也接受每行一个代理的字符串。支持 `http://host:port`、`https://host:port`、`socks5://host:port` 和省略协议的 `host:port`（按 HTTP 解析）；端口必须显式指定。支持 IPv6 方括号及 `user:password@host:port` 认证，凭证中的特殊字符须做 URL 编码。SOCKS 依赖包含在 `requirements.txt` 的 `httpx[http2,socks]` 中。 + +使用供应商接口自动获取代理时，增加或单独使用： + +```json +{ + "url": "https://your-authorized-target.example/health", + "proxy_source": "https://your-proxy-provider.example/api/proxies" +} +``` + +接口返回 UTF-8 文本，每行一个代理，或以下 JSON 格式之一： + +```json +["http://127.0.0.1:8081", "127.0.0.1:8082"] +``` + +```json +{"proxies": ["http://127.0.0.1:8081"]} +``` + +也支持 `{"data": ["http://127.0.0.1:8081"]}`。列表元素还可为 `{"host":"127.0.0.1","port":8081,"scheme":"http","username":"user","password":"pass"}`（`ip` 可代替 `host`），或 `{"url":"http://127.0.0.1:8081"}`。`scheme` 默认 HTTP，认证字段可省略；结构化凭证自动做 URL 编码。需要其他供应商字段格式时，应由供应商或自建适配接口转换为上述格式。项目不内置公共免费代理来源。 + +供应商接口在调用开始时拉取一次；每个请求或重定向跳的可用代理耗尽时，最多再拉取一次。新地址与现有地址去重合并，已有冷却记录保留。连接/读取超时为 10 秒，不跟随重定向，响应上限 1 MiB，代理总数上限 256。首次拉取失败时，有静态列表则发出提示并继续使用静态代理；仅配置供应商时结束调用；切换过程中拉取失败则保留最后的 HTTP 失败响应，若还未收到响应则报错。 + +## 切换与结果 + +| 参数 | 默认值 | 行为 | +| --- | --- | --- | +| `proxy_pool` | 空 | JSON 字符串数组或换行列表 | +| `proxy_strategy` | `"sticky"` | `sticky` 成功后复用，`round_robin` 轮询并跳过冷却代理 | +| `proxy_source` | 空 | 用户配置的 HTTP(S) 供应商接口，可与列表合用 | +| `proxy_max_attempts` | `3` | 每个请求/重定向跳最多尝试 1–10 个不同代理 | +| `proxy_cooldown` | `"60"` | 失败代理冷却 0–86400 秒 | +| `proxy_fail_status` | `"403,429"` | 逗号分隔的 400–599 状态码，传 `"none"` 禁用状态码切换 | + +具体规则： + +- 默认保持成功代理；设置 `proxy_strategy="round_robin"` 后，每个请求/重定向跳轮询可用代理；失败后冷却，在剩余可用代理中切换。每个请求或重定向跳内，同一代理最多尝试一次。 +- 仅 GET、HEAD、OPTIONS 自动重放。POST、PUT、DELETE 等方法发生错误或命中状态码时保留首次结果；用户显式设置的 `repeat` 仍表示额外执行次数。 +- 连接、代理、超时等传输错误可触发上述切换；响应体开始读取后的中断直接报错，不重放部分响应。 +- 429 若包含有效且尚未到期的 `Retry-After`，停止自动重放并提示稍后重试。同一工具调用中，对该目标源的后续请求也会被阻止至等待期结束，等待值最多记为 24 小时。 +- 最后一次尝试若收到 HTTP 失败响应，保留该响应并返回非零退出码；网络故障或所有代理处于冷却时报告错误。不会回退到直连，也不会运行额外的目标 DNS/TCP/TLS 直连探针。 +- stderr 记录尝试编号、代理编号及失败类型；stdout 保留最终响应及最终代理编号。代理池自身诊断不输出代理凭证或供应商 URL。工具调用参数仍沿用平台既有的轨迹、审计与日志规则，带凭证的参数应按敏感配置管理。 + +## 调研结论与覆盖范围 + +[#357](https://github.com/AIPentest/CyberStrikeAI/issues/357) 仅提出自动获取代理与被封后切换，未定义供应商、识别方式或工具范围。当前实现从能读取 HTTP 响应的 `http-framework-test` 接入，使 HTTP 和 HTTPS 的切换行为都可验证。 + +HTTP CONNECT 隧道中的 HTTPS 响应需要由发起请求的客户端解密才能读取，因此单独增加全局转发代理不足以识别全部工具的 HTTPS 封禁响应。HTTPX 的代理与自定义传输接口适合在本工具内完成切换，见 [HTTPX 代理文档](https://www.python-httpx.org/advanced/proxies/)与[传输文档](https://www.python-httpx.org/advanced/transports/)。 + +403 也可能是业务权限错误，429 也可能按账号或其他维度限流；默认状态码只是可配置的失败策略，不宣称已确认 IP 被封。不匹配状态码的验证码页、HTTP 200 拦截页目前保持原样。重放和等待规则参考 [RFC 9110 §9.2.2](https://www.rfc-editor.org/rfc/rfc9110.html#section-9.2.2) 和 [RFC 6585 §4](https://www.rfc-editor.org/rfc/rfc6585.html#section-4)。 + +池状态只在单次工具调用内保留，不跨会话共享。其他扫描工具、`exec`、Eino `execute`、外部 MCP、LLM API 连接暂不接入本池；系统设置中的代理页面只管理上述 HTTP 工具。供应商拉取使用独立客户端，开启 TLS 校验，不继承目标请求的 Cookie、Authorization、环境代理或 `allow_insecure`；生产供应商接口建议使用 HTTPS。 + +## 排错与源码 + +| 提示 | 处理 | +| --- | --- | +| `Proxy pool exhausted` | 检查代理可达性，等待冷却或更换列表/供应商返回值 | +| `failure policy still matched` | 查看保留的响应；确认是 IP 限制还是正常的权限/限流结果，按需调整 `proxy_fail_status` | +| `Retry-After is still active` | 等待目标指定的时间后再发起调用 | +| `Could not fetch proxy source` | 检查供应商接口网络、认证和 TLS；目标的 `allow_insecure` 不影响供应商连接 | +| `SOCKS support requires ...` | 在工具使用的 Python 环境安装 `httpx[socks]` 或更新 `requirements.txt` 依赖 | + +执行代码与 MCP 参数定义位于 [tools/http-framework-test.yaml](../../tools/http-framework-test.yaml)，回归场景位于 [scripts/test_http_proxy_pool.py](../../scripts/test_http_proxy_pool.py)。 + +## 本地验证 + +```sh +python3 -m pip install 'httpx[http2,socks]>=0.27.0' PyYAML +python3 -m unittest discover -s scripts -p 'test_http_proxy_pool.py' -v +``` + +测试从 YAML 读取实际执行代码，使用本地 HTTP 代理、CONNECT 隧道及 TLS 服务验证切换、路径保真、重放限制、等待期、供应商刷新与凭证隔离；不访问外部目标或真实供应商。HTTPS 测试需要 `openssl`。 + +## 系统设置与稳定出口 + +系统设置 → 代理设置提供三种模式：沿用工具参数、固定代理、代理池。固定模式在不同调用之间持续使用配置的同一代理地址，不做失败切换；需要稳定 IP 的登录态优先选它。池模式可选择复用或轮询，界面默认最短切换间隔 60 秒。间隔同时限制故障切换，尚未到期时保留响应或报错，不提前更换出口。计时只在单次调用内有效；跨调用仍需稳定出口时使用固定模式及供应商的固定 IP 服务。 + +保存通过既有 `PUT /api/config` 的 `proxy` 段持久化并热应用，只影响以后启动的 `http-framework-test`。`proxy_url`、`proxy_pool`、`proxy_source` 在 GET 响应中整体脱敏；`********` 原样提交会保留已有值，空字符串表示清除。设置页列表每行一个 URL；结构化供应商格式仍可由供应商接口提供。配置示例见 `config.example.yaml` 的 `proxy` 段。受管设置通过该工具专用的进程环境快照传递，不加入命令参数或自动注入模型上下文；它不限制任意 Shell 命令的网络行为。 + +### 参考项目与采用的做法 + +| 参考 | 做法与本项目选择 | +| --- | --- | +| [Zyte Sessions](https://docs.zyte.com/zyte-api/usage/features.html#sessions) | IP、Cookie 和连接环境共同影响会话。提供固定出口模式，登录态不默认轮询。 | +| [HTTPX Clients](https://www.python-httpx.org/advanced/clients/) | 复用连接和 Cookie。运行中的调用保持快照，设置变更不拆除已有连接。 | +| [Nuclei](https://docs.projectdiscovery.io/opensource/nuclei/running) | 原生代理选项和请求速率分别配置。其他 Kali 工具使用自身代理与限速能力，不从文本输出猜测封禁并重新启动进程。 | +| [Nmap 代理限制](https://nmap.org/book/man-bypass-firewalls-ids.html) | 代理支持有协议和扫描阶段限制，不能承诺统一 HTTP 环境变量覆盖所有原始报文扫描。未适配工具保持原路由。 | + +大模型 API 的流式响应、连接池和提供商重试继续使用既有客户端，不接入扫描代理池；模型侧的账号/Token 限流应走现有退避策略。不同供应商是否要求固定公网 IP 需按其接口约定判断，本功能不会自动改动模型出口或轮询 API 连接。 diff --git a/internal/app/app.go b/internal/app/app.go index b7b2d8ff5..545a6788b 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -162,6 +162,7 @@ func New(cfg *config.Config, log *logger.Logger, configPath string) (*App, error // 创建安全工具执行器 executor := security.NewExecutor(&cfg.Security, mcpServer, log.Logger) + executor.SetProxyConfig(cfg.Proxy) executor.SetShellNoOutputTimeoutSeconds(cfg.Agent.ShellNoOutputTimeoutSeconds) executor.SetToolOutputMaxBytes(cfg.MultiAgent.EinoMiddleware.ReductionMaxLengthForTruncEffective()) executor.SetToolOutputSpillRoot(cfg.MultiAgent.EinoMiddleware.ReductionRootDir) diff --git a/internal/config/config.go b/internal/config/config.go index f4eaa2a98..ed52d2670 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -21,6 +21,7 @@ import ( ) type Config struct { + Proxy ProxyConfig `yaml:"proxy,omitempty" json:"proxy"` Version string `yaml:"version,omitempty" json:"version,omitempty"` // 前端显示的版本号，如 v1.3.3 Server ServerConfig `yaml:"server"` Log LogConfig `yaml:"log"` @@ -1653,6 +1654,9 @@ func Load(path string) (*Config, error) { if err := yaml.Unmarshal(data, &cfg); err != nil { return nil, fmt.Errorf("解析配置文件失败: %w", err) } + if err := cfg.Proxy.Validate(); err != nil { + return nil, err + } if err := cfg.Agent.validateTaskTimeout(); err != nil { return nil, err } diff --git a/internal/config/proxy.go b/internal/config/proxy.go new file mode 100644 index 000000000..bb133936b --- /dev/null +++ b/internal/config/proxy.go @@ -0,0 +1,73 @@ +package config + +import ( + "fmt" + "net/url" + "strconv" + "strings" +) + +// ProxyConfig governs only the built-in HTTP test tool, never process-wide routing. +type ProxyConfig struct { + Mode string `yaml:"mode" json:"mode"` + URL string `yaml:"proxy_url" json:"proxy_url"` + Pool string `yaml:"proxy_pool" json:"proxy_pool"` + Source string `yaml:"proxy_source" json:"proxy_source"` + Strategy string `yaml:"strategy" json:"strategy"` + MaxAttempts int `yaml:"max_attempts" json:"max_attempts"` + Cooldown int `yaml:"cooldown" json:"cooldown"` + MinSwitchInterval int `yaml:"min_switch_interval" json:"min_switch_interval"` +} + +func (p ProxyConfig) Validate() error { + if p.Mode != "" && p.Mode != "manual" && p.Mode != "fixed" && p.Mode != "pool" { + return fmt.Errorf("proxy.mode must be manual, fixed or pool") + } + if p.Strategy != "" && p.Strategy != "sticky" && p.Strategy != "round_robin" { + return fmt.Errorf("proxy.strategy must be sticky or round_robin") + } + if p.MaxAttempts < 0 || p.MaxAttempts > 10 || p.Cooldown < 0 || p.Cooldown > 86400 || p.MinSwitchInterval < 0 || p.MinSwitchInterval > 86400 { + return fmt.Errorf("invalid proxy attempt or timing limits") + } + if p.Mode == "fixed" { + return validateProxyEndpoint(p.URL) + } + if p.Mode != "pool" { + return nil + } + count := 0 + for _, line := range strings.Split(p.Pool, "\n") { + if strings.TrimSpace(line) != "" { + count++ + if err := validateProxyEndpoint(line); err != nil { + return err + } + } + } + if count > 256 { + return fmt.Errorf("proxy pool exceeds 256 endpoints") + } + if p.Source != "" { + u, err := url.Parse(p.Source) + if err != nil || u.Hostname() == "" || (u.Scheme != "http" && u.Scheme != "https") || u.Fragment != "" { + return fmt.Errorf("proxy source must be an HTTP(S) URL") + } + } + if count == 0 && p.Source == "" { + return fmt.Errorf("proxy pool requires endpoints or a provider URL") + } + return nil +} +func validateProxyEndpoint(value string) error { + value = strings.TrimSpace(value) + if !strings.Contains(value, "://") { + value = "http://" + value + } + u, err := url.Parse(value) + if err == nil && u.Hostname() != "" && (u.Scheme == "http" || u.Scheme == "https" || u.Scheme == "socks5") && (u.Path == "" || u.Path == "/") && u.RawQuery == "" && u.Fragment == "" && !strings.ContainsAny(value, "\r\n\t ") { + if port, err := strconv.Atoi(u.Port()); err == nil && port > 0 && port <= 65535 { + return nil + } + } + return fmt.Errorf("proxy endpoint must be http(s)://host:port or socks5://host:port") +} diff --git a/internal/config/proxy_test.go b/internal/config/proxy_test.go new file mode 100644 index 000000000..9ec19c210 --- /dev/null +++ b/internal/config/proxy_test.go @@ -0,0 +1,16 @@ +package config + +import "testing" + +func TestProxyValidation(t *testing.T) { + for _, p := range []ProxyConfig{{}, {Mode: "manual"}, {Mode: "fixed", URL: "http://u:p@localhost:8080"}, {Mode: "pool", Pool: "[::1]:8080\nlocalhost:8081", MinSwitchInterval: 60}} { + if err := p.Validate(); err != nil { + t.Fatal(err) + } + } + for _, p := range []ProxyConfig{{Mode: "bad"}, {Mode: "fixed"}, {Mode: "fixed", URL: "file:///secret"}, {Mode: "pool"}, {Mode: "pool", Pool: "http://host:80/path"}, {Mode: "pool", Source: "file:///secret"}, {MinSwitchInterval: -1}, {MaxAttempts: 11}} { + if p.Validate() == nil { + t.Fatalf("accepted invalid config: mode=%s", p.Mode) + } + } +} diff --git a/internal/handler/config.go b/internal/handler/config.go index f963f47a4..f808a12d6 100644 --- a/internal/handler/config.go +++ b/internal/handler/config.go @@ -264,6 +264,7 @@ func (h *ConfigHandler) ApplyWechatRobotBinding(wc config.RobotWechatConfig) err // GetConfigResponse 获取配置响应 type GetConfigResponse struct { + Proxy config.ProxyConfig `json:"proxy"` AI config.AIConfig `json:"ai"` OpenAI config.OpenAIConfig `json:"openai"` Vision config.VisionConfig `json:"vision"` @@ -374,6 +375,7 @@ func (h *ConfigHandler) GetConfig(c *gin.Context) { } response, err := maskedConfigResponse(GetConfigResponse{ + Proxy: h.config.Proxy, AI: h.config.AI, OpenAI: h.config.OpenAI, Vision: h.config.Vision, @@ -723,6 +725,7 @@ func (h *ConfigHandler) GetTools(c *gin.Context) { // UpdateConfigRequest 更新配置请求 type UpdateConfigRequest struct { + Proxy *config.ProxyConfig `json:"proxy,omitempty"` AI *config.AIConfig `json:"ai,omitempty"` OpenAI *config.OpenAIConfig `json:"openai,omitempty"` Vision *config.VisionConfig `json:"vision,omitempty"` @@ -807,6 +810,21 @@ func (h *ConfigHandler) UpdateConfig(c *gin.Context) { return } + previousProxy := h.config.Proxy + proxySaved := false + defer func() { + if !proxySaved { + h.config.Proxy = previousProxy + } + }() + if req.Proxy != nil { + if err := req.Proxy.Validate(); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + h.config.Proxy = *req.Proxy + } + // 更新OpenAI配置 if req.AI != nil { h.config.AI = *req.AI @@ -1254,6 +1272,10 @@ func (h *ConfigHandler) UpdateConfig(c *gin.Context) { return } + proxySaved = true + if req.Proxy != nil && h.executor != nil { + h.executor.SetProxyConfig(h.config.Proxy) + } if h.audit != nil { h.audit.RecordOK(c, "config", "update", "更新内存配置", "config", "", nil) } @@ -1763,6 +1785,7 @@ func (h *ConfigHandler) ApplyConfig(c *gin.Context) { h.mcpServer.ClearTools() // 重新注册安全工具 + h.executor.SetProxyConfig(h.config.Proxy) h.executor.SetToolOutputMaxBytes(h.config.MultiAgent.EinoMiddleware.ReductionMaxLengthForTruncEffective()) h.executor.SetToolOutputSpillRoot(h.config.MultiAgent.EinoMiddleware.ReductionRootDir) h.executor.RegisterTools(h.mcpServer) @@ -1955,6 +1978,7 @@ func (h *ConfigHandler) saveConfig() error { return fmt.Errorf("解析配置文件失败: %w", err) } + updateProxyConfig(root, h.config.Proxy) updateAgentConfig(root, h.config.Agent) updateMCPConfig(root, h.config.MCP) updateAIConfig(root, h.config.AI) @@ -2940,3 +2964,15 @@ func convertParamType(t string) string { return "string" } } + +func updateProxyConfig(doc *yaml.Node, cfg config.ProxyConfig) { + node := ensureMap(doc.Content[0], "proxy") + setStringInMap(node, "mode", cfg.Mode) + setStringInMap(node, "proxy_url", cfg.URL) + setStringInMap(node, "proxy_pool", cfg.Pool) + setStringInMap(node, "proxy_source", cfg.Source) + setStringInMap(node, "strategy", cfg.Strategy) + setIntInMap(node, "max_attempts", cfg.MaxAttempts) + setIntInMap(node, "cooldown", cfg.Cooldown) + setIntInMap(node, "min_switch_interval", cfg.MinSwitchInterval) +} diff --git a/internal/handler/config_secrets.go b/internal/handler/config_secrets.go index d095d9fb0..2a0a02de2 100644 --- a/internal/handler/config_secrets.go +++ b/internal/handler/config_secrets.go @@ -13,14 +13,14 @@ const maskedSecret = "********" func secretConfigKey(key string) bool { switch strings.ToLower(key) { - case "api_key", "apikey", "password", "secret", "client_secret", "app_secret", "token", "bot_token", "app_token", "verify_token", "encoding_aes_key", "authorization", "x-api-key": + case "proxy_url", "proxy_pool", "proxy_source", "api_key", "apikey", "password", "secret", "client_secret", "app_secret", "token", "bot_token", "app_token", "verify_token", "encoding_aes_key", "authorization", "x-api-key": return true } return false } func configSecretSnapshot(cfg *config.Config) GetConfigResponse { - return GetConfigResponse{AI: cfg.AI, OpenAI: cfg.OpenAI, Vision: cfg.Vision, FOFA: cfg.FOFA, + return GetConfigResponse{Proxy: cfg.Proxy, AI: cfg.AI, OpenAI: cfg.OpenAI, Vision: cfg.Vision, FOFA: cfg.FOFA, ZoomEye: cfg.ZoomEye, Quake: cfg.Quake, Shodan: cfg.Shodan, MCP: cfg.MCP, Hitl: cfg.Hitl, Knowledge: cfg.Knowledge, Robots: cfg.Robots} } diff --git a/internal/handler/openapi.go b/internal/handler/openapi.go index 26b22115e..bb55ca751 100644 --- a/internal/handler/openapi.go +++ b/internal/handler/openapi.go @@ -846,6 +846,7 @@ func (h *OpenAPIHandler) GetOpenAPISpec(c *gin.Context) { "type": "object", "description": "配置信息（含 openai、vision、multi_agent 等）", "properties": map[string]interface{}{ + "proxy": map[string]interface{}{"type": "object", "description": "HTTP-tool-only egress policy: mode manual/fixed/pool, masked proxy_url/proxy_pool/proxy_source, strategy sticky/round_robin, max_attempts 1-10 (0=3), cooldown and min_switch_interval 0-86400 seconds. PUT persists and applies to subsequent HTTP invocations; model/other tool egress is unchanged."}, "agent": map[string]interface{}{ "$ref": "#/components/schemas/AgentConfig", }, @@ -858,6 +859,7 @@ func (h *OpenAPIHandler) GetOpenAPISpec(c *gin.Context) { "type": "object", "description": "更新配置请求", "properties": map[string]interface{}{ + "proxy": map[string]interface{}{"type": "object", "description": "HTTP-tool-only egress policy: mode manual/fixed/pool, masked proxy_url/proxy_pool/proxy_source, strategy sticky/round_robin, max_attempts 1-10 (0=3), cooldown and min_switch_interval 0-86400 seconds. PUT persists and applies to subsequent HTTP invocations; model/other tool egress is unchanged."}, "agent": map[string]interface{}{ "$ref": "#/components/schemas/AgentConfig", }, diff --git a/internal/handler/proxy_settings_test.go b/internal/handler/proxy_settings_test.go new file mode 100644 index 000000000..37d133bae --- /dev/null +++ b/internal/handler/proxy_settings_test.go @@ -0,0 +1,92 @@ +package handler + +import ( + "bytes" + "context" + "cyberstrike-ai/internal/config" + "cyberstrike-ai/internal/security" + "encoding/json" + "fmt" + "github.com/gin-gonic/gin" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestProxySettingsPersistMaskAndRollback(t *testing.T) { + path := filepath.Join(t.TempDir(), "settings.yaml") + if err := os.WriteFile(path, []byte("version: test\n"), 0600); err != nil { + t.Fatal(err) + } + cfg := &config.Config{} + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + executor := security.NewExecutor(&config.SecurityConfig{Tools: []config.ToolConfig{{Name: "http-framework-test", Enabled: true, Command: executable, Args: []string{"-test.run=^TestProxySettingsChild$"}}}}, nil, zap.NewNop()) + h := &ConfigHandler{config: cfg, configPath: path, logger: zap.NewNop(), executor: executor} + r := gin.New() + r.PUT("/config", h.UpdateConfig) + r.GET("/config", h.GetConfig) + put := func(p config.ProxyConfig) *httptest.ResponseRecorder { + data, _ := json.Marshal(map[string]interface{}{"proxy": p}) + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPut, "/config", bytes.NewReader(data)) + req.Header.Set("Content-Type", "application/json") + r.ServeHTTP(w, req) + return w + } + policy := config.ProxyConfig{Mode: "fixed", URL: "http://user:test-secret@localhost:8080", MinSwitchInterval: 60} + if w := put(policy); w.Code != 200 { + t.Fatal(w.Code, w.Body.String()) + } + result, err := executor.ExecuteTool(context.Background(), "http-framework-test", map[string]interface{}{}) + if err != nil || result.IsError || !strings.Contains(result.Content[0].Text, "test-secret") { + t.Fatal("saved proxy not applied to executor", err) + } + raw, _ := os.ReadFile(path) + var disk config.Config + if err := yaml.Unmarshal(raw, &disk); err != nil || disk.Proxy.URL != policy.URL { + t.Fatal("proxy not persisted", err) + } + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest("GET", "/config", nil)) + if strings.Contains(w.Body.String(), "test-secret") { + t.Fatal("secret exposed") + } + var response struct { + Proxy config.ProxyConfig `json:"proxy"` + } + json.Unmarshal(w.Body.Bytes(), &response) + if response.Proxy.URL != maskedSecret { + t.Fatal("not masked") + } + if w := put(response.Proxy); w.Code != 200 || cfg.Proxy.URL != policy.URL { + t.Fatal("masked save lost secret") + } + invalid := policy + invalid.Mode = "pool" + invalid.Pool = "bad" + invalid.URL = "" + if w := put(invalid); w.Code != 400 || cfg.Proxy != policy { + t.Fatal("invalid save mutated state") + } + h.configPath = filepath.Join(t.TempDir(), "missing", "settings.yaml") + changed := policy + changed.URL = "http://localhost:9090" + if w := put(changed); w.Code != 500 || cfg.Proxy != policy { + t.Fatal("failed save changed proxy config") + } +} + +func TestProxySettingsChild(t *testing.T) { + if raw := os.Getenv("CSAI_HTTP_PROXY_CONFIG"); raw != "" { + fmt.Print(raw) + os.Exit(0) + } +} diff --git a/internal/security/executor.go b/internal/security/executor.go index e956c9f67..8058fe3a5 100644 --- a/internal/security/executor.go +++ b/internal/security/executor.go @@ -32,6 +32,8 @@ var ToolOutputCallbackCtxKey = toolOutputCallbackCtxKey{} // Executor 安全工具执行器 type Executor struct { + proxyMu sync.RWMutex + proxyConfig config.ProxyConfig config *config.SecurityConfig toolIndex map[string]*config.ToolConfig // 工具索引，用于 O(1) 查找 mcpServer *mcp.Server @@ -183,9 +185,11 @@ func (e *Executor) ExecuteTool(ctx context.Context, toolName string, args map[st }, nil } + proxySnapshot := e.proxySnapshot(toolName) // 执行命令 cmd := exec.CommandContext(ctx, toolConfig.Command, cmdArgs...) applyDefaultTerminalEnv(cmd) + applyHTTPProxySnapshot(cmd, proxySnapshot) attachNonInteractiveStdin(cmd) _ = prepareShellCmdSession(cmd) @@ -207,6 +211,7 @@ func (e *Executor) ExecuteTool(ctx context.Context, toolName string, args map[st ) cmd2 := exec.CommandContext(ctx, toolConfig.Command, cmdArgs...) applyDefaultTerminalEnv(cmd2) + applyHTTPProxySnapshot(cmd2, proxySnapshot) _ = prepareShellCmdSession(cmd2) output, err = runCommandWithPTY(ctx, cmd2, cb, e.toolOutputMaxBytes, spill) } @@ -219,6 +224,7 @@ func (e *Executor) ExecuteTool(ctx context.Context, toolName string, args map[st ) cmd2 := exec.CommandContext(ctx, toolConfig.Command, cmdArgs...) applyDefaultTerminalEnv(cmd2) + applyHTTPProxySnapshot(cmd2, proxySnapshot) _ = prepareShellCmdSession(cmd2) output, err = runCommandWithPTY(ctx, cmd2, nil, e.toolOutputMaxBytes, spill) } diff --git a/internal/security/http_proxy_pool_test.go b/internal/security/http_proxy_pool_test.go new file mode 100644 index 000000000..224c99a7e --- /dev/null +++ b/internal/security/http_proxy_pool_test.go @@ -0,0 +1,42 @@ +package security + +import ( + "path/filepath" + "testing" + + "cyberstrike-ai/internal/config" + "go.uber.org/zap" +) + +// Check the public tool's YAML through the same argument mapping used by MCP. +// In particular, an empty optional string is dropped by that mapping, so the +// status-policy off switch must use a nonempty sentinel. +func TestHTTPProxyPoolToolArguments(t *testing.T) { + tool, err := config.LoadToolFromFile(filepath.Join("..", "..", "tools", "http-framework-test.yaml")) + if err != nil { + t.Fatal(err) + } + executor := NewExecutor(&config.SecurityConfig{Tools: []config.ToolConfig{*tool}}, nil, zap.NewNop()) + pool := "http://user:p%40ss@localhost:8081\nhttp://localhost:8082" + args := executor.buildCommandArgs(tool.Name, tool, map[string]interface{}{ + "url": "https://target.invalid/a/../b", "proxy_pool": pool, + "proxy_source": "https://provider.invalid/?key=placeholder", + "proxy_strategy": "round_robin", "proxy_max_attempts": float64(2), "proxy_cooldown": "0", "proxy_fail_status": "none", + }) + want := map[string]string{ + "--proxy-pool": pool, "--proxy-source": "https://provider.invalid/?key=placeholder", + "--proxy-strategy": "round_robin", "--proxy-max-attempts": "2", "--proxy-cooldown": "0", "--proxy-fail-status": "none", + } + for flag, value := range want { + found := false + for i, arg := range args { + if arg == flag && i+1 < len(args) && args[i+1] == value { + found = true + break + } + } + if !found { + t.Errorf("tool argument mapping lost or changed %s", flag) + } + } +} diff --git a/internal/security/proxy.go b/internal/security/proxy.go new file mode 100644 index 000000000..8cc6154e8 --- /dev/null +++ b/internal/security/proxy.go @@ -0,0 +1,47 @@ +package security + +import ( + "encoding/json" + "os/exec" + "strings" + + "cyberstrike-ai/internal/config" +) + +func (e *Executor) SetProxyConfig(cfg config.ProxyConfig) { + e.proxyMu.Lock() + defer e.proxyMu.Unlock() + e.proxyConfig = cfg +} +func (e *Executor) proxySnapshot(tool string) string { + if tool != "http-framework-test" { + return "" + } + e.proxyMu.RLock() + defer e.proxyMu.RUnlock() + if e.proxyConfig.Mode == "" || e.proxyConfig.Mode == "manual" { + return "" + } + policy := e.proxyConfig + if policy.Mode == "fixed" { + policy.Pool, policy.Source = "", "" + } else { + policy.URL = "" + } + raw, _ := json.Marshal(policy) + return string(raw) +} +func applyHTTPProxySnapshot(cmd *exec.Cmd, snapshot string) { + // Never mutate os.Environ or HTTP_PROXY: models and other tools are isolated. + env := cmd.Environ() + filtered := make([]string, 0, len(env)+1) + for _, entry := range env { + if !strings.HasPrefix(entry, "CSAI_HTTP_PROXY_CONFIG=") { + filtered = append(filtered, entry) + } + } + if snapshot != "" { + filtered = append(filtered, "CSAI_HTTP_PROXY_CONFIG="+snapshot) + } + cmd.Env = filtered +} diff --git a/internal/security/proxy_test.go b/internal/security/proxy_test.go new file mode 100644 index 000000000..8203e6454 --- /dev/null +++ b/internal/security/proxy_test.go @@ -0,0 +1,34 @@ +package security + +import ( + "cyberstrike-ai/internal/config" + "go.uber.org/zap" + "os" + "os/exec" + "strings" + "testing" +) + +func TestProxySnapshotIsolation(t *testing.T) { + t.Setenv("HTTP_PROXY", "http://existing:8080") + e := NewExecutor(&config.SecurityConfig{}, nil, zap.NewNop()) + e.SetProxyConfig(config.ProxyConfig{Mode: "fixed", URL: "http://private:secret@localhost:8081"}) + snapshot := e.proxySnapshot("http-framework-test") + if snapshot == "" || e.proxySnapshot("nmap") != "" || e.proxySnapshot("exec") != "" { + t.Fatal("wrong routing scope") + } + e.SetProxyConfig(config.ProxyConfig{Mode: "fixed", URL: "http://new:8082"}) + cmd := exec.Command("unused") + applyHTTPProxySnapshot(cmd, snapshot) + if !strings.Contains(strings.Join(cmd.Env, "\n"), "private:secret") || strings.Contains(strings.Join(cmd.Args, " "), "secret") { + t.Fatal("snapshot lost or leaked to argv") + } + if os.Getenv("HTTP_PROXY") != "http://existing:8080" { + t.Fatal("global proxy changed") + } + other := exec.Command("unused") + applyHTTPProxySnapshot(other, e.proxySnapshot("nmap")) + if strings.Contains(strings.Join(other.Env, "\n"), "CSAI_HTTP_PROXY_CONFIG=") { + t.Fatal("policy leaked to other tool") + } +} diff --git a/internal/security/shell_noninteractive_test.go b/internal/security/shell_noninteractive_test.go index e420bca6a..cd6ca5ad3 100644 --- a/internal/security/shell_noninteractive_test.go +++ b/internal/security/shell_noninteractive_test.go @@ -107,13 +107,17 @@ func TestNonInteractiveStdinReadBlocksWithoutRedirect(t *testing.T) { t.Fatal(err) } defer r.Close() + defer w.Close() // 保持 w 打开且不写数据，模拟「等待用户输入」 cmd := exec.Command("sh", "-c", `read x; echo done`) cmd.Stdin = r done := make(chan error, 1) - go func() { done <- cmd.Run() }() + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + go func() { done <- cmd.Wait() }() select { case err := <-done: diff --git a/requirements.txt b/requirements.txt index 2997f703e..ba67df551 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # Python HTTP helpers leveraged by tools like api-fuzzer, dnslog, http-intruder, http-framework-test requests>=2.32.3 -httpx[http2]>=0.27.0 +httpx[http2,socks]>=0.27.0 charset-normalizer>=3.3.2 chardet>=5.2.0,<6 diff --git a/scripts/test_http_proxy_pool.py b/scripts/test_http_proxy_pool.py new file mode 100644 index 000000000..98c718536 --- /dev/null +++ b/scripts/test_http_proxy_pool.py @@ -0,0 +1,552 @@ +"""Local-only integration tests for the exact Python shipped in the tool YAML. + +Run: python3 -m unittest discover -s scripts -p 'test_http_proxy_pool.py' -v +Requires: httpx[http2,socks], PyYAML; openssl for the CONNECT/TLS fixture. +""" + +import contextlib +import datetime +import email.utils +import http.server +import importlib.util +import io +import json +import os +from pathlib import Path +import select +import shutil +import socket +import socketserver +import ssl +import subprocess +import sys +import tempfile +import threading +import unittest +from unittest import mock + +import httpx +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +TOOL = yaml.safe_load((ROOT / "tools/http-framework-test.yaml").read_text()) +SCRIPT = TOOL["args"][1] +NS = {"__name__": "http_framework_tool"} +exec(compile(SCRIPT, "http-framework-test.yaml", "exec"), NS) +Pool = NS["ProxyPoolTransport"] + + +def relay(connection, upstream): + while True: + readable, _, _ = select.select([connection, upstream], [], [], 3) + if not readable: + return + for current in readable: + data = current.recv(65536) + if not data: + return + (upstream if current is connection else connection).sendall(data) + + +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def respond(self): + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) if length else b"" + self.server.requests.append((self.command, self.path, self.headers, body)) + status, headers, body = self.server.reply(self) + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + for key, value in headers.items(): + self.send_header(key, value) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + + do_GET = do_POST = do_HEAD = do_OPTIONS = do_PUT = do_DELETE = respond + + def do_CONNECT(self): + self.server.requests.append(("CONNECT", self.path, self.headers, b"")) + # Connect only to the local TLS fixture, regardless of the requested host. + with socket.create_connection(("127.0.0.1", self.server.tunnel_port), timeout=3) as upstream: + self.send_response(200) + self.end_headers() + self.wfile.flush() + self.close_connection = True + relay(self.connection, upstream) + + +class SocksHandler(socketserver.StreamRequestHandler): + def handle(self): + version, count = self.rfile.read(2) + self.rfile.read(count) + if version != 5: + return + self.wfile.write(b"\x05\x00") + self.wfile.flush() + version, command, _, address_type = self.rfile.read(4) + if version != 5 or command != 1 or address_type != 3: + return + length = self.rfile.read(1)[0] + host = self.rfile.read(length).decode("ascii") + port = int.from_bytes(self.rfile.read(2), "big") + self.server.requests.append((host, port)) + with socket.create_connection(("127.0.0.1", self.server.tunnel_port), timeout=3) as upstream: + self.wfile.write(b"\x05\x00\x00\x01\x7f\x00\x00\x01\x00\x00") + self.wfile.flush() + relay(self.connection, upstream) + + +@contextlib.contextmanager +def socks_server(tunnel_port): + instance = socketserver.ThreadingTCPServer(("127.0.0.1", 0), SocksHandler) + instance.daemon_threads = True + instance.tunnel_port = tunnel_port + instance.requests = [] + instance.url = f"socks5://127.0.0.1:{instance.server_address[1]}" + worker = threading.Thread(target=instance.serve_forever, kwargs={"poll_interval": 0.02}, daemon=True) + worker.start() + try: + yield instance + finally: + instance.shutdown() + instance.server_close() + worker.join(timeout=3) + + +@contextlib.contextmanager +def server(status=200, body=b"ok", headers=None, reply=None, tls=None, tunnel_port=None): + instance = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + instance.daemon_threads = True + instance.requests = [] + instance.reply = reply or (lambda _: (status, headers or {}, body)) + instance.tunnel_port = tunnel_port + if tls: + instance.socket = tls.wrap_socket(instance.socket, server_side=True) + instance.url = f"http://127.0.0.1:{instance.server_port}" + worker = threading.Thread(target=instance.serve_forever, kwargs={"poll_interval": 0.02}, daemon=True) + worker.start() + try: + yield instance + finally: + instance.shutdown() + instance.server_close() + worker.join(timeout=3) + + +def send(client, method="GET", url="http://target.invalid/a/../b?x=%2F", **kwargs): + return NS["send_path_as_is"](client, method=method, url=url, headers=httpx.Headers(), **kwargs) + + +class ProxyPoolTests(unittest.TestCase): + def setUp(self): + self.diagnostics = io.StringIO() + self.redirect_stderr = contextlib.redirect_stderr(self.diagnostics) + self.redirect_stderr.__enter__() + + def tearDown(self): + self.redirect_stderr.__exit__(None, None, None) + + def test_parse_formats_validate_and_deduplicate(self): + parse = NS["parse_proxy_list"] + self.assertEqual(parse("LOCALHOST:8080\nhttp://localhost:8080/"), ["http://localhost:8080"]) + self.assertEqual(parse("[::1]:8080\n127.0.0.1:8081"), + ["http://[::1]:8080", "http://127.0.0.1:8081"]) + self.assertEqual(parse('["http://[::1]:8080", "http://127.0.0.1:8081"]'), + ["http://[::1]:8080", "http://127.0.0.1:8081"]) + self.assertEqual(parse('{"data":["socks5://[::1]:1080"]}'), ["socks5://[::1]:1080"]) + self.assertEqual(parse('{"proxies":["https://user:p%40ss@EXAMPLE.COM:443"]}'), + ["https://user:p%40ss@example.com:443"]) + for value in ["http://user:secret@host:bad", "http://host:80/path", "file://host:80", + "http://host:0", "http://host:65536", "http://host:80?q=secret", "[invalid", + {"data": [{"ip": "127.0.0.1"}]}, ["host:80"] * 257]: + with self.subTest(value=type(value)): + with self.assertRaises(ValueError) as caught: + parse(value) + self.assertNotIn("secret", str(caught.exception)) + + def test_deeply_nested_provider_json_is_a_validation_error(self): + body = ("[" * 20000 + "]" * 20000).encode() + with server(body=body) as provider: + with self.assertRaisesRegex(ValueError, "Invalid proxy list JSON"): + Pool("", source=provider.url + "/?key=secret") + + def test_provider_text_starting_with_bare_ipv6(self): + with server(body=b"[::1]:8080\n127.0.0.1:8081") as provider: + pool = Pool("", source=provider.url) + self.assertEqual(pool.proxies, ["http://[::1]:8080", "http://127.0.0.1:8081"]) + + def test_refresh_failure_preserves_response_and_closes_transport(self): + with server(403, b"blocked evidence") as proxy: + def reply(handler): + if len(handler.server.requests) == 1: + return 200, {}, proxy.url.encode() + return 503, {}, b"provider unavailable" + with server(reply=reply) as provider: + pool = Pool("", source=provider.url + "/?key=secret") + with httpx.Client(transport=pool, trust_env=False) as client: + response, *_ = send(client) + with contextlib.closing(response): + self.assertEqual(response.read(), b"blocked evidence") + self.assertTrue(response.extensions["proxy_pool"]["failed"]) + transport = pool.transports[pool.proxies[0]] + with mock.patch.object(transport, "close", wraps=transport.close) as close: + client.close() + close.assert_called_once() + self.assertEqual(len(provider.requests), 2) + self.assertNotIn("secret", self.diagnostics.getvalue()) + + def test_configuration_bounds(self): + for kwargs in [dict(max_attempts=0), dict(max_attempts=11), dict(cooldown=-1), + dict(cooldown=float("nan")), dict(cooldown=float("inf")), + dict(fail_status="200"), dict(fail_status="oops")]: + with self.subTest(kwargs=kwargs), self.assertRaises(ValueError): + Pool("localhost:8080", **kwargs) + + def test_structured_provider_entries(self): + raw = {"data": [{"ip": "::1", "port": 1080, "scheme": "socks5", + "username": "u@", "password": "p:/"}, + {"url": "http://localhost:8080"}]} + self.assertEqual(NS["parse_proxy_list"](raw), + ["socks5://u%40:p%3A%2F@[::1]:1080", "http://localhost:8080"]) + for row in [{"host": "evil@host", "port": 80}, {"host": "host", "port": True}, + {"host": "host", "port": 80, "password": "secret"}]: + with self.assertRaises(ValueError) as caught: + NS["parse_proxy_list"]([row]) + self.assertNotIn("secret", str(caught.exception)) + + def test_round_robin_rotates_without_skipping_failover_candidate(self): + with server(403) as bad, server() as second, server() as third: + pool = Pool(json.dumps([bad.url, second.url, third.url]), strategy="round_robin") + with httpx.Client(transport=pool, trust_env=False) as client: + for expected in [2, 3, 2]: + response, *_ = send(client) + with contextlib.closing(response): + self.assertEqual(response.read(), b"ok") + self.assertEqual(response.extensions["proxy_pool"]["proxy"], expected) + self.assertEqual(len(bad.requests), 1) + + def test_provider_outage_keeps_static_pool_and_source_only_fails(self): + with server() as proxy, server(503) as provider: + pool = Pool(proxy.url, source=provider.url + "/?key=secret") + with httpx.Client(transport=pool, trust_env=False) as client: + response, *_ = send(client) + with contextlib.closing(response): + self.assertEqual(response.read(), b"ok") + with self.assertRaisesRegex(ValueError, "HTTP 503"): + Pool("", source=provider.url) + self.assertNotIn("secret", self.diagnostics.getvalue()) + + def test_failover_and_sticky_connection_preserve_raw_path(self): + with server(403, b"blocked") as blocked, server() as good: + pool = Pool(json.dumps([blocked.url, good.url]), cooldown=60) + with httpx.Client(transport=pool, trust_env=False) as client: + for _ in range(2): + response, *_ = send(client) + with contextlib.closing(response): + self.assertEqual(response.read(), b"ok") + self.assertFalse(response.extensions["proxy_pool"]["failed"]) + self.assertEqual(len(blocked.requests), 1) + self.assertEqual(len(good.requests), 2) + self.assertEqual(good.requests[0][1], "http://target.invalid/a/../b?x=%2F") + + def test_network_failure_switches_and_never_retries_same_proxy(self): + # Reserve then close a local port: connection refused, no Internet access. + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + dead = f"http://127.0.0.1:{sock.getsockname()[1]}" + sock.close() + with server() as good: + with httpx.Client(transport=Pool(json.dumps([dead, good.url])), trust_env=False, timeout=1) as client: + response, *_ = send(client) + with contextlib.closing(response): + self.assertEqual(response.read(), b"ok") + with httpx.Client(transport=Pool(dead, cooldown=0, max_attempts=10), trust_env=False, timeout=1) as client: + with self.assertRaisesRegex(httpx.ProxyError, "exhausted"): + send(client) + + def test_no_replacement_preserves_block_response_and_skips_direct_probe(self): + with server(403, b"evidence") as blocked: + with httpx.Client(transport=Pool(blocked.url), trust_env=False) as client: + response, *_ = send(client) + with contextlib.closing(response): + self.assertEqual(response.read(), b"evidence") + self.assertTrue(response.extensions["proxy_pool"]["failed"]) + with self.assertRaisesRegex(httpx.ProxyError, "direct fallback disabled"): + send(client) + self.assertEqual(len(blocked.requests), 1) + + def test_attempt_budget(self): + with server(403) as first, server(403) as second, server() as third: + with httpx.Client(transport=Pool(json.dumps([first.url, second.url, third.url]), max_attempts=2), trust_env=False) as client: + response, *_ = send(client) + with contextlib.closing(response): + response.read() + self.assertTrue(response.extensions["proxy_pool"]["failed"]) + self.assertEqual(len(third.requests), 0) + + def test_post_put_delete_never_replayed(self): + for method in ["POST", "PUT", "DELETE"]: + with self.subTest(method=method), server(403) as first, server() as second: + with httpx.Client(transport=Pool(json.dumps([first.url, second.url])), trust_env=False) as client: + response, *_ = send(client, method, content=b"write once") + with contextlib.closing(response): + response.read() + self.assertEqual(response.status_code, 403) + self.assertEqual(len(second.requests), 0) + self.assertEqual(first.requests[0][3], b"write once") + + def test_post_transport_error_never_replayed(self): + pool = Pool("localhost:8001\nlocalhost:8002") + first = mock.Mock() + first.handle_request.side_effect = httpx.ReadTimeout("private credentials") + pool.transports[pool.proxies[0]] = first + with httpx.Client(transport=pool, trust_env=False) as client: + with self.assertRaisesRegex(httpx.ProxyError, "automatic replay stopped") as caught: + send(client, "POST", content=b"once") + self.assertNotIn("private", str(caught.exception)) + self.assertEqual(len(pool.transports), 1) + + def test_retry_after_delta_and_date_stop_replay(self): + date = email.utils.format_datetime(datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=5)) + for value in ["120", date]: + with self.subTest(value=value), server(429, headers={"Retry-After": value}) as first, server() as second: + with httpx.Client(transport=Pool(json.dumps([first.url, second.url])), trust_env=False) as client: + response, *_ = send(client) + with contextlib.closing(response): + response.read() + self.assertEqual(response.status_code, 429) + with self.assertRaisesRegex(httpx.ProxyError, "Retry-After is still active"): + send(client) + self.assertEqual(len(second.requests), 0) + self.assertEqual(NS["retry_after_seconds"]("9" * 5000), 86400) + self.assertEqual(NS["retry_after_seconds"]("0" * 5000 + "1"), 1) + self.assertEqual(NS["retry_after_seconds"]("²"), 0) + self.assertEqual(NS["retry_after_seconds"]("garbage"), 0) + + def test_429_without_retry_after_and_safe_methods_switch(self): + for method in ["GET", "HEAD", "OPTIONS"]: + with self.subTest(method=method), server(429) as first, server() as second: + with httpx.Client(transport=Pool(json.dumps([first.url, second.url])), trust_env=False) as client: + response, *_ = send(client, method) + with contextlib.closing(response): + response.read() + self.assertEqual(response.status_code, 200) + self.assertEqual(len(second.requests), 1) + + def test_body_stream_failure_is_not_replayed(self): + class BrokenBody(httpx.SyncByteStream): + def __iter__(self): + yield b"partial" + raise httpx.ReadError("connection lost during body") + pool = Pool("localhost:8001\nlocalhost:8002") + first = mock.Mock() + first.handle_request.return_value = httpx.Response(200, stream=BrokenBody()) + pool.transports[pool.proxies[0]] = first + with httpx.Client(transport=pool, trust_env=False) as client: + response, *_ = send(client) + with contextlib.closing(response), self.assertRaises(httpx.ReadError): + response.read() + self.assertEqual(len(pool.transports), 1) + + def test_status_policy_is_configurable(self): + with server(403) as first, server() as second: + with httpx.Client(transport=Pool(json.dumps([first.url, second.url]), fail_status="none"), trust_env=False) as client: + response, *_ = send(client) + with contextlib.closing(response): + response.read() + self.assertFalse(response.extensions["proxy_pool"]["failed"]) + self.assertEqual(len(second.requests), 0) + + def test_cooldown_expiry(self): + pool = Pool("localhost:8001\nlocalhost:8002", cooldown=10) + pool.cooling[pool.proxies[0]] = 110 + pool.cooling[pool.proxies[1]] = 110 + with mock.patch.object(NS["time"], "monotonic", return_value=100): + self.assertIsNone(pool.select(set())) + with mock.patch.object(NS["time"], "monotonic", return_value=111): + self.assertEqual(pool.select(set()), pool.proxies[0]) + + def test_source_refresh_has_no_target_credentials_or_environment_proxy(self): + with server(403) as blocked, server() as good: + def reply(handler): + proxies = [blocked.url] if len(handler.server.requests) == 1 else [good.url] + return 200, {}, json.dumps({"proxies": proxies}).encode() + with server(reply=reply) as provider: + with mock.patch.dict(os.environ, {"HTTP_PROXY": "http://127.0.0.1:1", "NO_PROXY": ""}): + pool = Pool("", source=provider.url + "/?key=source-secret") + with httpx.Client(transport=pool, trust_env=False, cookies={"session": "target-secret"}) as client: + response, *_ = NS["send_path_as_is"]( + client, method="GET", url="http://target.invalid/", + headers=httpx.Headers({"Authorization": "Bearer target-secret"})) + with contextlib.closing(response): + self.assertEqual(response.read(), b"ok") + self.assertEqual(len(provider.requests), 2) + for _, _, headers, _ in provider.requests: + self.assertNotIn("Cookie", headers) + self.assertNotIn("Authorization", headers) + self.assertNotIn("Proxy-Authorization", headers) + self.assertNotIn("secret", self.diagnostics.getvalue()) + + def test_source_validation_limits_and_errors_redact_url(self): + for body in [b'[{"ip":"127.0.0.1"}]', b"", b"x" * (1024 * 1024 + 1)]: + with self.subTest(size=len(body)), server(body=body) as provider: + with self.assertRaises(ValueError) as caught: + Pool("", source=provider.url + "/?key=secret") + self.assertNotIn("secret", str(caught.exception)) + with server(302, headers={"Location": "http://127.0.0.1:1/?key=secret"}) as provider: + with self.assertRaisesRegex(ValueError, "HTTP 302"): + Pool("", source=provider.url) + with self.assertRaisesRegex(ValueError, "HTTP\\(S\\)"): + Pool("", source="file:///secret") + + def test_source_exhaustion_refresh_is_bounded(self): + with server(403) as blocked, server(body=blocked.url.encode()) as provider: + with httpx.Client(transport=Pool("", source=provider.url, cooldown=0), trust_env=False) as client: + response, *_ = send(client) + with contextlib.closing(response): + response.read() + self.assertEqual(response.status_code, 403) + self.assertEqual(len(provider.requests), 2) + self.assertEqual(len(blocked.requests), 1) + + def test_proxy_auth_never_in_diagnostics(self): + with server() as good: + proxy = good.url.replace("http://", "http://private-user:private-password@") + with httpx.Client(transport=Pool(proxy), trust_env=False) as client: + response, *_ = send(client) + with contextlib.closing(response): + response.read() + self.assertIn("Proxy-Authorization", good.requests[0][2]) + self.assertNotIn("private", self.diagnostics.getvalue()) + + def test_redirects_keep_literal_path(self): + def reply(handler): + if handler.path.endswith("/start"): + return 302, {"Location": "/x/../final"}, b"" + return 200, {}, b"redirected" + with server(reply=reply) as proxy: + with httpx.Client(transport=Pool(proxy.url), trust_env=False) as client: + response, final_url, _, history = send(client, url="http://target.invalid/start", follow_redirects=True) + with contextlib.closing(response): + self.assertEqual(response.read(), b"redirected") + self.assertEqual(final_url, "http://target.invalid/x/../final") + for item in history: + item.close() + self.assertEqual(proxy.requests[-1][1], "http://target.invalid/x/../final") + + @unittest.skipUnless(shutil.which("openssl"), "openssl required for local TLS fixture") + def test_https_connect_switches_on_decrypted_status(self): + with tempfile.TemporaryDirectory() as directory: + cert = str(Path(directory) / "cert.pem") + key = str(Path(directory) / "key.pem") + subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", + "-subj", "/CN=target.invalid", "-keyout", key, "-out", cert], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls.load_cert_chain(cert, key) + with server(403, b"tls-blocked", tls=tls) as blocked, server(tls=tls) as good: + with server(tunnel_port=blocked.server_port) as first, server(tunnel_port=good.server_port) as second: + pool = Pool(json.dumps([first.url, second.url]), transport_kwargs={"verify": False}) + with httpx.Client(transport=pool, trust_env=False) as client: + response, *_ = send(client, url="https://target.invalid/a/../b") + with contextlib.closing(response): + self.assertEqual(response.read(), b"ok") + self.assertEqual(blocked.requests[0][1], "/a/../b") + self.assertEqual(good.requests[0][1], "/a/../b") + self.assertEqual(first.requests[0][0], "CONNECT") + self.assertEqual(second.requests[0][0], "CONNECT") + + @unittest.skipUnless(importlib.util.find_spec("socksio"), "install httpx[socks] for SOCKS fixture") + def test_socks5_switches_and_preserves_path(self): + with server(403) as blocked, server() as good: + with socks_server(blocked.server_port) as first, socks_server(good.server_port) as second: + with httpx.Client(transport=Pool(json.dumps([first.url, second.url])), trust_env=False) as client: + response, *_ = send(client) + with contextlib.closing(response): + self.assertEqual(response.read(), b"ok") + self.assertEqual(first.requests, [("target.invalid", 80)]) + self.assertEqual(second.requests, [("target.invalid", 80)]) + self.assertEqual(good.requests[0][1], "/a/../b?x=%2F") + + def test_min_switch_interval_prevents_early_failover(self): + with server(403) as first, server() as second: + pool = Pool(json.dumps([first.url, second.url]), min_switch_interval=60, cooldown=0) + with httpx.Client(transport=pool, trust_env=False) as client: + with mock.patch.object(NS["time"], "monotonic", return_value=100): + response, *_ = send(client) + with contextlib.closing(response): + response.read() + self.assertEqual(response.status_code, 403) + self.assertEqual(len(second.requests), 0) + with mock.patch.object(NS["time"], "monotonic", return_value=161): + response, *_ = send(client) + with contextlib.closing(response): + self.assertEqual(response.read(), b"ok") + + def test_managed_fixed_policy_overrides_model_pool(self): + with server(body=b"managed") as fixed, server(body=b"model-choice") as other: + policy = {"mode": "fixed", "proxy_url": fixed.url} + result = subprocess.run([sys.executable, "-c", SCRIPT, "--url", "http://target.invalid/", + "--proxy-pool", other.url, "--proxy-strategy", "round_robin"], + text=True, capture_output=True, timeout=15, + env={**os.environ, "CSAI_HTTP_PROXY_CONFIG": json.dumps(policy)}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("managed", result.stdout) + self.assertEqual(len(other.requests), 0) + + def run_cli(self, *args): + return subprocess.run([sys.executable, "-c", SCRIPT, *args], text=True, capture_output=True, + timeout=15, env={**os.environ, "HTTP_PROXY": "http://127.0.0.1:1"}) + + def test_cli_pool_exhaustion_exit_code_and_no_direct_probe(self): + with server(403, b"blocked evidence") as proxy: + result = self.run_cli("--url", "http://target.invalid/", "--proxy-pool", proxy.url) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("blocked evidence", result.stdout) + self.assertIn("direct fallback disabled", result.stdout) + self.assertNotIn("DNS", result.stdout) + + def test_cli_single_proxy_and_no_pool_remain_compatible(self): + with server(body=b"single-proxy") as proxy: + result = self.run_cli("--url", "http://target.invalid/", "--proxy", proxy.url) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("single-proxy", result.stdout) + with server(body=b"direct") as target: + result = self.run_cli("--url", target.url, "--additional-args", "trust_env=false") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("direct", result.stdout) + + def test_cli_conflicting_proxy_options_fail_before_network(self): + result = self.run_cli("--url", "http://target.invalid", "--proxy", "localhost:8000", + "--proxy-pool", "localhost:8001") + self.assertEqual(result.returncode, 2) + self.assertIn("mutually exclusive", result.stderr) + + def test_cli_pool_flags(self): + # Exercise all public pool flags through the real CLI parser. + with server() as proxy: + result = self.run_cli("--url", "http://target.invalid/", "--proxy-pool", proxy.url, + "--proxy-max-attempts", "1", "--proxy-cooldown", "0", + "--proxy-fail-status", "none") + self.assertEqual(result.returncode, 0, result.stderr) + + def test_cli_round_robin_repeats_use_different_proxies(self): + with server(body=b"first-proxy") as first, server(body=b"second-proxy") as second: + result = self.run_cli("--url", "http://target.invalid/", "--proxy-pool", + json.dumps([first.url, second.url]), "--repeat", "2", + "--proxy-strategy", "round_robin") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("first-proxy", result.stdout) + self.assertIn("second-proxy", result.stdout) + self.assertEqual(len(first.requests), 1) + self.assertEqual(len(second.requests), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/http-framework-test.yaml b/tools/http-framework-test.yaml index 73454b164..4379af254 100644 --- a/tools/http-framework-test.yaml +++ b/tools/http-framework-test.yaml @@ -4,7 +4,10 @@ args: - "-c" - | import argparse + import datetime + import email.utils import json + import math import os import re import shlex @@ -274,6 +277,271 @@ args: request.url = original_url + def parse_proxy_list(raw): + """Accept newline-separated endpoints or JSON strings/{proxies: [...]}. + + Validate the entire update before using it. Never include credentials in errors. + """ + if isinstance(raw, str): + raw = raw.strip() + if not raw: + return [] + # A bare [IPv6]:port endpoint is a text line, not a JSON array. + ipv6_line = re.match(r'^\[[^\[\]"\r\n]+\]:', raw) is not None + if raw.startswith("{") or (raw.startswith("[") and not ipv6_line): + try: + raw = json.loads(raw) + except (ValueError, RecursionError): + raise ValueError("Invalid proxy list JSON") from None + else: + raw = raw.splitlines() + if isinstance(raw, dict): + raw = raw.get("proxies", raw.get("data")) + if not isinstance(raw, list) or len(raw) > 256: + raise ValueError("Proxy list must contain at most 256 entries") + proxies = [] + for value in raw: + if isinstance(value, dict): + if "url" in value: + value = value["url"] + else: + host = value.get("host", value.get("ip")) + port = value.get("port") + scheme = value.get("scheme", "http") + username = value.get("username", "") + password = value.get("password", "") + if (not isinstance(host, str) or not host or + not isinstance(port, (str, int)) or isinstance(port, bool) or + not str(port).isascii() or not str(port).isdigit() or + not all(isinstance(v, str) for v in (scheme, username, password)) or + any(c in host for c in "@/?#") or (password and not username)): + raise ValueError("Invalid structured proxy entry") + if ":" in host and not host.startswith("["): + host = "[" + host + "]" + auth = (urllib.parse.quote(username, safe="") + ":" + + urllib.parse.quote(password, safe="") + "@") if username else "" + value = f"{scheme}://{auth}{host}:{port}" + if not isinstance(value, str): + raise ValueError("Proxy entries must be URL strings or host:port strings") + value = value.strip() + if not value: + continue + if "://" not in value: + value = "http://" + value + try: + parts = urllib.parse.urlsplit(value) + valid = ( + parts.scheme.lower() in {"http", "https", "socks5"} + and parts.hostname and parts.port and 0 < parts.port <= 65535 + and parts.path in {"", "/"} and not parts.query and not parts.fragment + and not any(c.isspace() or ord(c) < 32 or ord(c) == 127 for c in value) + ) + if not valid: + raise ValueError() + # Normalize host/scheme for deduplication, preserving encoded credentials. + host = parts.hostname.lower() + if ":" in host: + host = "[" + host + "]" + auth = parts.netloc.rsplit("@", 1)[0] + "@" if "@" in parts.netloc else "" + value = f"{parts.scheme.lower()}://{auth}{host}:{parts.port}" + httpx.URL(value) + except (ValueError, httpx.InvalidURL): + raise ValueError("Invalid proxy endpoint (expected http(s)://host:port or socks5://host:port)") from None + if value not in proxies: + proxies.append(value) + return proxies + + + def fetch_proxy_source(source): + try: + parts = urllib.parse.urlsplit(source) + if parts.scheme not in {"http", "https"} or not parts.hostname or parts.fragment: + raise ValueError() + httpx.URL(source) + except (ValueError, httpx.InvalidURL): + raise ValueError("Proxy source must be an HTTP(S) URL") from None + # Separate control-plane client: no target cookies, headers, proxy or env proxy. + try: + with httpx.Client(timeout=10.0, trust_env=False, follow_redirects=False) as client: + with client.stream("GET", source) as response: + if response.status_code != 200: + raise ValueError("Proxy source returned HTTP " + str(response.status_code)) + body = bytearray() + for chunk in response.iter_bytes(chunk_size=65536): + body.extend(chunk) + if len(body) > 1024 * 1024: + raise ValueError("Proxy source response exceeds 1 MiB") + proxies = parse_proxy_list(body.decode("utf-8-sig")) + if not proxies: + raise ValueError("Proxy source returned an empty list") + return proxies + except (httpx.HTTPError, UnicodeError): + raise ValueError("Could not fetch proxy source (check connectivity, credentials and TLS)") from None + + + def retry_after_seconds(value): + if not value: + return 0.0 + value = value.strip() + if value.isascii() and value.isdigit(): + value = value.lstrip("0") or "0" + return 86400.0 if len(value) > 6 else float(min(int(value), 86400)) + try: + date = email.utils.parsedate_to_datetime(value) + if date.tzinfo is None: + date = date.replace(tzinfo=datetime.timezone.utc) + return min(86400.0, max(0.0, (date - datetime.datetime.now(datetime.timezone.utc)).total_seconds())) + except (ValueError, TypeError, OverflowError): + return 0.0 + + + class ProxyPoolTransport(httpx.BaseTransport): + """Invocation-local sticky proxy pool; retries only GET/HEAD/OPTIONS headers. + + Each hop has a bounded budget and never falls back to a direct connection. + Body-stream failures are surfaced rather than replaying a partial response. + """ + RETRY_METHODS = {"GET", "HEAD", "OPTIONS"} + + def __init__(self, proxies, source="", max_attempts=3, cooldown=60.0, + fail_status="403,429", transport_kwargs=None, strategy="sticky", min_switch_interval=0): + if not math.isfinite(min_switch_interval) or not 0 <= min_switch_interval <= 86400: + raise ValueError("proxy_min_switch_interval must be between 0 and 86400") + self.min_switch_interval = min_switch_interval + self.last_switch = None + if strategy not in {"sticky", "round_robin"}: + raise ValueError("proxy_strategy must be sticky or round_robin") + self.strategy = strategy + self.cursor = 0 + if not 1 <= max_attempts <= 10: + raise ValueError("proxy_max_attempts must be between 1 and 10") + if not math.isfinite(cooldown) or not 0 <= cooldown <= 86400: + raise ValueError("proxy_cooldown must be between 0 and 86400 seconds") + try: + statuses = set() if fail_status.strip().lower() == "none" else { + int(v.strip()) for v in fail_status.split(",") if v.strip() + } + if any(v < 400 or v > 599 for v in statuses): + raise ValueError() + except ValueError: + raise ValueError("proxy_fail_status must be comma-separated HTTP error codes (400-599)") from None + self.proxies = parse_proxy_list(proxies) + self.source = source.strip() + self.max_attempts = max_attempts + self.cooldown = cooldown + self.fail_status = statuses + self.transport_kwargs = dict(transport_kwargs or {}) + self.transport_kwargs.pop("proxy", None) + self.transport_kwargs["trust_env"] = False + self.transports = {} + self.cooling = {} + self.target_wait = {} + self.current = None + if self.source: + try: + self.refresh() + except ValueError as exc: + if not self.proxies: + raise + print(f"[proxy pool] {exc}; using configured static endpoints", file=sys.stderr) + if not self.proxies: + raise ValueError("Proxy pool has no endpoints") + + def refresh(self): + proxies = fetch_proxy_source(self.source) + merged = list(dict.fromkeys(self.proxies + proxies)) + if len(merged) > 256: + raise ValueError("Proxy pool exceeds 256 endpoints") + self.proxies = merged + + def select(self, attempted, advance=True): + now = time.monotonic() + if self.current and self.last_switch is not None and now - self.last_switch < self.min_switch_interval: + candidates = [self.current] + elif self.strategy == "round_robin": + candidates = self.proxies[self.cursor:] + self.proxies[:self.cursor] + else: + candidates = ([self.current] if self.current else []) + self.proxies + for proxy in candidates: + if proxy not in attempted and self.cooling.get(proxy, 0) <= now: + if advance: + if self.current != proxy: + self.last_switch = now + self.current = proxy + self.cursor = (self.proxies.index(proxy) + 1) % len(self.proxies) + return proxy + return None + + def handle_request(self, request): + target = (request.url.scheme, request.url.host, request.url.port) + if self.target_wait.get(target, 0) > time.monotonic(): + raise httpx.ProxyError("Target Retry-After is still active; no request sent", request=request) + attempted = set() + refreshed = False + retryable = request.method.upper() in self.RETRY_METHODS + for attempt in range(1, self.max_attempts + 1): + proxy = self.select(attempted) + if proxy is None and self.source and not refreshed: + refreshed = True + try: + self.refresh() + except ValueError as exc: + raise httpx.ProxyError(str(exc), request=request) from None + proxy = self.select(attempted) + if proxy is None: + raise httpx.ProxyError("Proxy pool exhausted: no unused healthy endpoint; direct fallback disabled", request=request) + attempted.add(proxy) + label = self.proxies.index(proxy) + 1 + print(f"[proxy pool] attempt {attempt}/{self.max_attempts} via proxy #{label}", file=sys.stderr) + try: + if proxy not in self.transports: + self.transports[proxy] = PathAsIsTransport(proxy=proxy, **self.transport_kwargs) + response = self.transports[proxy].handle_request(request) + except ImportError: + raise httpx.ProxyError("SOCKS support requires pip install 'httpx[socks]'", request=request) from None + except httpx.TransportError as exc: + self.cooling[proxy] = time.monotonic() + self.cooldown + print(f"[proxy pool] proxy #{label}: {type(exc).__name__}; cooling down", file=sys.stderr) + if not retryable or attempt == self.max_attempts: + raise httpx.ProxyError( + f"Proxy pool request failed ({type(exc).__name__}); automatic replay stopped", + request=request, + ) from None + continue + blocked = response.status_code in self.fail_status + wait = retry_after_seconds(response.headers.get("Retry-After")) if response.status_code == 429 else 0.0 + response.extensions["proxy_pool"] = {"proxy": label, "attempts": attempt, "failed": blocked} + if wait > 0: + self.target_wait[target] = time.monotonic() + wait + print("[proxy pool] Retry-After received; automatic replay stopped, retry later", file=sys.stderr) + if not blocked: + return response + self.cooling[proxy] = time.monotonic() + max(self.cooldown, wait) + print(f"[proxy pool] proxy #{label}: HTTP {response.status_code} matched failure policy; cooling down", file=sys.stderr) + if not retryable: + print("[proxy pool] method is not GET/HEAD/OPTIONS; automatic replay stopped", file=sys.stderr) + if not retryable or wait > 0 or attempt == self.max_attempts: + return response + # Keep the last response observable if no replacement is available. + available = self.select(attempted, advance=False) + if available is None and self.source and not refreshed: + refreshed = True + try: + self.refresh() + except ValueError as exc: + print(f"[proxy pool] {exc}; preserving last response", file=sys.stderr) + available = self.select(attempted, advance=False) + if available is None: + print("[proxy pool] no replacement available; preserving last response", file=sys.stderr) + return response + response.close() + raise httpx.ProxyError("Proxy pool attempt budget exhausted", request=request) + + def close(self): + for transport in self.transports.values(): + transport.close() + + def send_path_as_is( client: "httpx.Client", *, @@ -964,6 +1232,13 @@ args: parser.add_argument("--cookies", default="") parser.add_argument("--user-agent", dest="user_agent", default="") parser.add_argument("--proxy", default="") + parser.add_argument("--proxy-pool", default="") + parser.add_argument("--proxy-source", default="") + parser.add_argument("--proxy-strategy", choices=["sticky", "round_robin"], default="sticky") + parser.add_argument("--proxy-min-switch-interval", type=float, default=0) + parser.add_argument("--proxy-max-attempts", type=int, default=3) + parser.add_argument("--proxy-cooldown", type=float, default=60.0) + parser.add_argument("--proxy-fail-status", default="403,429") parser.add_argument("--timeout", default="") parser.add_argument("--repeat", type=int, default=1) parser.add_argument("--delay", default="0") @@ -1008,6 +1283,27 @@ args: ) # parse_known_args: tolerate legacy executor that appended bare "http2=true" tokens args, unknown_args = parser.parse_known_args() + managed = os.environ.get("CSAI_HTTP_PROXY_CONFIG", "") + if managed: + try: + policy = json.loads(managed) + args.proxy = "" + args.proxy_pool = "" + args.proxy_source = "" + if policy["mode"] == "fixed": + args.proxy = parse_proxy_list(policy["proxy_url"])[0] + elif policy["mode"] == "pool": + args.proxy_pool = policy["proxy_pool"] + args.proxy_source = policy["proxy_source"] + args.proxy_strategy = policy.get("strategy") or "sticky" + args.proxy_max_attempts = policy.get("max_attempts") or 3 + args.proxy_cooldown = policy.get("cooldown", 60) + args.proxy_min_switch_interval = policy.get("min_switch_interval", 60) + args.proxy_fail_status = "403,429" + else: + raise ValueError() + except (ValueError, KeyError, TypeError, IndexError, RecursionError): + parser.error("Invalid managed HTTP proxy configuration") leaked_options = parse_additional_options(" ".join(unknown_args)) if unknown_args else {} response_filter = (args.response_filter or "").strip() @@ -1118,6 +1414,9 @@ args: sys.exit(1) cert = additional_options.get("cert") explicit_proxy = (args.proxy or "").strip() + pool_enabled = bool(args.proxy_pool.strip() or args.proxy_source.strip()) + if pool_enabled and explicit_proxy: + parser.error("proxy and proxy_pool/proxy_source are mutually exclusive") forward_proxy = resolve_forward_proxy(explicit_proxy, trust_env, encoded_url) transport_kwargs = { @@ -1133,6 +1432,21 @@ args: if forward_proxy: transport_kwargs["proxy"] = forward_proxy + if pool_enabled: + try: + transport = ProxyPoolTransport( + args.proxy_pool, source=args.proxy_source, + max_attempts=args.proxy_max_attempts, cooldown=args.proxy_cooldown, + fail_status=args.proxy_fail_status, transport_kwargs=transport_kwargs, + strategy=args.proxy_strategy, + min_switch_interval=args.proxy_min_switch_interval, + ) + except ValueError as exc: + parser.error(str(exc)) + forward_proxy = "" + else: + transport = PathAsIsTransport(**transport_kwargs) + # Always path-as-is (security-testing default; matches curl --path-as-is) initial_absolute = needs_http_absolute_form(forward_proxy, encoded_url) initial_wire_target = wire_request_target(encoded_url, absolute_form=initial_absolute) @@ -1143,8 +1457,8 @@ args: "verify": verify_option, "follow_redirects": False, # redirects handled by send_path_as_is "cookies": cookie_jar, - "trust_env": trust_env, - "transport": PathAsIsTransport(**transport_kwargs), + "trust_env": False if pool_enabled else trust_env, + "transport": transport, } if args.show_command: @@ -1164,7 +1478,7 @@ args: exit_code = 0 verify_tls = verify_option if isinstance(verify_option, bool) else True - skip_probe = bool(forward_proxy) + skip_probe = pool_enabled or bool(forward_proxy) client = httpx.Client(**client_kwargs) try: @@ -1214,6 +1528,12 @@ args: decoded_body, used_encoding, encoding_source = decode_body_bytes(bytes(body_buffer), response.headers, args.response_encoding) print(f"\n===== Response #{run_index + 1} =====") + pool_meta = response.extensions.get("proxy_pool") + if pool_meta: + print(f"[proxy pool] final proxy #{pool_meta['proxy']}, attempts={pool_meta['attempts']}") + if pool_meta["failed"]: + exit_code = 1 + print("[proxy pool] failure policy still matched; response preserved, direct fallback disabled") download_target = (args.download or "").strip() if download_target: # 支持多次请求时通过 {i} 占位符区分文件名 @@ -1344,7 +1664,7 @@ args: print("\nInterrupted.", file=sys.stderr) sys.exit(130) enabled: true -short_description: "纯Python HTTP测试框架（httpx，会话复用+编码守护）" +short_description: "纯Python HTTP测试框架（httpx，代理池切换+会话复用+编码守护）" description: | 纯 Python 构建的 HTTP 测试框架：基于 httpx 会话，结合 socket 探针补齐 DNS/TCP/TLS 观测， 提供全流程的请求体编码守护、重复播放与可观测能力，适配盲注延时、复杂 payload 调优等场景。 @@ -1358,6 +1678,7 @@ description: | - 可重复观测：repeat/delay + TTFB/total/speed_download 统计，便于盲注/时序测试 - 扩展开关：http2 / httpx_options 解析 http2、cert、verify、trust_env、max_redirects 等 httpx 选项 - 响应体瘦身：response_filter 按行/块正则提取，配合 max_lines/max_bytes 限制 stdout，降低 Agent token 消耗 + - 代理池：proxy_pool 列表 / proxy_source HTTP(S) 接口，故障冷却与有界切换；仅自动重放 GET/HEAD/OPTIONS，429 Retry-After 停止重放；耗尽不直连 **响应过滤最佳实践：** - 大页面/HTML：用 `response_filter` 抓 error|exception|password|token|uid 等关键字行 @@ -1401,6 +1722,46 @@ parameters: description: "代理（http(s)://或socks5://地址，直接传递给httpx Client）" required: false flag: "--proxy" + - name: "proxy_pool" + type: "string" + description: "代理池：JSON 字符串数组或换行列表，支持 http(s)://host:port、socks5://host:port；与 proxy 互斥；状态仅在本次工具调用内保留" + required: false + flag: "--proxy-pool" + - name: "proxy_source" + type: "string" + description: "用户配置的代理供应商 HTTP(S) 接口；返回换行列表、JSON 数组或 {proxies:[...]} / {data:[...]}，元素可为 URL 或 host/ip+port 对象；开始及耗尽时有界拉取；与 proxy 互斥" + required: false + flag: "--proxy-source" + - name: "proxy_strategy" + type: "string" + description: "代理选择策略：sticky（默认，成功后复用），round_robin（每次请求/重定向跳轮询可用代理）；均跳过冷却代理" + required: false + default: "sticky" + flag: "--proxy-strategy" + - name: "proxy_min_switch_interval" + type: "string" + description: "代理最短切换间隔秒数（0-86400，工具参数默认0）；间隔内失败也不提前切换；受管设置可覆盖此值" + required: false + default: "0" + flag: "--proxy-min-switch-interval" + - name: "proxy_max_attempts" + type: "int" + description: "每个请求/重定向跳的代理尝试上限（1-10）；每个代理最多尝试一次；仅 GET/HEAD/OPTIONS 自动重放" + required: false + default: 3 + flag: "--proxy-max-attempts" + - name: "proxy_cooldown" + type: "string" + description: "失败代理冷却秒数（0-86400，默认60）；仅在本次工具调用内生效" + required: false + default: "60" + flag: "--proxy-cooldown" + - name: "proxy_fail_status" + type: "string" + description: "触发代理切换的 HTTP 错误状态码，逗号分隔（默认403,429）；只是切换策略，不代表确认封禁；传 none 禁用状态码切换；429 带有效 Retry-After 时停止自动重放" + required: false + default: "403,429" + flag: "--proxy-fail-status" - name: "timeout" type: "string" description: "最大超时时间（秒，支持小数）" diff --git a/web/static/css/style.css b/web/static/css/style.css index b423436a5..5f98025cd 100644 --- a/web/static/css/style.css +++ b/web/static/css/style.css @@ -47451,3 +47451,14 @@ html[data-theme="dark"] .token-usage-panel { --token-cache-color: #2dd4bf; } .token-usage-subtitle { max-width: 220px; line-height: 1.6; } .token-usage-trend-svg { min-width: 480px; } } + +/* HTTP-tool proxy policy settings */ +#settings-section-proxy { max-width: 960px; } +#settings-section-proxy .form-group { margin-bottom: 18px; } +#settings-section-proxy .form-hint { line-height: 1.65; margin: 12px 0 18px; } +#settings-section-proxy textarea { min-height: 120px; max-height: 240px; } +#settings-section-proxy [data-proxy-mode][hidden] { display: none !important; } +#settings-section-proxy .proxy-limit-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 16px; } +@media (max-width: 700px) { + #settings-section-proxy .proxy-limit-grid { grid-template-columns: 1fr; gap: 0; } +} diff --git a/web/static/i18n/en-US.json b/web/static/i18n/en-US.json index ec2660f9e..bb256e893 100644 --- a/web/static/i18n/en-US.json +++ b/web/static/i18n/en-US.json @@ -5133,5 +5133,31 @@ "promptRequired": "Enter a retest prompt.", "navigationChanged": "You switched conversations. Return to the retest branch to send the prompt.", "branchCreated": "The retest branch was created. Continue verification in that branch." + }, + "proxySettings": { + "title": "Proxy settings", + "scope": "Configure egress for the HTTP test tool. Use a fixed proxy for logins and stable sessions; use a pool for independent requests that allow egress changes.", + "mode": "Egress policy", + "manual": "Use tool arguments (default)", + "fixed": "Fixed proxy (recommended for sessions)", + "pool": "Proxy pool (opt in)", + "proxy_url": "Fixed proxy URL", + "proxy_pool": "Static proxies (one URL per line)", + "proxy_source": "Provider URL (optional)", + "strategy": "Pool selection strategy", + "sticky": "Reuse successful proxy", + "round_robin": "Rotate at intervals", + "max_attempts": "Maximum proxy attempts per request", + "cooldown": "Failure cooldown (seconds)", + "min_switch_interval": "Minimum switch interval (seconds)", + "secrets": "Saved addresses and credentials appear as ********. Keep the marker to retain a value, or clear it to delete. Pool mode requires a static list or provider.", + "isolation": "Kali tools and model APIs: this setting applies only to http-framework-test. Other scanners, terminals, external MCP and model APIs keep their own egress. System HTTP_PROXY is not modified; running tools are not restarted or rerouted. Use native fixed-proxy options for other tools that need a proxy.", + "stability": "A fixed proxy URL does not guarantee a fixed public IP from its provider. Use a provider with fixed egress or sticky sessions. Changes affect subsequent HTTP tool invocations only. Managed mode overrides model-supplied proxy options; it is not a network sandbox for arbitrary shell commands.", + "intervalHint": "Within each HTTP tool invocation, keep the proxy until the interval ends, even on failure. Use fixed mode for stable IPs or logins across invocations.", + "save": "Save and apply proxy settings", + "saved": "Proxy settings applied to subsequent HTTP tool invocations.", + "loadError": "Could not load proxy settings", + "saveError": "Could not apply proxy settings", + "invalidNumber": "Enter valid attempt and timing values" } } diff --git a/web/static/i18n/ru-RU.json b/web/static/i18n/ru-RU.json index 91d84d18d..aab82688d 100644 --- a/web/static/i18n/ru-RU.json +++ b/web/static/i18n/ru-RU.json @@ -5146,5 +5146,31 @@ "promptRequired": "Введите инструкцию для проверки.", "navigationChanged": "Вы переключились на другой диалог. Вернитесь в ветку проверки для отправки инструкции.", "branchCreated": "Ветка проверки создана. Продолжите проверку в этой ветке." + }, + "proxySettings": { + "title": "Настройки прокси", + "scope": "Исходящее соединение HTTP-инструмента. Фиксированный прокси подходит для авторизации и стабильных сеансов; пул — для независимых запросов.", + "mode": "Режим подключения", + "manual": "Параметры инструмента (по умолчанию)", + "fixed": "Фиксированный прокси (для сеансов)", + "pool": "Пул прокси (включить явно)", + "proxy_url": "URL фиксированного прокси", + "proxy_pool": "Статические прокси (по одному URL в строке)", + "proxy_source": "URL поставщика (необязательно)", + "strategy": "Выбор прокси из пула", + "sticky": "Сохранять рабочий прокси", + "round_robin": "Чередовать с интервалом", + "max_attempts": "Максимум попыток на запрос", + "cooldown": "Пауза после сбоя (секунды)", + "min_switch_interval": "Минимальный интервал смены (секунды)", + "secrets": "Сохранённые адреса и учётные данные показаны как ********. Сохраните маркер, чтобы оставить значение, или очистите поле для удаления. Для пула нужен список или поставщик.", + "isolation": "Инструменты Kali и API моделей: настройка действует только на http-framework-test. Другие сканеры, терминалы, внешние MCP и модели сохраняют свои подключения. Системный HTTP_PROXY не меняется. Для других инструментов используйте их собственные параметры фиксированного прокси.", + "stability": "Постоянный URL прокси не гарантирует постоянный публичный IP. Выберите поставщика с фиксированным выходом или привязкой сеанса. Изменения действуют только для новых запусков HTTP-инструмента. Управляемый режим заменяет параметры прокси от модели и не изолирует произвольные команды Shell.", + "intervalHint": "В рамках одного запуска HTTP-инструмента прокси сохраняется до конца интервала даже при сбое. Для стабильного IP между запусками и авторизации используйте фиксированный режим.", + "save": "Сохранить и применить", + "saved": "Настройки применены для следующих запусков HTTP-инструмента.", + "loadError": "Не удалось загрузить настройки прокси", + "saveError": "Не удалось применить настройки прокси", + "invalidNumber": "Укажите допустимые значения попыток и интервалов" } } diff --git a/web/static/i18n/zh-CN.json b/web/static/i18n/zh-CN.json index c43d1be74..1db08849a 100644 --- a/web/static/i18n/zh-CN.json +++ b/web/static/i18n/zh-CN.json @@ -5121,5 +5121,31 @@ "promptRequired": "请输入复测提示词。", "navigationChanged": "已切换到其他对话，请返回复测分支继续发送。", "branchCreated": "复测分支已创建，请在该分支继续验证。" + }, + "proxySettings": { + "title": "代理设置", + "scope": "配置 HTTP 测试工具的出口。固定代理适合登录态和需要稳定 IP 的任务；代理池用于允许切换出口的独立请求。", + "mode": "出口策略", + "manual": "沿用工具参数（默认）", + "fixed": "固定代理（推荐稳定会话）", + "pool": "代理池（显式启用）", + "proxy_url": "固定代理 URL", + "proxy_pool": "静态代理列表（每行一个 URL）", + "proxy_source": "供应商接口 URL（可选）", + "strategy": "代理池选择策略", + "sticky": "复用成功代理", + "round_robin": "按间隔轮询", + "max_attempts": "每个请求最多尝试的代理数", + "cooldown": "失败冷却（秒）", + "min_switch_interval": "最短切换间隔（秒）", + "secrets": "已保存的地址与凭证显示为 ********，原样保存会保留，清空可删除。代理池模式中，静态列表和供应商至少填写一项。", + "isolation": "Kali 工具与大模型 API：此设置仅作用于 http-framework-test。其他扫描工具、终端、外部 MCP 和模型 API 保持自身出口；不会修改系统 HTTP_PROXY，也不会在运行中重启或切换其他工具的连接。需要代理的其他工具请使用其原生固定代理参数。", + "stability": "固定代理地址不保证供应商固定公网 IP。稳定会话请使用供应商的固定出口或粘性会话服务。设置修改仅影响后续启动的 HTTP 工具调用。受管模式会覆盖模型传入的代理选项；这不是对任意 Shell 命令的网络隔离。", + "intervalHint": "每次 HTTP 工具调用内保持当前代理至间隔结束；失败时也不提前切换。跨调用需要稳定 IP 或登录态时，请使用固定代理。", + "save": "保存并应用代理设置", + "saved": "代理设置已应用，后续 HTTP 工具调用生效。", + "loadError": "加载代理设置失败", + "saveError": "应用代理设置失败", + "invalidNumber": "请填写有效的代理次数和时间参数" } } diff --git a/web/static/js/proxy-settings.js b/web/static/js/proxy-settings.js new file mode 100644 index 000000000..9ea60130a --- /dev/null +++ b/web/static/js/proxy-settings.js @@ -0,0 +1,43 @@ +// Managed HTTP-tool policy. Model transports and unrelated tool processes are separate. +let proxySettingsLoaded = false; +async function loadProxySettings() { + proxySettingsLoaded = false; + document.getElementById("proxy-settings-save").disabled = true; + try { + const response = await apiFetch('/api/config'); + if (!response.ok) throw new Error(window.t('proxySettings.loadError')); + const saved = (await response.json()).proxy || {}; + const cfg = saved.mode ? saved : {}; + const defaults = {mode: 'manual', proxy_url: '', proxy_pool: '', proxy_source: '', strategy: 'sticky', max_attempts: 3, cooldown: 60, min_switch_interval: 60}; + for (const [key, fallback] of Object.entries(defaults)) { + document.getElementById(`proxy-setting-${key}`).value = cfg[key] === undefined || cfg[key] === '' || (key === 'max_attempts' && cfg[key] === 0) ? fallback : cfg[key]; + } + syncProxySettingsMode(); + proxySettingsLoaded = true; + document.getElementById("proxy-settings-save").disabled = false; + } catch (error) { alert(error.message); } +} +function syncProxySettingsMode() { + const mode = document.getElementById('proxy-setting-mode').value; + document.querySelectorAll('[data-proxy-mode]').forEach(el => { el.hidden = el.dataset.proxyMode !== mode; }); +} +async function saveProxySettings() { + if (!proxySettingsLoaded) return; + const button = document.getElementById('proxy-settings-save'); + button.disabled = true; + try { + const proxy = {}; + for (const key of ['mode', 'proxy_url', 'proxy_pool', 'proxy_source', 'strategy']) proxy[key] = document.getElementById(`proxy-setting-${key}`).value.trim(); + for (const key of ['max_attempts', 'cooldown', 'min_switch_interval']) { + const input = document.getElementById(`proxy-setting-${key}`); + const valid = input.checkValidity() && input.value !== ''; + if (!valid && proxy.mode === 'pool') throw new Error(window.t('proxySettings.invalidNumber')); + proxy[key] = valid ? Number(input.value) : ({max_attempts: 3, cooldown: 60, min_switch_interval: 60})[key]; + } + const response = await apiFetch('/api/config', {method: 'PUT', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({proxy})}); + if (!response.ok) throw new Error((await response.json()).error || window.t('proxySettings.saveError')); + await loadProxySettings(); + alert(window.t('proxySettings.saved')); + } catch (error) { alert(error.message); } + finally { button.disabled = !proxySettingsLoaded; } +} diff --git a/web/static/js/proxy-settings.test.cjs b/web/static/js/proxy-settings.test.cjs new file mode 100644 index 000000000..f8d616a29 --- /dev/null +++ b/web/static/js/proxy-settings.test.cjs @@ -0,0 +1,35 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const vm = require('node:vm'); +const fs = require('node:fs'); +function fixture(saved = {}) { + const elements = {}; + for (const key of ['mode','proxy_url','proxy_pool','proxy_source','strategy','max_attempts','cooldown','min_switch_interval']) elements[`proxy-setting-${key}`] = {value:'',checkValidity:()=>true}; + elements['proxy-settings-save'] = {disabled:false}; + const calls = [], alerts = [], panels = ['fixed','pool'].map(mode=>({dataset:{proxyMode:mode},hidden:true})); + const ctx = {window:{t:x=>x}, alert:x=>alerts.push(x),document:{getElementById:id=>elements[id],querySelectorAll:()=>panels},apiFetch:async(url,options)=>{calls.push({url,options});return {ok:true,json:async()=>({proxy:saved})};}}; + vm.createContext(ctx);vm.runInContext(fs.readFileSync(__dirname+'/proxy-settings.js','utf8'),ctx); + return {ctx,elements,calls,alerts,panels}; +} +test('initial defaults and mode panels are usable', async()=>{ + const f=fixture({mode:'',max_attempts:0});await f.ctx.loadProxySettings(); + assert.equal(f.elements['proxy-setting-max_attempts'].value,3); + assert.equal(f.elements['proxy-setting-min_switch_interval'].value,60); + f.elements['proxy-setting-mode'].value='fixed';f.ctx.syncProxySettingsMode();assert.equal(f.panels[0].hidden,false);assert.equal(f.panels[1].hidden,true); +}); +test('masked credentials survive save without restarting other services',async()=>{ + const f=fixture({mode:'fixed',proxy_url:'********',max_attempts:3,cooldown:60,min_switch_interval:60});await f.ctx.loadProxySettings();await f.ctx.saveProxySettings(); + const write=f.calls.find(x=>x.options?.method==='PUT');assert.equal(JSON.parse(write.options.body).proxy.proxy_url,'********'); + assert.equal(f.calls.some(x=>x.url==='/api/config/apply'),false); + assert.equal(f.elements['proxy-settings-save'].disabled,false); +}); +test('hidden pool validation does not prevent switching to a fixed proxy',async()=>{ + const f=fixture({mode:'pool'});await f.ctx.loadProxySettings(); + f.elements['proxy-setting-max_attempts'].value=''; + f.elements['proxy-setting-max_attempts'].checkValidity=()=>false; + f.elements['proxy-setting-mode'].value='fixed'; + f.elements['proxy-setting-proxy_url'].value='http://localhost:8080'; + await f.ctx.saveProxySettings(); + const write=f.calls.find(x=>x.options?.method==='PUT');assert.ok(write); + assert.equal(JSON.parse(write.options.body).proxy.max_attempts,3); +}); diff --git a/web/static/js/settings.js b/web/static/js/settings.js index fca5c0694..83615af64 100644 --- a/web/static/js/settings.js +++ b/web/static/js/settings.js @@ -686,6 +686,7 @@ function syncC2NavFromConfig(cfg) { // 切换设置分类 function switchSettingsSection(section) { + if (section === "proxy") loadProxySettings(); if (section === 'rbac') { if (typeof switchPage === 'function') { switchPage('platform-rbac'); diff --git a/web/templates/index.html b/web/templates/index.html index 65c782bf2..60b0a4ee2 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -39,7 +39,7 @@ } })(); - + @@ -3605,6 +3605,7 @@

系统设置

Token 用量
+
代理设置
人机协同
@@ -4017,7 +4018,22 @@

Agent 配置

- +

代理设置

+
+ + +

已保存的地址与凭证显示为 ********，原样保存会保留，清空可删除。代理池模式中，静态列表和供应商至少填写一项。

+

Kali 工具与大模型 API：此设置仅作用于 http-framework-test。其他扫描工具、终端、外部 MCP 和模型 API 保持自身出口；不会修改系统 HTTP_PROXY，也不会在运行中重启或切换其他工具的连接。需要代理的其他工具请使用其原生固定代理参数。

+

固定代理地址不保证供应商固定公网 IP。稳定会话请使用供应商的固定出口或粘性会话服务。设置修改仅影响后续启动的 HTTP 工具调用。受管模式会覆盖模型传入的代理选项；这不是对任意 Shell 命令的网络隔离。

+
+

人机协同

@@ -7031,7 +7047,8 @@

已绑定平台账号

- + +