From c47bda8ad013ca150be9172b8e06d8d5553660cf Mon Sep 17 00:00:00 2001 From: Reyn Date: Tue, 13 Oct 2026 19:39:11 +1100 Subject: [PATCH 1/4] docs: add project README with architecture, results and quick start Co-Authored-By: Claude Opus 5.5 --- README.md | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..34ece18 --- /dev/null +++ b/README.md @@ -0,0 +1,107 @@ +# 行界 WanderAI · 智能旅行规划助手 + +输入目的地、天数和偏好,生成**每个地点都真实存在、交通时长可执行**的行程,并在地图上展示;之后可以继续用聊天追问、修改行程。 + +2025 百度文心黑客松参赛项目,赛后独立重构为可评测、可容错的版本(重构过程见 [docs/upgrade_plan.md](docs/upgrade_plan.md))。 + +## 功能 + +| 能力 | 做法 | +|---|---| +| **Agent 行程规划** | Function Calling 循环编排游记检索、高德 POI / 路线 / 天气等工具,以结构化 JSON 提交行程;规则校验器检查地点真实性、城市一致、相邻两站 ≤60 分钟、每日 ≤12 小时,不通过的问题回填给 LLM 自行修正 | +| **混合检索 RAG** | 地名过滤 + BM25(jieba)与 bge 向量的 RRF 融合 + bge-reranker 重排,基于 217 篇游记回答经验类问题并标注引用 | +| **知识图谱** | 游记中的地点对齐到高德 POI,存入 Neo4j(共现、邻近、所属关系);“附近有什么”“去了 A 还去哪”类问题通过参数化 Cypher 模板查询 | +| **动态路由** | 聊天消息按 5 类意图分发:修改行程 / 游记问答 / 地点关系 / 天气与交通 / 闲聊;本地 RoBERTa 路由模型置信度不足时交给 LLM | +| **长期记忆** | 记住同行人、饮食、行动限制、预算、喜好等 8 类信息并在规划时遵守;写入时校验证据必须出自用户原话,过滤敏感信息,支持改口覆盖 | +| **工程化** | 统一错误码与用户提示语;LLM / 高德 / 图谱客户端均有超时、按错误类型重试、熔断与并发控制,任一依赖失败时逐级降级;request-id 全链路追踪;配置外置 | + +## 架构 + +```mermaid +flowchart LR + B[浏览器
ai-chat / output-map / rag-search] --> G[Node 网关 :3000
request-id · 统一错误 · 高德代理] + G --> A[agent_service :8002
FastAPI] + G --> AM[高德 Web 服务] + A --> R{动态路由} + R -->|修改行程| P[PlanAgent
Function Calling + 校验自纠] + R -->|游记问答| Q[RAG
BM25 + 向量 + 重排] + R -->|附近 / 常一起去| T[工具问答] + R -->|天气 / 交通| T + P & T --> AM + P & Q --> V[(Chroma 向量库)] + T --> N[(Neo4j 图谱)] + A --> M[(SQLite 记忆)] + A --> L[DeepSeek API] +``` + +## 评测结果 + +下表的数字都可以用 `eval/` 下的脚本复现,报告在同一目录。 + +| 场景 | 对比 | 结果 | 报告 | +|---|---|---|---| +| 行程可执行性 | 旧流程(文本生成 + 正则解析)vs Agent,同批 20 个需求 | 超过 60 分钟的路段 22.3% → **0%** | [agent_report](eval/agent_report.md) | +| 校验自纠 | 首次提交 vs 自纠后 | 通过率 80% → **100%** | 同上 | +| 动态路由 | 每句都重新规划 vs 按意图分发,40 条人工消息 | 延迟 14.5s → **5.3s**,token -80%,误改行程 17/32 → **0/32** | [router_report](eval/router_report.md) | +| 长期记忆 | 不带记忆 vs 带记忆,6 个场景 × 3 轮 | 违反用户约束 14/18 → **3/18** | [memory_report](eval/memory_report.md) | +| 知识图谱 | 只用高德周边搜索 vs 加图谱,16 个问题 × 3 次 | 有游记依据的推荐 2.2 → **5.3** 个,延迟 -33%,token -23% | [graph_report](eval/graph_report.md) | +| 实体对齐 | 三轮人工抽检,每轮重新抽样 | 精确率 70% → **82%~88%** | [alignment_review](eval/graph_alignment_review.md) | + +检索指标(nDCG / Recall / MRR)的相关性标注由 LLM 完成,人工抽检完成前不列出,见 [rag_report](eval/rag_report.md)。 + +## 快速开始 + +需要 Node 18+、Python 3.10+、[DeepSeek API Key](https://platform.deepseek.com/)(或任意 OpenAI 兼容服务)和[高德 Web 服务 Key](https://console.amap.com/)。 + +```bash +cp .env.example .env # 填入 LLM_API_KEY、AMAP_WEB_SERVICE_KEY +npm install + +cd agent_service +python -m venv .venv-win +.venv-win/Scripts/pip install -r requirements.txt +.venv-win/Scripts/python ingest_csv.py --csv ../data/travel_notes.csv --reset # 游记入库(向量库) +.venv-win/Scripts/python app.py # Python 服务 :8002 +``` + +另开一个终端: + +```bash +npm start # 网关 :3000,浏览器打开 http://localhost:3000/ +``` + +可选组件: + +- **知识图谱**:在 `.env` 设置 `NEO4J_PASSWORD`,执行 `docker compose up -d neo4j`,然后在 `agent_service` 目录下运行 `python -m graph.build --reset`。不启用时,地点关系类问题自动改用高德周边搜索。 +- **本地分类模型**:模型文件体积较大,没有放进仓库。按 [agent_service/README.md](agent_service/README.md#意图分类) 训练后放到 `agent_service/models/`;缺少模型时自动改用 LLM 分类。 + +健康检查:`GET /api/readyz`,会列出 LLM、向量库、分类模型、图谱各自的状态。 + +## 目录 + +```text +server.js Node 网关 +ai-chat.html 等 前端页面 +agent_service/ + core/ 配置、日志、错误码、LLM / 高德客户端、重试与熔断 + agent/ PlanAgent、工具、校验器、动态路由、工具问答 + rag/ 分块、入库、元数据抽取、BM25、重排、混合检索 + graph/ 实体对齐、建图、Cypher 模板、Neo4j 客户端 + memory/ 长期记忆、会话摘要 + classifier/ 风格 / 路由分类模型与训练脚本 + tests/ pytest 用例 +eval/ 评测脚本与报告 +docs/ 升级规划与设计说明 +``` + +## 文档 + +- [project_documentation.md](project_documentation.md):模块说明、配置、接口、已知问题 +- [agent_service/README.md](agent_service/README.md):Python 服务的安装、入库、训练与测试 +- [docs/upgrade_plan.md](docs/upgrade_plan.md):从参赛版本到当前版本的分阶段改造 + +## 局限 + +- 游记语料只有 217 篇,图谱中多数地点只被 1 篇游记提到。 +- 没有登录体系,记忆按浏览器生成的匿名 ID 隔离,仅适合演示。 +- 一次 Agent 规划约 30~50 秒;高德个人 Key 有 QPS 和日配额限制。 From d5fcee7d12e259980e86066652615dc7458b7560 Mon Sep 17 00:00:00 2001 From: Reyn Date: Tue, 13 Oct 2026 22:40:47 +1100 Subject: [PATCH 2/4] P8: metrics, monitoring page, rate limiting, overload protection, retrieval cache, load test, Docker Compose, CI - metrics: per-route counts and P50/P95/P99, error codes, LLM calls/tokens/queue, tool calls, cache hit rates, breaker states; JSON and Prometheus formats - gateway: per-IP token-bucket rate limiting (429 + Retry-After) and a global in-flight cap for plan/chat (503 SERVER_BUSY); aggregated /api/metrics and admin.html behind ADMIN_TOKEN (loopback-only when unset) - retrieval result cache (thread-safe LRU+TTL, cleared on re-ingest) - load test with a mock LLM: capacity, overload, rate limit, circuit breaker - fix: open-circuit rejections were retried with backoff (1-1.8s -> 33-75ms) - fix: Neo4j server-side transaction timeouts now degrade instead of INTERNAL - Docker images for gateway and agent_service, compose with health-gated startup; agent not published; GitHub Actions for tests and image builds - map page loads the Amap JS key from /api/config instead of hardcoding it - pinned requirements; neo4j was missing; training deps split out Co-Authored-By: Claude Opus 5.5 --- .dockerignore | 18 ++ .env.example | 13 +- .github/workflows/ci.yml | 66 +++++ Dockerfile | 20 ++ README.md | 20 ++ admin.html | 210 ++++++++++++++ agent_service/Dockerfile | 30 ++ agent_service/agent/tools.py | 4 + agent_service/app.py | 21 +- agent_service/config.py | 2 + agent_service/core/amap.py | 29 +- agent_service/core/cache.py | 44 +++ agent_service/core/llm.py | 23 +- agent_service/core/metrics.py | 164 +++++++++++ agent_service/core/resilience.py | 10 +- agent_service/graph/client.py | 11 +- agent_service/rag/retriever.py | 15 + agent_service/requirements-dev.txt | 4 +- agent_service/requirements-train.txt | 2 + agent_service/requirements.txt | 24 +- agent_service/tests/test_graph.py | 37 +++ agent_service/tests/test_ops.py | 106 +++++++ config.yaml | 2 + docker-compose.yml | 56 +++- docs/interview_notes.md | 85 +++++- docs/upgrade_plan.md | 2 +- eval/data/load_test_results.json | 394 +++++++++++++++++++++++++++ eval/data/load_test_results_v1.json | 394 +++++++++++++++++++++++++++ eval/load_test.md | 81 ++++++ eval/load_test.py | 323 ++++++++++++++++++++++ eval/load_test_v1.md | 70 +++++ eval/mock_llm.py | 60 ++++ gateway/ops.js | 140 ++++++++++ gateway/ops.test.js | 63 +++++ output-map.html | 26 +- package.json | 3 +- project_documentation.md | 13 +- server.js | 114 +++++++- src/map.js | 12 +- 39 files changed, 2643 insertions(+), 68 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/ci.yml create mode 100644 Dockerfile create mode 100644 admin.html create mode 100644 agent_service/Dockerfile create mode 100644 agent_service/core/cache.py create mode 100644 agent_service/core/metrics.py create mode 100644 agent_service/requirements-train.txt create mode 100644 agent_service/tests/test_ops.py create mode 100644 eval/data/load_test_results.json create mode 100644 eval/data/load_test_results_v1.json create mode 100644 eval/load_test.md create mode 100644 eval/load_test.py create mode 100644 eval/load_test_v1.md create mode 100644 eval/mock_llm.py create mode 100644 gateway/ops.js create mode 100644 gateway/ops.test.js diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..84547a8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +# 构建上下文里不需要的大文件与敏感文件 +.env +.env.* +!.env.example +.git +node_modules +logs +data +agent_service/.venv +agent_service/.venv-win +agent_service/models +agent_service/chroma_db +**/__pycache__ +**/*.pyc +eval +docs +example +best_model diff --git a/.env.example b/.env.example index 716bf36..6d5db17 100644 --- a/.env.example +++ b/.env.example @@ -3,11 +3,22 @@ LLM_API_KEY=your_deepseek_key_here # 高德 Web 服务 Key(仅网关使用) AMAP_WEB_SERVICE_KEY=your_amap_web_service_key_here -AMAP_JS_KEY=your_amap_js_key_here # 仅 example/runAmapExample.js 使用 +# 高德 JS API Key 与安全密钥(地图页使用,由网关 /api/config 下发到浏览器) +AMAP_JS_KEY=your_amap_js_key_here +AMAP_JS_SECURITY_CODE=your_amap_js_security_code_here # ===== Node 网关 ===== PORT=3000 AGENT_BASE=http://127.0.0.1:8002 +# 监控页 /admin.html 与 /api/metrics:设置后凭 token 访问;不设置时只允许本机访问(Docker 部署时必须设置) +ADMIN_TOKEN= +# 限流:每个 IP 每分钟的请求数(0 表示不限);重请求 = 规划 / 聊天 / 问答 +RATE_LIMIT_API_PER_MIN=120 +RATE_LIMIT_HEAVY_PER_MIN=10 +# 过载保护:同时进行的重请求上限,超过直接返回“当前使用人数较多” +HEAVY_MAX_IN_FLIGHT=8 # Neo4j(本地 Docker 容器) NEO4J_PASSWORD=change_me +# Docker 部署(可选):指向本机已下载的 HuggingFace 模型缓存,避免容器首次启动重新下载 +# HF_CACHE_DIR=C:/Users/you/.cache/huggingface diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a95b36b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,66 @@ +# 每次推送和 PR 都跑:Python 单元测试、网关单元测试、两个 Docker 镜像能否构建 +name: CI + +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true # 同一分支连续推送时,只保留最新一次 + +jobs: + python: + name: agent_service 测试 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: agent_service/requirements*.txt + - name: 安装依赖(CPU 版 torch) + run: | + pip install torch==2.14.1 --index-url https://download.pytorch.org/whl/cpu + pip install -r agent_service/requirements.txt -r agent_service/requirements-dev.txt -r agent_service/requirements-train.txt + - name: pytest + working-directory: agent_service + run: python -m pytest -q tests + + gateway: + name: 网关测试 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: node --check server.js + - run: npm test + + docker: + name: Docker 镜像构建 + runs-on: ubuntu-latest + needs: [python, gateway] # 测试通过后再构建,省时间 + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - name: 构建网关镜像 + uses: docker/build-push-action@v6 + with: + context: . + push: false + cache-from: type=gha + cache-to: type=gha,mode=max + - name: 构建 agent_service 镜像 + uses: docker/build-push-action@v6 + with: + context: . + file: agent_service/Dockerfile + push: false + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..246af44 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +# Node 网关镜像:docker build -t wanderai-gateway . +FROM node:22-alpine + +WORKDIR /app +ENV NODE_ENV=production + +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev + +# 只拷网关运行需要的文件(页面、前端脚本、网关代码) +COPY server.js ./ +COPY gateway/ gateway/ +COPY src/ src/ +COPY ai-chat.html output-map.html rag-search.html admin.html ./ + +USER node +EXPOSE 3000 +HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 \ + CMD wget -qO- http://127.0.0.1:3000/ai-chat.html >/dev/null || exit 1 +CMD ["node", "server.js"] diff --git a/README.md b/README.md index 34ece18..c5e6069 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ | **动态路由** | 聊天消息按 5 类意图分发:修改行程 / 游记问答 / 地点关系 / 天气与交通 / 闲聊;本地 RoBERTa 路由模型置信度不足时交给 LLM | | **长期记忆** | 记住同行人、饮食、行动限制、预算、喜好等 8 类信息并在规划时遵守;写入时校验证据必须出自用户原话,过滤敏感信息,支持改口覆盖 | | **工程化** | 统一错误码与用户提示语;LLM / 高德 / 图谱客户端均有超时、按错误类型重试、熔断与并发控制,任一依赖失败时逐级降级;request-id 全链路追踪;配置外置 | +| **运维** | 网关按 IP 令牌桶限流与重请求过载保护;检索结果缓存;指标接口(JSON / Prometheus)与监控页;Docker Compose 一键部署;GitHub Actions 运行测试与镜像构建 | ## 架构 @@ -46,11 +47,26 @@ flowchart LR | 长期记忆 | 不带记忆 vs 带记忆,6 个场景 × 3 轮 | 违反用户约束 14/18 → **3/18** | [memory_report](eval/memory_report.md) | | 知识图谱 | 只用高德周边搜索 vs 加图谱,16 个问题 × 3 次 | 有游记依据的推荐 2.2 → **5.3** 个,延迟 -33%,token -23% | [graph_report](eval/graph_report.md) | | 实体对齐 | 三轮人工抽检,每轮重新抽样 | 精确率 70% → **82%~88%** | [alignment_review](eval/graph_alignment_review.md) | +| 压测 | 模拟 LLM(固定 1 秒),单机 | 聊天并发 1 → 32 无失败、吞吐受 LLM 闸门约束在 7.4 次/秒;40 个同时请求时 32 个在 0.2 秒内收到“繁忙”;熔断后失败耗时约 1.5s → **33~75ms** | [load_test](eval/load_test.md) | 检索指标(nDCG / Recall / MRR)的相关性标注由 LLM 完成,人工抽检完成前不列出,见 [rag_report](eval/rag_report.md)。 ## 快速开始 +### Docker(推荐) + +需要 Docker Desktop,以及 DeepSeek 与高德的 Key。 + +```bash +cp .env.example .env # 填入 LLM_API_KEY、AMAP_WEB_SERVICE_KEY、AMAP_JS_KEY、NEO4J_PASSWORD、ADMIN_TOKEN +docker compose up -d --build # 启动网关、agent_service、Neo4j +``` + +浏览器打开 http://localhost:3000/。监控页是 http://localhost:3000/admin.html,打开时输入 `ADMIN_TOKEN`。 +向量库(`agent_service/chroma_db`)和知识图谱需要先按下面的本地方式入库、建图,容器会挂载这些目录。 + +### 本地运行 + 需要 Node 18+、Python 3.10+、[DeepSeek API Key](https://platform.deepseek.com/)(或任意 OpenAI 兼容服务)和[高德 Web 服务 Key](https://console.amap.com/)。 ```bash @@ -76,11 +92,14 @@ npm start # 网关 :3000,浏览器打开 http://localhost: - **本地分类模型**:模型文件体积较大,没有放进仓库。按 [agent_service/README.md](agent_service/README.md#意图分类) 训练后放到 `agent_service/models/`;缺少模型时自动改用 LLM 分类。 健康检查:`GET /api/readyz`,会列出 LLM、向量库、分类模型、图谱各自的状态。 +运行指标:`GET /api/metrics`(JSON,监控页使用)或 `GET /api/metrics?format=prometheus`。 ## 目录 ```text server.js Node 网关 +gateway/ 限流、过载保护、指标 +admin.html 监控页 ai-chat.html 等 前端页面 agent_service/ core/ 配置、日志、错误码、LLM / 高德客户端、重试与熔断 @@ -105,3 +124,4 @@ docs/ 升级规划与设计说明 - 游记语料只有 217 篇,图谱中多数地点只被 1 篇游记提到。 - 没有登录体系,记忆按浏览器生成的匿名 ID 隔离,仅适合演示。 - 一次 Agent 规划约 30~50 秒;高德个人 Key 有 QPS 和日配额限制。 +- 限流、缓存与指标都在进程内,只适合单实例;多实例部署需要换成 Redis 等共享存储。 diff --git a/admin.html b/admin.html new file mode 100644 index 0000000..9925de9 --- /dev/null +++ b/admin.html @@ -0,0 +1,210 @@ + + + + + + WanderAI 运行监控 + + + +
+

WanderAI 运行监控

+ 检查中 + 每 5 秒刷新 · — · +
+
+
+
+ +

依赖状态

+
+ +

接口(网关)

+
+ +

接口(Agent 服务)

+
+ +
+
+

LLM 调用

+
+
+
+

工具调用

+
+
+
+ +
+
+

错误码

+
+
+
+

缓存与保护

+
+
+
+
+ + + + diff --git a/agent_service/Dockerfile b/agent_service/Dockerfile new file mode 100644 index 0000000..ff86c6f --- /dev/null +++ b/agent_service/Dockerfile @@ -0,0 +1,30 @@ +# agent_service 镜像。构建上下文是项目根目录(需要 config.yaml): +# docker build -f agent_service/Dockerfile -t wanderai-agent . +FROM python:3.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + HF_HOME=/models/hf + +WORKDIR /app + +# 先装依赖再拷代码:代码改动不会让依赖层失效,重新构建只需几秒 +RUN pip install torch==2.14.1 --index-url https://download.pytorch.org/whl/cpu +COPY agent_service/requirements.txt agent_service/requirements.txt +RUN pip install -r agent_service/requirements.txt + +COPY config.yaml ./ +COPY agent_service/ agent_service/ + +# 不用 root 运行 +RUN useradd --create-home --uid 1000 app && mkdir -p /app/logs /models/hf && chown -R app /app /models +USER app + +WORKDIR /app/agent_service +EXPOSE 8002 +# 容器内必须监听 0.0.0.0,端口是否对外暴露由 compose 决定 +ENV SERVER__HOST=0.0.0.0 +HEALTHCHECK --interval=15s --timeout=5s --start-period=120s --retries=5 \ + CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8002/healthz', timeout=3).status == 200 else 1)" +CMD ["python", "app.py"] diff --git a/agent_service/agent/tools.py b/agent_service/agent/tools.py index 72000c3..b5527f2 100644 --- a/agent_service/agent/tools.py +++ b/agent_service/agent/tools.py @@ -16,6 +16,7 @@ from graph.queries import TEMPLATES, run_template from rag.metadata import normalize_place from core.logging import get_logger +from core.metrics import metrics from rag.retriever import Retriever log = get_logger("agent.tools") @@ -260,6 +261,7 @@ def _record(self, ctx: PlanContext, name: str, args: Dict[str, Any], result: Dic latency_ms=round((time.perf_counter() - started) * 1000), ) ctx.records.append(rec) + metrics.tool_call(name, rec.ok) log.info("tool call", extra={"tool": name, "ok": rec.ok, "empty": rec.empty, "latency_ms": rec.latency_ms, "error": rec.error or None}) @@ -343,6 +345,8 @@ async def _graph_query(self, ctx: PlanContext, args: Dict[str, Any]) -> Dict[str rows = await run_template(self.graph, template, place=str(args.get("place") or "").strip(), city=city, tag=str(args.get("tag") or "").strip()) except AppError as e: + log.warning("graph query failed", extra={"template": template, "code": e.code.value, + "detail": str(e.detail)[:300] if e.detail else None}) if e.code == ErrorCode.GRAPH_UNAVAILABLE: return degraded raise diff --git a/agent_service/app.py b/agent_service/app.py index ad21169..3407efb 100644 --- a/agent_service/app.py +++ b/agent_service/app.py @@ -12,7 +12,7 @@ from fastapi import FastAPI, Request from fastapi.exceptions import RequestValidationError -from fastapi.responses import JSONResponse, StreamingResponse +from fastapi.responses import JSONResponse, PlainTextResponse, StreamingResponse from pydantic import BaseModel, Field from starlette.concurrency import run_in_threadpool from starlette.exceptions import HTTPException as StarletteHTTPException @@ -28,6 +28,7 @@ from core.errors import AppError, ErrorCode, error_body from core.llm import LLMClient from core.logging import get_logger, request_id_var, setup_logging +from core.metrics import metrics from rag.ingest import load_csv_records from rag.pipeline import RagPipeline from rag.retriever import Retriever @@ -79,6 +80,10 @@ async def lifespan(_app: FastAPI): services.amap, services.memory) if not settings.amap_enabled: log.warning("AMAP_WEB_SERVICE_KEY is not set; /plan will fail") + metrics.register_gauge("caches", lambda: {"amap": services.amap.cache.stats(), + "retrieval": services.retriever.cache.stats()}) + metrics.register_gauge("breakers", lambda: [services.llm.breaker.snapshot(), services.graph.breaker.snapshot()]) + metrics.register_gauge("llm_concurrency", services.llm.concurrency) # 预加载 embedding 模型、向量库、BM25 索引和重排模型,避免首个请求因冷启动超时; # 失败不阻止启动,由 /readyz 反映,请求时会再次尝试加载 try: @@ -108,12 +113,16 @@ async def request_context(request: Request, call_next): token = request_id_var.set(rid) started = time.perf_counter() status = 500 + metrics.request_started() try: response = await call_next(request) status = response.status_code response.headers["X-Request-ID"] = rid return response finally: + # 按路由模板统计(/memory/{memory_id} 而不是每个 id 一行);未匹配的路径归为 unmatched + route = getattr(request.scope.get("route"), "path", "unmatched") + metrics.request_finished(request.method, route, status, (time.perf_counter() - started) * 1000) log.info( "access", extra={ @@ -137,6 +146,7 @@ def _error_response(code: ErrorCode, message: Optional[str] = None) -> JSONRespo @app.exception_handler(AppError) async def handle_app_error(_request: Request, exc: AppError): + metrics.error(exc.code.value) level = log.error if exc.code == ErrorCode.INTERNAL else log.warning level("request failed", extra={"code": exc.code.value, "detail": str(exc.detail)[:500] if exc.detail else None}) return _error_response(exc.code, exc.message) @@ -157,6 +167,7 @@ async def handle_http_error(_request: Request, exc: StarletteHTTPException): @app.exception_handler(Exception) async def handle_unexpected(_request: Request, exc: Exception): + metrics.error(ErrorCode.INTERNAL.value) log.exception("unhandled error") return _error_response(ErrorCode.INTERNAL) @@ -212,6 +223,14 @@ async def healthz(): return {"status": "ok"} +@app.get("/metrics") +async def get_metrics(format: str = "json"): + """运行指标。format=prometheus 时返回 Prometheus 文本格式,便于接入 Prometheus / Grafana。""" + if format == "prometheus": + return PlainTextResponse(metrics.prometheus(), media_type="text/plain; version=0.0.4") + return metrics.snapshot() + + @app.get("/readyz") async def readyz(): """就绪探针:逐项检查依赖。任一关键依赖不可用返回 503。""" diff --git a/agent_service/config.py b/agent_service/config.py index 2d8b57e..0432253 100644 --- a/agent_service/config.py +++ b/agent_service/config.py @@ -53,6 +53,8 @@ class RetrievalConfig(BaseModel): reranker_max_length: int = Field(256, ge=64, le=1024) city_filter: bool = True # 查询中识别到城市/省份时,只在相关笔记中检索 metadata_path: Path = Path("data/notes_metadata.jsonl") + cache_size: int = Field(1024, ge=0) # 检索结果缓存条数;0 关闭。重新入库时清空 + cache_ttl_s: int = Field(3600, ge=0) class AmapConfig(BaseModel): diff --git a/agent_service/core/amap.py b/agent_service/core/amap.py index fc0b1ed..20c07dc 100644 --- a/agent_service/core/amap.py +++ b/agent_service/core/amap.py @@ -6,12 +6,12 @@ """ import asyncio import time -from collections import OrderedDict from typing import Any, Dict, List, Optional, Tuple import httpx from config import Settings +from core.cache import TTLCache from core.errors import AppError, ErrorCode from core.logging import get_logger from core.resilience import retry_async @@ -44,33 +44,6 @@ async def wait(self) -> None: await asyncio.sleep(delay) -class TTLCache: - def __init__(self, maxsize: int, ttl_s: int): - self.maxsize = maxsize - self.ttl_s = ttl_s - self._data: "OrderedDict[Tuple, Tuple[float, Any]]" = OrderedDict() - self.hits = 0 - self.misses = 0 - - def get(self, key: Tuple) -> Optional[Any]: - item = self._data.get(key) - if item is None or (self.ttl_s and time.monotonic() - item[0] > self.ttl_s): - self._data.pop(key, None) - self.misses += 1 - return None - self._data.move_to_end(key) - self.hits += 1 - return item[1] - - def set(self, key: Tuple, value: Any) -> None: - if self.maxsize <= 0: - return - self._data[key] = (time.monotonic(), value) - self._data.move_to_end(key) - while len(self._data) > self.maxsize: - self._data.popitem(last=False) - - def parse_location(text: str) -> Optional[Dict[str, float]]: try: lng, lat = (float(x) for x in str(text).split(",")) diff --git a/agent_service/core/cache.py b/agent_service/core/cache.py new file mode 100644 index 0000000..4f71f31 --- /dev/null +++ b/agent_service/core/cache.py @@ -0,0 +1,44 @@ +"""线程安全的 LRU + TTL 缓存,带命中统计。高德请求与检索结果共用。""" +import threading +import time +from collections import OrderedDict +from typing import Any, Hashable, Optional, Tuple + + +class TTLCache: + def __init__(self, maxsize: int, ttl_s: float): + self.maxsize = maxsize + self.ttl_s = ttl_s + self._data: "OrderedDict[Hashable, Tuple[float, Any]]" = OrderedDict() + self._lock = threading.Lock() # 检索在线程池里执行,可能并发读写 + self.hits = 0 + self.misses = 0 + + def get(self, key: Hashable) -> Optional[Any]: + with self._lock: + item = self._data.get(key) + if item is None or (self.ttl_s and time.monotonic() - item[0] > self.ttl_s): + self._data.pop(key, None) + self.misses += 1 + return None + self._data.move_to_end(key) + self.hits += 1 + return item[1] + + def set(self, key: Hashable, value: Any) -> None: + if self.maxsize <= 0: + return + with self._lock: + self._data[key] = (time.monotonic(), value) + self._data.move_to_end(key) + while len(self._data) > self.maxsize: + self._data.popitem(last=False) + + def clear(self) -> None: + with self._lock: + self._data.clear() + + def stats(self) -> dict: + total = self.hits + self.misses + return {"size": len(self._data), "maxsize": self.maxsize, "hits": self.hits, "misses": self.misses, + "hit_rate": round(self.hits / total, 3) if total else None} diff --git a/agent_service/core/llm.py b/agent_service/core/llm.py index 1880978..39e49aa 100644 --- a/agent_service/core/llm.py +++ b/agent_service/core/llm.py @@ -12,7 +12,8 @@ from config import Settings from core.errors import AppError, ErrorCode from core.logging import get_logger -from core.resilience import CircuitBreaker, retry_async +from core.metrics import metrics +from core.resilience import CircuitBreaker, CircuitOpenError, retry_async log = get_logger("llm") @@ -40,6 +41,8 @@ def __init__(self, settings: Settings): self._endpoint = f"{self._cfg.base_url}/chat/completions" self._client = httpx.AsyncClient(timeout=httpx.Timeout(self._cfg.timeout_s, connect=10.0)) self._gate = asyncio.Semaphore(self._cfg.max_concurrency) + self._waiting = 0 # 等待并发闸门的调用数(排队长度),用于监控 + self._in_use = 0 # 进程内累计用量(评测时取前后差值即可得到单次请求的 token 消耗) self.usage_total = {"calls": 0, "prompt_tokens": 0, "completion_tokens": 0} self.breaker = CircuitBreaker( @@ -59,6 +62,9 @@ def default_model(self) -> str: async def aclose(self) -> None: await self._client.aclose() + def concurrency(self) -> Dict[str, int]: + return {"limit": self._cfg.max_concurrency, "in_use": self._in_use, "waiting": self._waiting} + async def chat( self, messages: List[Dict[str, Any]], @@ -108,17 +114,26 @@ async def complete( async def _call_once(self, payload: Dict[str, Any], purpose: str) -> Dict[str, Any]: if not self.breaker.allow(): - raise AppError(ErrorCode.UPSTREAM_LLM_UNAVAILABLE, detail="circuit open") + raise CircuitOpenError(ErrorCode.UPSTREAM_LLM_UNAVAILABLE, "llm") started = time.perf_counter() status: Optional[int] = None try: - async with self._gate: + self._waiting += 1 + try: + await self._gate.acquire() + finally: + self._waiting -= 1 + self._in_use += 1 + try: resp = await self._client.post( self._endpoint, headers={"Authorization": f"Bearer {self._api_key}"}, json=payload, ) + finally: + self._in_use -= 1 + self._gate.release() status = resp.status_code if status != 200: code = _classify_http_error(status, resp.text) @@ -158,6 +173,8 @@ def _finish(self, payload, purpose, started, status, code: Optional[ErrorCode], "latency_ms": round((time.perf_counter() - started) * 1000), "ok": code is None, } + metrics.llm_call(purpose, code is None, fields["latency_ms"], + int((usage or {}).get("prompt_tokens") or 0), int((usage or {}).get("completion_tokens") or 0)) if code is None: self.usage_total["calls"] += 1 self.usage_total["prompt_tokens"] += int((usage or {}).get("prompt_tokens") or 0) diff --git a/agent_service/core/metrics.py b/agent_service/core/metrics.py new file mode 100644 index 0000000..0072fac --- /dev/null +++ b/agent_service/core/metrics.py @@ -0,0 +1,164 @@ +"""进程内指标:请求量、延迟分位、错误码、LLM 调用与 token、缓存命中、熔断状态。 + +单进程服务,不引入 Prometheus 客户端库:计数直接累加,延迟保留每个路由最近 N 次用于计算分位数。 +GET /metrics 返回 JSON(监控页使用),GET /metrics?format=prometheus 返回 Prometheus 文本格式。 +""" +import threading +import time +from collections import Counter, defaultdict, deque +from typing import Callable, Deque, Dict, List, Optional + +WINDOW = 2000 # 每个序列保留最近多少次耗时 + + +def percentile(values: List[float], p: float) -> Optional[float]: + if not values: + return None + s = sorted(values) + k = min(len(s) - 1, max(0, round(p / 100 * (len(s) - 1)))) + return s[k] + + +def _summary(latencies: Deque[float]) -> Dict[str, Optional[float]]: + vals = list(latencies) + return {"p50": percentile(vals, 50), "p95": percentile(vals, 95), "p99": percentile(vals, 99), + "max": max(vals) if vals else None} + + +class Metrics: + def __init__(self) -> None: + self._lock = threading.Lock() + self.started_at = time.time() + self.in_flight = 0 + self.requests: Counter = Counter() # (method, route, status) -> 次数 + self.errors: Counter = Counter() # 错误码 -> 次数 + self.latency: Dict[str, Deque[float]] = defaultdict(lambda: deque(maxlen=WINDOW)) + self.llm_calls: Counter = Counter() # (purpose, ok) -> 次数 + self.llm_tokens: Counter = Counter() # prompt / completion + self.llm_latency: Dict[str, Deque[float]] = defaultdict(lambda: deque(maxlen=WINDOW)) + self.tool_calls: Counter = Counter() # (tool, ok) -> 次数 + # 由各组件注册的“现场读取”数据源:缓存命中、熔断状态等 + self._gauges: Dict[str, Callable[[], dict]] = {} + + # ---------- 记录 ---------- + def request_started(self) -> None: + with self._lock: + self.in_flight += 1 + + def request_finished(self, method: str, route: str, status: int, latency_ms: float, + error_code: Optional[str] = None) -> None: + with self._lock: + self.in_flight -= 1 + self.requests[(method, route, status)] += 1 + self.latency[f"{method} {route}"].append(latency_ms) + if error_code: + self.errors[error_code] += 1 + + def error(self, code: str) -> None: + with self._lock: + self.errors[code] += 1 + + def llm_call(self, purpose: str, ok: bool, latency_ms: float, prompt_tokens: int = 0, + completion_tokens: int = 0) -> None: + with self._lock: + self.llm_calls[(purpose, ok)] += 1 + self.llm_latency[purpose].append(latency_ms) + self.llm_tokens["prompt"] += prompt_tokens + self.llm_tokens["completion"] += completion_tokens + + def tool_call(self, tool: str, ok: bool) -> None: + with self._lock: + self.tool_calls[(tool, ok)] += 1 + + def register_gauge(self, name: str, fn: Callable[[], dict]) -> None: + self._gauges[name] = fn + + # ---------- 读取 ---------- + def snapshot(self) -> dict: + with self._lock: + routes: Dict[str, dict] = {} + for (method, route, status), n in self.requests.items(): + r = routes.setdefault(f"{method} {route}", {"count": 0, "errors_5xx": 0, "errors_4xx": 0}) + r["count"] += n + if status >= 500: + r["errors_5xx"] += n + elif status >= 400: + r["errors_4xx"] += n + for key, r in routes.items(): + r["latency_ms"] = _summary(self.latency[key]) + total = sum(self.requests.values()) + errors_5xx = sum(n for (_, _, s), n in self.requests.items() if s >= 500) + llm: Dict[str, dict] = {} + for (purpose, ok), n in self.llm_calls.items(): + d = llm.setdefault(purpose, {"ok": 0, "failed": 0}) + d["ok" if ok else "failed"] += n + for purpose, d in llm.items(): + d["latency_ms"] = _summary(self.llm_latency[purpose]) + tools: Dict[str, dict] = {} + for (tool, ok), n in self.tool_calls.items(): + d = tools.setdefault(tool, {"ok": 0, "failed": 0}) + d["ok" if ok else "failed"] += n + snap = { + "uptime_s": round(time.time() - self.started_at), + "in_flight": self.in_flight, + "requests_total": total, + "errors_5xx_total": errors_5xx, + "routes": dict(sorted(routes.items())), + "error_codes": dict(self.errors.most_common()), + "llm": {"by_purpose": dict(sorted(llm.items())), "tokens": dict(self.llm_tokens)}, + "tools": dict(sorted(tools.items())), + } + for name, fn in self._gauges.items(): + try: + snap[name] = fn() + except Exception as e: # 指标读取失败不能影响接口本身 + snap[name] = {"error": str(e)[:100]} + return snap + + def prometheus(self) -> str: + """Prometheus 文本格式。延迟以 summary 的分位数形式输出(窗口内估计,非累计直方图)。""" + lines: List[str] = [] + + def esc(v: str) -> str: + return str(v).replace("\\", "\\\\").replace('"', '\\"') + + snap = self.snapshot() + lines += ["# TYPE wanderai_up gauge", "wanderai_up 1", + "# TYPE wanderai_uptime_seconds gauge", f"wanderai_uptime_seconds {snap['uptime_s']}", + "# TYPE wanderai_in_flight_requests gauge", f"wanderai_in_flight_requests {snap['in_flight']}", + "# TYPE wanderai_http_requests_total counter"] + with self._lock: + items = list(self.requests.items()) + for (method, route, status), n in sorted(items): + lines.append(f'wanderai_http_requests_total{{method="{method}",route="{esc(route)}",status="{status}"}} {n}') + lines.append("# TYPE wanderai_http_latency_ms summary") + for key, r in snap["routes"].items(): + method, route = key.split(" ", 1) + for q in ("p50", "p95", "p99"): + v = r["latency_ms"][q] + if v is not None: + lines.append(f'wanderai_http_latency_ms{{method="{method}",route="{esc(route)}",' + f'quantile="0.{q[1:]}"}} {v:.1f}') + lines.append("# TYPE wanderai_errors_total counter") + for code, n in snap["error_codes"].items(): + lines.append(f'wanderai_errors_total{{code="{code}"}} {n}') + lines.append("# TYPE wanderai_llm_calls_total counter") + for purpose, d in snap["llm"]["by_purpose"].items(): + for outcome in ("ok", "failed"): + lines.append(f'wanderai_llm_calls_total{{purpose="{esc(purpose)}",outcome="{outcome}"}} {d[outcome]}') + lines.append("# TYPE wanderai_llm_tokens_total counter") + for kind, n in snap["llm"]["tokens"].items(): + lines.append(f'wanderai_llm_tokens_total{{kind="{kind}"}} {n}') + lines += ["# TYPE wanderai_cache_hits_total counter", "# TYPE wanderai_cache_misses_total counter"] + for name, cache in (snap.get("caches") or {}).items(): + if isinstance(cache, dict) and "hits" in cache: + lines.append(f'wanderai_cache_hits_total{{cache="{name}"}} {cache["hits"]}') + lines.append(f'wanderai_cache_misses_total{{cache="{name}"}} {cache["misses"]}') + lines.append("# TYPE wanderai_breaker_state gauge") # 0 关闭 / 1 半开 / 2 打开 + for b in snap.get("breakers") or []: + state = {"closed": 0, "half_open": 1, "open": 2}.get(b.get("state"), -1) + lines.append(f'wanderai_breaker_state{{name="{b.get("name")}"}} {state}') + return "\n".join(lines) + "\n" + + +metrics = Metrics() diff --git a/agent_service/core/resilience.py b/agent_service/core/resilience.py index d4507b2..5096335 100644 --- a/agent_service/core/resilience.py +++ b/agent_service/core/resilience.py @@ -15,6 +15,14 @@ _now = time.monotonic +class CircuitOpenError(AppError): + """熔断器打开时的拒绝。错误码沿用对应服务的“暂时不可用”(对用户而言可稍后重试), + 但重试逻辑必须立即放弃:冷却期内再试也一定被拒,退避等待只会拖慢失败(压测中实测多耗 1~1.5 秒)。""" + + def __init__(self, code: ErrorCode, breaker: str): + super().__init__(code, detail=f"circuit open: {breaker}") + + class CircuitBreaker: """连续失败达到阈值后断开,冷却期内直接拒绝;冷却结束放行一次试探请求(半开)。""" @@ -79,7 +87,7 @@ async def retry_async( try: return await fn() except AppError as e: - if not e.retryable or attempt == attempts: + if not e.retryable or attempt == attempts or isinstance(e, CircuitOpenError): raise delay = min(max_delay_s, base_delay_s * 2 ** (attempt - 1)) * random.uniform(0.5, 1.5) log.warning( diff --git a/agent_service/graph/client.py b/agent_service/graph/client.py index 6b4f70e..ead72e3 100644 --- a/agent_service/graph/client.py +++ b/agent_service/graph/client.py @@ -9,7 +9,7 @@ from config import Settings from core.errors import AppError, ErrorCode from core.logging import get_logger -from core.resilience import CircuitBreaker +from core.resilience import CircuitBreaker, CircuitOpenError log = get_logger("graph") @@ -34,7 +34,7 @@ async def run(self, query: str, params: Optional[Dict[str, Any]] = None, write: if not self.enabled: raise AppError(ErrorCode.GRAPH_UNAVAILABLE, detail="graph disabled or NEO4J_PASSWORD missing") if not self.breaker.allow(): - raise AppError(ErrorCode.GRAPH_UNAVAILABLE, detail="circuit open") + raise CircuitOpenError(ErrorCode.GRAPH_UNAVAILABLE, "graph") started = time.perf_counter() try: # 注意:execute_query 的未知关键字参数会被当成 Cypher 参数,读超时要放在 Query 对象上 @@ -45,7 +45,12 @@ async def run(self, query: str, params: Optional[Dict[str, Any]] = None, write: except (ServiceUnavailable, OSError, asyncio.TimeoutError) as e: self.breaker.record_failure() raise AppError(ErrorCode.GRAPH_UNAVAILABLE, detail=str(e)[:200]) from e - except Neo4jError as e: # 查询本身的错误(语法、约束),不代表服务挂了 + except Neo4jError as e: + # 服务端事务超时属于“暂时不可用”:降级并计入熔断;其余(语法、约束)是查询本身的错误,不代表服务挂了 + code = (e.code or "").lower() # 如 Neo.ClientError.Transaction.TransactionTimedOutClientConfiguration + if "timedout" in code or "timeout" in code: + self.breaker.record_failure() + raise AppError(ErrorCode.GRAPH_UNAVAILABLE, detail=f"query timeout: {str(e)[:200]}") from e raise AppError(ErrorCode.INTERNAL, "图谱查询失败", detail=str(e)[:300]) from e self.breaker.record_success() log.info("graph query", extra={"latency_ms": round((time.perf_counter() - started) * 1000), diff --git a/agent_service/rag/retriever.py b/agent_service/rag/retriever.py index f093ea5..ce5ee81 100644 --- a/agent_service/rag/retriever.py +++ b/agent_service/rag/retriever.py @@ -2,11 +2,13 @@ 全部为同步 CPU 操作,异步层通过线程池调用。每一步都可通过参数开关,供评测脚本做消融对比。 """ +import copy import threading import time from typing import Any, Dict, List, Optional, Set from config import RetrievalConfig +from core.cache import TTLCache from core.logging import get_logger from rag.bm25 import BM25Index, add_user_words, rrf_fuse from rag.metadata import PlaceMatcher, load_metadata, note_places @@ -26,11 +28,13 @@ def __init__(self, store: VectorStore, cfg: RetrievalConfig): self._rows_by_place: Dict[str, Set[str]] = {} self._bm25: Optional[BM25Index] = None self._lock = threading.Lock() + self.cache = TTLCache(cfg.cache_size, cfg.cache_ttl_s) # ---------- 构建 ---------- def load(self) -> None: """加载元数据、构建 BM25 索引;入库后需要重新调用。""" with self._lock: + self.cache.clear() # 语料或元数据变了,旧结果作废 self.metadata = load_metadata(self.cfg.metadata_path) self.matcher = PlaceMatcher.from_metadata(self.metadata) rows_by_place: Dict[str, Set[str]] = {} @@ -79,6 +83,17 @@ def retrieve( rerank = rerank or self.cfg.rerank city_filter = self.cfg.city_filter if city_filter is None else city_filter candidate_k = candidate_k or self.cfg.candidate_k + key = (query, top_k, mode, rerank, city_filter, candidate_k, self.cfg.rrf_k, self.cfg.rerank_k) + cached = self.cache.get(key) + if cached is not None: + # 返回副本:调用方会往 hit 里补字段,不能改到缓存里的对象 + return {**copy.deepcopy(cached), "timings_ms": {"cache_hit": 1}} + result = self._retrieve_uncached(query, top_k, mode, rerank, city_filter, candidate_k) + self.cache.set(key, copy.deepcopy(result)) + return result + + def _retrieve_uncached(self, query: str, top_k: int, mode: str, rerank: str, city_filter: bool, + candidate_k: int) -> Dict[str, Any]: timings: Dict[str, int] = {} places = self.matcher.find(query) if city_filter else [] diff --git a/agent_service/requirements-dev.txt b/agent_service/requirements-dev.txt index ee4ba01..766f89b 100644 --- a/agent_service/requirements-dev.txt +++ b/agent_service/requirements-dev.txt @@ -1,2 +1,2 @@ -pytest -pytest-asyncio +pytest==9.1.1 +pytest-asyncio==1.4.0 diff --git a/agent_service/requirements-train.txt b/agent_service/requirements-train.txt new file mode 100644 index 0000000..7b3d59f --- /dev/null +++ b/agent_service/requirements-train.txt @@ -0,0 +1,2 @@ +# 训练意图分类模型时才需要(python -m classifier.train.*) +scikit-learn==1.9.1 diff --git a/agent_service/requirements.txt b/agent_service/requirements.txt index 1b96a12..0c01a44 100644 --- a/agent_service/requirements.txt +++ b/agent_service/requirements.txt @@ -1,10 +1,14 @@ -fastapi -uvicorn -pandas -chromadb -sentence-transformers -httpx -pydantic-settings -pyyaml -jieba -rank_bm25 +# 运行依赖(版本与开发环境一致,Docker 镜像与 CI 用同一份) +# torch 不在这里:CPU 版从 PyTorch 官方源单独安装(见 Dockerfile),否则会拉下 GPU 版的数 GB 依赖 +fastapi==0.143.0 +uvicorn==0.54.0 +pandas==3.0.6 +chromadb==1.5.9 +sentence-transformers==6.1.0 +transformers==5.19.0 +httpx==0.28.1 +pydantic-settings==2.15.0 +PyYAML==6.0.3 +jieba==0.42.1 +rank-bm25==0.2.2 +neo4j==6.4.0 diff --git a/agent_service/tests/test_graph.py b/agent_service/tests/test_graph.py index 70b6445..33aad2d 100644 --- a/agent_service/tests/test_graph.py +++ b/agent_service/tests/test_graph.py @@ -111,3 +111,40 @@ async def go(): return e.code assert asyncio.run(go()) == ErrorCode.GRAPH_UNAVAILABLE assert not g.breaker.allow() # 失败计入熔断 + + +def test_graph_server_timeout_becomes_unavailable(): + """Neo4j 服务端事务超时(容器里实测出现过)应按“暂时不可用”降级,而不是报内部错误。""" + import asyncio + from types import SimpleNamespace + + from neo4j.exceptions import Neo4jError + + from core.errors import AppError, ErrorCode + from core.resilience import CircuitBreaker + from graph.client import GraphClient + + class TimeoutDriver: + async def execute_query(self, *a, **k): + raise Neo4jError._hydrate_neo4j(code="Neo.ClientError.Transaction.TransactionTimedOutClientConfiguration", + message="The transaction has been terminated") + + class SyntaxDriver: + async def execute_query(self, *a, **k): + raise Neo4jError._hydrate_neo4j(code="Neo.ClientError.Statement.SyntaxError", message="bad") + + def client(driver): + g = GraphClient.__new__(GraphClient) + g._cfg = SimpleNamespace(timeout_s=1) + g.enabled, g._driver = True, driver + g.breaker = CircuitBreaker("graph-test", failure_threshold=5, recovery_timeout_s=30) + return g + + async def code_of(g): + try: + await g.run("RETURN 1") + except AppError as e: + return e.code + + assert asyncio.run(code_of(client(TimeoutDriver()))) == ErrorCode.GRAPH_UNAVAILABLE + assert asyncio.run(code_of(client(SyntaxDriver()))) == ErrorCode.INTERNAL # 查询写错不算服务不可用 diff --git a/agent_service/tests/test_ops.py b/agent_service/tests/test_ops.py new file mode 100644 index 0000000..345671a --- /dev/null +++ b/agent_service/tests/test_ops.py @@ -0,0 +1,106 @@ +"""P8:指标、缓存、/metrics 接口。""" +import time + +from core.cache import TTLCache +from core.metrics import Metrics, percentile + + +def test_percentile_nearest_rank(): + vals = list(range(1, 101)) + assert percentile(vals, 50) in (50, 51) + assert percentile(vals, 95) in (95, 96) + assert percentile([], 95) is None + + +def test_ttl_cache_lru_ttl_and_stats(monkeypatch): + c = TTLCache(maxsize=2, ttl_s=10) + c.set("a", 1) + c.set("b", 2) + assert c.get("a") == 1 # a 变为最近使用 + c.set("c", 3) # 淘汰最久未用的 b + assert c.get("b") is None + assert c.stats()["hits"] == 1 and c.stats()["misses"] == 1 + now = time.monotonic() + monkeypatch.setattr("core.cache.time.monotonic", lambda: now + 11) + assert c.get("a") is None # 过期 + c.clear() + assert c.stats()["size"] == 0 + + +def test_metrics_snapshot_and_prometheus(): + m = Metrics() + for ms in (10, 20, 30): + m.request_started() + m.request_finished("POST", "/retrieve", 200, ms) + m.request_started() + m.request_finished("POST", "/chat", 503, 5, error_code="UPSTREAM_LLM_UNAVAILABLE") + m.llm_call("chat.tool_qa", True, 800, prompt_tokens=100, completion_tokens=20) + m.llm_call("chat.tool_qa", False, 30) + m.register_gauge("caches", lambda: {"retrieval": {"hits": 3, "misses": 1}}) + m.register_gauge("broken", lambda: 1 / 0) # 数据源出错不影响整体 + + snap = m.snapshot() + assert snap["requests_total"] == 4 and snap["errors_5xx_total"] == 1 and snap["in_flight"] == 0 + assert snap["routes"]["POST /retrieve"]["latency_ms"]["max"] == 30 + assert snap["error_codes"] == {"UPSTREAM_LLM_UNAVAILABLE": 1} + assert snap["llm"]["by_purpose"]["chat.tool_qa"] == {"ok": 1, "failed": 1, "latency_ms": snap["llm"]["by_purpose"]["chat.tool_qa"]["latency_ms"]} + assert snap["llm"]["tokens"] == {"prompt": 100, "completion": 20} + assert "error" in snap["broken"] + + text = m.prometheus() + assert 'wanderai_http_requests_total{method="POST",route="/chat",status="503"} 1' in text + assert 'wanderai_llm_tokens_total{kind="prompt"} 100' in text + assert 'wanderai_cache_hits_total{cache="retrieval"} 3' in text + + +def test_retriever_cache_returns_copies(monkeypatch): + from config import RetrievalConfig + from rag.retriever import Retriever + + calls = [] + + class FakeRetriever(Retriever): + ready = True + + def _retrieve_uncached(self, query, *args): + calls.append(query) + return {"hits": [{"chunk_id": "c1", "text": "t"}], "places": [], "filtered": False, "timings_ms": {}} + + r = FakeRetriever.__new__(FakeRetriever) + r.cfg = RetrievalConfig() + r.cache = TTLCache(16, 60) + first = r.retrieve("成都美食") + first["hits"][0]["mutated"] = True # 调用方修改结果不能污染缓存 + second = r.retrieve("成都美食") + assert calls == ["成都美食"] + assert "mutated" not in second["hits"][0] + assert second["timings_ms"] == {"cache_hit": 1} + r.retrieve("成都美食", top_k=3) # 参数不同 → 不命中 + assert len(calls) == 2 + + +def test_open_circuit_is_not_retried(monkeypatch): + """熔断打开时立即失败:压测发现拒绝被当成可重试错误,白白退避 1~1.5 秒。""" + import asyncio + + import pytest + + from core.errors import ErrorCode + from core.resilience import CircuitOpenError, retry_async + + slept = [] + + async def fake_sleep(s): + slept.append(s) + + monkeypatch.setattr("core.resilience._sleep", fake_sleep) + calls = {"n": 0} + + async def rejected(): + calls["n"] += 1 + raise CircuitOpenError(ErrorCode.UPSTREAM_LLM_UNAVAILABLE, "llm") + + with pytest.raises(CircuitOpenError) as ei: + asyncio.run(retry_async(rejected, attempts=3)) + assert ei.value.code == ErrorCode.UPSTREAM_LLM_UNAVAILABLE and ei.value.retryable # 对用户仍是“稍后再试” + assert calls["n"] == 1 and slept == [] diff --git a/config.yaml b/config.yaml index 1f15660..e0225f4 100644 --- a/config.yaml +++ b/config.yaml @@ -26,6 +26,8 @@ retrieval: reranker_max_length: 256 city_filter: true # 查询里出现城市/省份时只检索相关笔记 metadata_path: data/notes_metadata.jsonl + cache_size: 1024 # 检索结果缓存(相同查询与参数直接返回);0 关闭 + cache_ttl_s: 3600 amap: base_url: https://restapi.amap.com diff --git a/docker-compose.yml b/docker-compose.yml index 3eb3f9a..d0c9e4a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,8 @@ -# 本地依赖服务。当前只有 Neo4j(P7 知识图谱);P8 会把网关与 agent_service 也加进来。 -# 启动:docker compose up -d neo4j 浏览器控制台:http://localhost:7474 +# 一键启动:网关 + agent_service + Neo4j +# 1) cp .env.example .env,填好 LLM_API_KEY、AMAP_WEB_SERVICE_KEY、AMAP_JS_KEY、NEO4J_PASSWORD +# 2) 首次需要入库(向量库在 agent_service/chroma_db,见 README) +# 3) docker compose up -d --build 浏览器打开 http://localhost:3000 +# 只要图数据库:docker compose up -d neo4j services: neo4j: image: neo4j:5.26-community @@ -21,3 +24,52 @@ services: timeout: 5s retries: 12 restart: unless-stopped + + agent: + build: + context: . + dockerfile: agent_service/Dockerfile + image: wanderai-agent + container_name: wanderai-agent + # 不发布端口:只有网关能访问(同一 compose 网络内用服务名 agent:8002) + expose: + - "8002" + environment: + LLM_API_KEY: ${LLM_API_KEY:?请在 .env 中设置 LLM_API_KEY} + AMAP_WEB_SERVICE_KEY: ${AMAP_WEB_SERVICE_KEY:-} + NEO4J_PASSWORD: ${NEO4J_PASSWORD:-} + GRAPH__URI: bolt://neo4j:7687 + volumes: + - ./agent_service/chroma_db:/app/agent_service/chroma_db # 向量库 + - ./data:/app/data # 语料、元数据、记忆数据库 + - ./agent_service/models:/app/agent_service/models:ro # 本地分类模型(缺失时退回 LLM) + - ./logs:/app/logs + - ${HF_CACHE_DIR:-hf-cache}:/models/hf # embedding / 重排模型缓存,首次启动会下载 + depends_on: + neo4j: + condition: service_healthy # 图谱是可选依赖,但同时启动时等它就绪,避免启动初期的降级 + restart: unless-stopped + + gateway: + build: . + image: wanderai-gateway + container_name: wanderai-gateway + ports: + - "3000:3000" + environment: + AGENT_BASE: http://agent:8002 + AMAP_WEB_SERVICE_KEY: ${AMAP_WEB_SERVICE_KEY:-} + AMAP_JS_KEY: ${AMAP_JS_KEY:-} + AMAP_JS_SECURITY_CODE: ${AMAP_JS_SECURITY_CODE:-} + # 容器里请求来自 Docker 网桥而不是 127.0.0.1,监控页需要 ADMIN_TOKEN 才能访问 + ADMIN_TOKEN: ${ADMIN_TOKEN:-} + RATE_LIMIT_API_PER_MIN: ${RATE_LIMIT_API_PER_MIN:-120} + RATE_LIMIT_HEAVY_PER_MIN: ${RATE_LIMIT_HEAVY_PER_MIN:-10} + HEAVY_MAX_IN_FLIGHT: ${HEAVY_MAX_IN_FLIGHT:-8} + depends_on: + agent: + condition: service_healthy # agent 加载完模型、能响应后再启动网关 + restart: unless-stopped + +volumes: + hf-cache: diff --git a/docs/interview_notes.md b/docs/interview_notes.md index 0446f9f..6e2376e 100644 --- a/docs/interview_notes.md +++ b/docs/interview_notes.md @@ -2,7 +2,7 @@ > 用途:面试前快速回忆“做了什么、为什么这么做、遇到什么问题、数字从哪来”。每完成一个阶段追加一节。 > 配套文档:现状说明 `project_documentation.md`,路线图 `docs/upgrade_plan.md`,检索评测 `eval/rag_report.md`,Agent 评测 `eval/agent_report.md`,分类评测 `eval/classifier_report.md`,路由评测 `eval/router_report.md`、`eval/route_threshold_report.md`,记忆评测 `eval/memory_report.md`。 -> 最后更新:2026-10-13(P1~P7 完成) +> 最后更新:2026-10-13(P1~P8 完成) --- @@ -20,7 +20,7 @@ | 降级 | RAG 的改写、重排、回答任一步失败都不让请求失败,跳过的步骤写入响应 `degraded`;已用“LLM 地址指向死端口”的实例实测 | §1.4 | | 可观测性 | JSON 结构化日志;request-id 从网关贯穿到 Python 服务和每次 LLM 调用;`/healthz` 存活、`/readyz` 就绪探针 | `core/logging.py` | | 配置外置 | `config.yaml` + `.env` + 环境变量三层覆盖,pydantic-settings 启动时校验 | `config.py` | -| 测试 | 62 个 pytest 用例(图谱对齐规则、查询工具与降级、驱动超时转降级、工具标记清洗;记忆写入规则与改口、证据校验与敏感信息过滤、摘要折叠、追问改写、从对话推断城市、规划步数耗尽的兜底;动态路由分发、日志轮转被占用时的容错;意图分类的格式、指标、阈值、LLM 兜底、路由只用于聊天消息;熔断、重试、错误映射、配置、分块、BM25、RRF、地名识别、CSV 清洗、行程校验规则、Agent 循环与自纠、预算保护) | `agent_service/tests/` | +| 测试 | 67 个 pytest 用例 + 5 个网关 node:test 用例,GitHub Actions 在每次推送和 PR 上运行(指标与分位数、缓存淘汰与过期、熔断打开时不重试、限流令牌桶、并发闸门;图谱对齐规则、查询工具与降级、驱动超时转降级、工具标记清洗;记忆写入规则与改口、证据校验与敏感信息过滤、摘要折叠、追问改写、从对话推断城市、规划步数耗尽的兜底;动态路由分发、日志轮转被占用时的容错;意图分类的格式、指标、阈值、LLM 兜底、路由只用于聊天消息;熔断、重试、错误映射、配置、分块、BM25、RRF、地名识别、CSV 清洗、行程校验规则、Agent 循环与自纠、预算保护) | `agent_service/tests/` | | 安全 | 发现并修复网关把整个项目目录作为静态资源暴露的问题(配置、数据、日志、源码可直接下载);高德 Key 移到服务端代理 | §1.5 | | 检索架构 | 地名过滤 + BM25(jieba)与 bge 向量的 RRF 混合召回 + bge-reranker 交叉编码器重排 | `rag/retriever.py` | | 检索延迟(单机 CPU) | 混合 + 地名过滤 P95 36ms;加交叉编码器重排 P95 3.7s(因此只在问答页开重排,行程规划走 36ms 路径) | `eval/rag_report.md` | @@ -40,6 +40,11 @@ | 实体对齐 | 三轮人工抽检迭代对齐规则(每轮重新抽样):精确率 70% → 82%~88%(后两轮 60 条) | §7.4、`eval/graph_alignment_review.md` | | 图谱效果 | 16 个地点关系问题 × 3 次运行,对比只用高德周边搜索:回答中有游记依据的地点 2.2 → 5.3 个,延迟 7.6s → 5.1s(-33%),token -23% | §7.5、`eval/graph_report.md` | | 图谱容错 | 读超时 + 熔断 + 工具层降级到高德周边搜索;停掉 Neo4j 实测:单次 6 秒内失败,3 次后熔断直接降级,回答正常 | §7.6 | +| 限流与过载保护 | 网关按 IP 令牌桶(普通 120 次/分、重请求 10 次/分,429 + Retry-After)+ 重请求全局并发上限 8(503 繁忙);压测:40 个同时请求 8 个完成、32 个 0.2 秒内被拒,同 IP 15 次重请求 10 过 5 拒 | §8、`eval/load_test.md` | +| 监控 | 指标接口(JSON + Prometheus 文本):路由级请求数、4xx/5xx、P50/P95/P99,错误码、LLM 调用/token/并发排队、缓存命中、熔断状态;网关汇总 + 监控页,token 鉴权 | §8.2、`admin.html` | +| 压测 | 模拟 LLM(1 秒)下聊天并发 1→32 零失败,吞吐受 LLM 闸门约束在 7.4 次/秒、上游并发恒为 8;检索缓存命中 P50 7~9ms,吞吐约为未命中的 7.5 倍 | §8.4 | +| 熔断快速失败 | 压测发现熔断打开后仍需 1~1.8 秒(拒绝被退避重试),修复后 33~75ms 且不打上游 | §8.5 | +| 部署与 CI | Docker Compose 编排网关、agent_service、Neo4j(健康检查控制启动顺序,agent 不对外暴露、非 root 运行);GitHub Actions 跑 Python / 网关测试与镜像构建 | §8.2 | ### ⏳ 人工核验通过后才能写 @@ -62,7 +67,8 @@ - **“地理匹配率提升”——已测量,不成立**:这批 20 个需求上旧流程的地点匹配率和城市一致率本来就是 100%(三级地理编码 184/185 走的是“同城 POI”)。Agent 的收益在“路线可执行性”和“坐标可信(只能来自 POI 检索)”,不是匹配率。 - “意图分类准确率 XX%”——P4 数字待测试集人工盲审;而且路由模型**不如** LLM(0.954 vs 0.967),只能讲“以 1.3 个点的代价换 35 倍速度、零调用成本” - 意图分类“SFT 微调”——旧模型已弃用,新模型在 P4 -- “高并发”——还没做压测(P8) +- “高并发 / 支持 XX QPS”——压测是单机 + 模拟 LLM,只能写“压测验证限流、过载保护与熔断生效”和具体场景数字,不能写成线上承载能力 +- “K8s / 微服务集群部署”——只做了单机 Docker Compose - “图谱让回答质量提升”——LLM 盲评三次运行结果互相矛盾(5/4/7、4/8/4、6/4/6),16 个问题不足以证明;只能写“有依据的推荐更多、更快、更省” - “Text2Cypher / GraphRAG”——用的是固定模板,不是 LLM 生成 Cypher @@ -74,6 +80,7 @@ > - **动态路由**:聊天消息按 5 类意图分发到行程规划、游记 RAG、周边与路线工具、天气工具或直接回复,本地路由模型低置信度时交给 LLM;相比“每句话都重新规划”,平均延迟 14.5s → 5.3s、token 降低约 80%,非修改类消息不再误改行程(17/32 → 0/32)。 > - **长期记忆**:设计固定键的用户画像记忆与“证据必须出自原话”的写入校验,过滤敏感信息、支持待确认与改口覆盖,会话侧保留最近对话并滚动摘要、改写指代追问;6 个约束场景各跑 3 轮,规划违反用户约束的次数由 14/18 降至 3/18。 > - **知识图谱**:将游记地点对齐到高德 POI 并构建 Neo4j 图谱(465 个地点、1,813 条共现关系、848 条邻近关系),三轮人工抽检迭代对齐规则,精确率由 70% 提升至 82%~88%;以参数化 Cypher 模板作为 Agent 工具回答“附近 / 常一起去”类问题,相比仅用高德周边搜索,回答中有游记依据的推荐由 2.2 个增至 5.3 个,延迟降低 33%、token 降低 23%;图谱不可用时超时熔断并自动降级。 +> - **稳定性与运维**:网关实现按 IP 令牌桶限流与重请求过载保护,LLM 客户端并发闸门 + 熔断(修复熔断拒绝被退避重试的问题,失败响应 1.5s → 50ms 左右);提供 JSON / Prometheus 指标接口与监控页;以模拟 LLM 压测验证 40 并发突发下 8 个正常完成、其余 0.2 秒内返回繁忙提示;Docker Compose 一键部署,GitHub Actions 自动测试与镜像构建。 > - **意图识别**:定位旧分类模型“训练用游记长文、推理用表单短串”的分布错配,统一输入格式并重建数据(训练/测试分开生成、未见城市测试、去近重复);微调 RoBERTa 实现 6 类多标签风格识别与 5 类意图路由,与逻辑回归、bge-small、LLM 零样本对比,本地 CPU 推理 23~43ms,较 LLM 零样本(约 0.8~1s)快 20~35 倍。 --- @@ -665,7 +672,77 @@ --- -## 8. 演示脚本(面试现场或录屏) +## 8. P8 性能与运维 + +### 8.1 做之前的状态 + +- 只有 `/healthz`、`/readyz` 两个探针,看不到请求量、延迟分位、错误分布、LLM token 消耗;出了问题只能翻日志。 +- 网关不限流:任何人都能连续发规划请求,每次几十秒、几万 token,既花钱又会把服务拖垮。 +- 没有压测数据,“高并发”无从谈起。 +- 部署要手动装 Python / Node 环境、分别启动三个进程;没有 CI,改坏了只能靠自己记得跑测试。 +- 地图页 JS Key 写死在 HTML 里,进了 Git。 + +### 8.2 做了什么 + +| 模块 | 做法 | 关键代码 | +|---|---|---| +| 指标 | 进程内计数:每个路由(按模板,如 `/memory/{memory_id}`)的请求数、4xx/5xx、最近 2000 次的 P50/P95/P99;错误码分布;LLM 按用途的调用数、失败数、延迟、token;工具调用成败;缓存命中;熔断状态;LLM 并发闸门占用与排队 | `core/metrics.py` | +| 指标接口 | `GET /metrics` 返回 JSON;`?format=prometheus` 返回 Prometheus 文本格式,可直接接 Prometheus / Grafana | `app.py` | +| 网关汇总 | `/api/metrics` = 网关自身指标(含限流、过载拒绝数)+ agent_service 指标;有 `ADMIN_TOKEN` 时凭 token 访问(常量时间比较),否则只允许本机 | `server.js`、`gateway/ops.js` | +| 监控页 | `admin.html`:概览卡片(请求量、5xx 比例、处理中、限流 / 繁忙拒绝、LLM 调用 / token / 并发排队、熔断器)、依赖状态(就绪探针)、网关与 agent 两层的路由表、LLM 用途、工具、错误码、缓存与熔断;5 秒刷新,支持深色模式 | `admin.html` | +| 限流 | 网关按 IP 的令牌桶:普通接口 120 次/分钟,重请求(规划 / 聊天 / 问答)10 次/分钟;超限返回 429 + `Retry-After`;定期清理空闲桶防内存增长 | `gateway/ops.js` | +| 过载保护 | 重请求全局并发上限 8,超出立即返回 503 `SERVER_BUSY`(“当前使用人数较多”),SSE 断开时也释放名额 | 同上 | +| 缓存 | 检索结果 LRU + TTL 缓存(键包含查询和全部检索参数,重新入库时清空,返回深拷贝防止调用方改坏缓存);与高德缓存共用线程安全的 `TTLCache` | `core/cache.py`、`rag/retriever.py` | +| 压测 | 本地模拟 LLM(固定延迟、可切换故障),脚本自动起停三个进程、按场景切换网关配置,测检索、聊天容量、过载、限流、熔断 | `eval/load_test.py`、`eval/mock_llm.py` | +| Docker | 网关、agent_service 各一个镜像;Compose 编排三个服务,健康检查 + `depends_on: service_healthy` 控制启动顺序;agent 不发布端口,只在内部网络被网关访问;非 root 运行;依赖层与代码层分开缓存(只改代码时重新构建约 5 秒) | `Dockerfile`、`agent_service/Dockerfile`、`docker-compose.yml` | +| CI | GitHub Actions:Python 测试、网关测试、两个镜像构建;同一分支新推送取消旧任务;依赖与 Docker 层缓存 | `.github/workflows/ci.yml` | +| JS Key | 地图页从 `/api/config` 获取 Key 和安全密钥后动态加载 SDK,加载失败在地图区域直接提示 | `output-map.html`、`src/map.js` | + +### 8.3 关键设计决策(面试常问“为什么”) + +- **为什么自己写指标,不用 prometheus_client?** 单进程服务,需要的指标就十几个,自己写一百多行就够了,还能直接给监控页返回 JSON;同时输出 Prometheus 文本格式,将来要接 Prometheus 不用改代码。多进程部署时就该换成 prometheus_client 的多进程模式或推到外部存储。 +- **延迟为什么只保留最近 2000 次?** 算分位数要原始值,全量保存内存会一直涨;滑动窗口反映的是“最近”的情况,监控本来就更关心现在。代价是不是严格的累计分位数,Prometheus 输出里我用 summary 类型并在报告里说明了。 +- **为什么限流放在网关而不是 Python 服务?** 越早拒绝越便宜:网关拒绝一个请求不到 1 毫秒,而进到 Python 服务可能已经开始调 LLM。分两档是因为一次规划的成本是一次检索的上千倍。 +- **限流和过载保护有什么区别?** 限流按“每个用户”限制频率,防止一个人刷;过载保护按“全局同时在处理的数量”限制,防止大家一起来时所有请求都变慢、最后一起超时。压测里 40 个并发请求:8 个正常完成,32 个在 0.2 秒内被告知繁忙,而不是 40 个都等到超时。 +- **为什么用令牌桶而不是固定窗口计数?** 固定窗口在窗口边界可以打出两倍流量;令牌桶允许合理的突发(桶容量),长期速率受限,而且能精确算出 `Retry-After`。 +- **为什么压测用模拟 LLM?** 一是要测的是我们自己的系统(并发闸门、限流、熔断是否生效),DeepSeek 的延迟波动会把结果搅乱;二是真打 DeepSeek 压测要花钱,还可能被它限流。代价是数字不代表线上真实延迟,报告里写明了。 +- **单进程检索为什么并发 16 反而比并发 4 吞吐低?** 检索是 CPU 密集计算(embedding + BM25),在线程池里执行,Python GIL 和线程争抢让并发更高时只是排队更长。要提升需要多进程(uvicorn workers)或多实例,这时进程内缓存和限流也要换成 Redis。 +- **agent_service 为什么不对外暴露端口?** 所有外部请求必须经过网关的限流、request-id、统一错误处理;Python 服务的 `/ingest/csv` 等内部接口也不应该能被直接访问。 +- **监控接口为什么要鉴权?** 指标里有接口列表、错误分布、依赖状态,对攻击者是有用的信息;本地开发时只允许 127.0.0.1,容器里请求来自 Docker 网桥,所以必须设置 token。 + +### 8.4 压测结果(`eval/load_test.md`) + +单机(16 逻辑核),客户端与服务同机,LLM 为本地模拟服务。 + +| 场景 | 结果 | +|---|---| +| 检索(无重排),首次查询 | 并发 1:P50 23ms、34 次/秒;并发 4:66 次/秒;并发 16:55 次/秒、P95 373ms(CPU 瓶颈) | +| 检索,命中缓存 | P50 7~9ms;并发 4 时约 500 次/秒,约为未命中的 7.5 倍 | +| 聊天(模拟 LLM 1 秒) | 并发 1 → 32:吞吐 0.95 → 7.4 次/秒,上游最大并发始终为 8(LLM 闸门生效),无失败 | +| 过载保护 | 40 个同时请求:8 个完成,32 个返回 `SERVER_BUSY`,被拒绝的 P95 172ms | +| 限流 | 同一 IP 连续 15 个重请求:10 个成功,5 个 429,`Retry-After: 6` | +| 熔断 | 模拟 LLM 全部 500:2 个请求后熔断打开,之后的请求 33~75ms 失败、上游调用 0 次;恢复后半开探测成功、自动闭合 | + +### 8.5 遇到的问题(讲故事用) + +1. **压测发现熔断打开后仍然要 1~1.8 秒才失败**:熔断器拒绝时抛的是“服务暂时不可用(可重试)”,重试逻辑对它做了两次指数退避——冷却期内再试一定被拒,等待纯属浪费。修复:单独的 `CircuitOpenError` 类型,重试逻辑立即放弃,对用户的错误码和提示语不变。修复后 33~75ms。这正是压测的价值:单元测试分别测了重试和熔断,都是对的,组合在一起才暴露问题。 +2. **监控页第一次打开是空白的**:不是 bug,是截图时第一次请求还没返回;但也提醒我加载中要有状态(标题旁的“检查中”)。 +3. **地图 JS Key**:从 HTML 移到服务端下发后,Key 未配置时地图页原本只在控制台报一个未处理的 Promise 拒绝,用户看到的是空白地图。改成在地图区域直接显示原因。 +4. **容器里图谱查询偶发失败,日志却看不出原因**:用真实问题端到端测 Docker 部署时,一次图谱查询 3.6 秒后报“图谱查询失败”。超过了 3 秒的服务端事务超时,但这类 Neo4j 错误被归成“查询本身写错了”(内部错误),既不降级也不计入熔断,工具层也没把原因写进日志。修复:事务超时按“暂时不可用”处理(降级 + 熔断计数),工具失败时记录错误详情。第一次写判断条件用的是 `"Timeout" in code`,补测试时才发现 Neo4j 的错误码是 `TransactionTimedOut…`,根本匹配不上——测试先于线上发现了这个 bug。 +5. **依赖没写全**:`requirements.txt` 漏了 neo4j(P7 加的),本地环境装过所以没发现,写 Dockerfile 时才暴露。顺便把所有版本钉死,并把训练才用的 scikit-learn 拆到 `requirements-train.txt`。CI 正是为了防这种“我机器上能跑”的问题。 + +### 8.6 可能被问到的问题 + +- **你的系统能扛多少 QPS?** 要分接口说:检索单进程约 55~70 次/秒(CPU 瓶颈,缓存命中时几百次/秒);规划和聊天的瓶颈在 LLM,我们限制同时 8 个 LLM 调用,吞吐 ≈ 8 ÷ 单次 LLM 耗时,真实 DeepSeek 一次 1~3 秒,所以每秒几个对话。这个量级对演示项目足够;要更高需要多实例 + 共享限流 + 更高的 LLM 配额。 +- **为什么不直接把 LLM 并发调高?** DeepSeek 有自己的限流,调高只会收到更多 429;而且并发越高单个请求越慢。闸门的意义是把排队放在我们这边、可观测、可控。 +- **多实例部署时你的限流和缓存还有效吗?** 无效,都是进程内的。要换成 Redis(令牌桶用 Lua 脚本保证原子性),或者把限流放到 Nginx / API 网关层。 +- **P95 和平均值有什么区别,为什么看 P95?** 平均值会被大量快请求拉低,掩盖少数很慢的请求;P95 表示 95% 的请求比这个快,更接近“用户体验最差的那一批”。 +- **Docker 镜像多大、怎么优化?** agent_service 548MB、网关 267MB。见 8.2:先装依赖再拷代码利用层缓存;torch 用 CPU 版(GPU 版多几个 GB);`.dockerignore` 排除虚拟环境、模型、数据;模型和向量库通过卷挂载,不打进镜像。 +- **CI 跑什么?** Python 和网关单元测试、两个镜像能否构建;不跑需要 LLM 和高德的评测(要密钥、花钱、结果有波动),那些用 `eval/` 脚本手动跑。 + +--- + +## 9. 演示脚本(面试现场或录屏) ```powershell # 启动 diff --git a/docs/upgrade_plan.md b/docs/upgrade_plan.md index 924e3c7..791679c 100644 --- a/docs/upgrade_plan.md +++ b/docs/upgrade_plan.md @@ -612,7 +612,7 @@ Chroma 中按用途拆分 collection:`notes`(游记经验)、`poi_profiles | P5 ✅ | 动态路由 + 多知识库 | 0.5 周 | 路由准确率 | | P6 ✅ | Memory:抽取/存储/摘要/约束校验/偏好面板 | 1 周 | 约束遵守率 | | P7 ✅(规划聚类未做) | 知识图谱:抽取、对齐、Neo4j、Cypher 模板、规划聚类 | 1.5 周 | 图谱规模、对齐率、交通时长对比 | -| P8 | 性能与运维:缓存、限流、压测、监控页面、Docker | 1 周 | `eval/load_test.md`、`admin.html` | +| P8 ✅ | 性能与运维:缓存、限流、压测、监控页面、Docker | 1 周 | `eval/load_test.md`、`admin.html` | | 可选 | LoRA 槽位解析模型 | 1–2 周 | 槽位 F1 | 总计约 10–11 周。时间不够时的优先级:**P1 → P2 → P3 → P4**(覆盖简历四条主线并产出全部【待填】数字)> P6 > P5 > P8 > P7。 diff --git a/eval/data/load_test_results.json b/eval/data/load_test_results.json new file mode 100644 index 0000000..8cbca9c --- /dev/null +++ b/eval/data/load_test_results.json @@ -0,0 +1,394 @@ +{ + "retrieval": { + "cold": [ + { + "concurrency": 1, + "requests": 40, + "ok": 40, + "throughput_rps": 33.78, + "wall_s": 1.18, + "p50_ms": 23, + "p95_ms": 34, + "max_ms": 256, + "status": { + "200": 40 + }, + "routes": {} + }, + { + "concurrency": 4, + "requests": 40, + "ok": 40, + "throughput_rps": 66.67, + "wall_s": 0.6, + "p50_ms": 53, + "p95_ms": 86, + "max_ms": 102, + "status": { + "200": 40 + }, + "routes": {} + }, + { + "concurrency": 16, + "requests": 40, + "ok": 40, + "throughput_rps": 54.72, + "wall_s": 0.73, + "p50_ms": 267, + "p95_ms": 373, + "max_ms": 546, + "status": { + "200": 40 + }, + "routes": {} + } + ], + "warm": [ + { + "concurrency": 1, + "requests": 80, + "ok": 80, + "throughput_rps": 111.98, + "wall_s": 0.71, + "p50_ms": 9, + "p95_ms": 11, + "max_ms": 42, + "status": { + "200": 80 + }, + "routes": {} + }, + { + "concurrency": 4, + "requests": 80, + "ok": 80, + "throughput_rps": 499.56, + "wall_s": 0.16, + "p50_ms": 7, + "p95_ms": 11, + "max_ms": 16, + "status": { + "200": 80 + }, + "routes": {} + }, + { + "concurrency": 16, + "requests": 80, + "ok": 80, + "throughput_rps": 277.67, + "wall_s": 0.29, + "p50_ms": 36, + "p95_ms": 153, + "max_ms": 208, + "status": { + "200": 80 + }, + "routes": {} + } + ], + "cache": { + "size": 120, + "maxsize": 1024, + "hits": 240, + "misses": 120, + "hit_rate": 0.667 + } + }, + "chat": { + "rows": [ + { + "concurrency": 1, + "requests": 12, + "ok": 12, + "throughput_rps": 0.95, + "wall_s": 12.69, + "p50_ms": 1058, + "p95_ms": 1067, + "max_ms": 1094, + "status": { + "200": 12 + }, + "routes": { + "chitchat": 12 + }, + "llm_calls": 12, + "llm_max_in_flight": 1 + }, + { + "concurrency": 8, + "requests": 24, + "ok": 24, + "throughput_rps": 6.66, + "wall_s": 3.6, + "p50_ms": 1171, + "p95_ms": 1275, + "max_ms": 1276, + "status": { + "200": 24 + }, + "routes": { + "chitchat": 24 + }, + "llm_calls": 24, + "llm_max_in_flight": 8 + }, + { + "concurrency": 16, + "requests": 48, + "ok": 48, + "throughput_rps": 7.18, + "wall_s": 6.68, + "p50_ms": 2048, + "p95_ms": 2499, + "max_ms": 2515, + "status": { + "200": 48 + }, + "routes": { + "chitchat": 48 + }, + "llm_calls": 48, + "llm_max_in_flight": 8 + }, + { + "concurrency": 32, + "requests": 96, + "ok": 96, + "throughput_rps": 7.39, + "wall_s": 12.99, + "p50_ms": 4060, + "p95_ms": 4697, + "max_ms": 4848, + "status": { + "200": 96 + }, + "routes": { + "chitchat": 96 + }, + "llm_calls": 96, + "llm_max_in_flight": 8 + } + ], + "llm_concurrency_limit": 8 + }, + "overload": { + "summary": { + "concurrency": 40, + "requests": 40, + "ok": 8, + "throughput_rps": 3.4, + "wall_s": 2.35, + "p50_ms": 2345, + "p95_ms": 2349, + "max_ms": 2349, + "status": { + "200": 8, + "503 SERVER_BUSY": 32 + }, + "routes": { + "chitchat": 8 + } + }, + "rejected_p95_ms": 172 + }, + "rate_limit": { + "rows": [ + { + "i": 1, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 2, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 3, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 4, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 5, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 6, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 7, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 8, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 9, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 10, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 11, + "status": 429, + "code": "RATE_LIMITED", + "retry_after": "6" + }, + { + "i": 12, + "status": 429, + "code": "RATE_LIMITED", + "retry_after": "6" + }, + { + "i": 13, + "status": 429, + "code": "RATE_LIMITED", + "retry_after": "6" + }, + { + "i": 14, + "status": 429, + "code": "RATE_LIMITED", + "retry_after": "6" + }, + { + "i": 15, + "status": 429, + "code": "RATE_LIMITED", + "retry_after": "6" + } + ], + "counts": { + "200": 10, + "429 RATE_LIMITED": 5 + } + }, + "breaker": { + "timeline": [ + { + "i": 1, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 2051, + "upstream_calls": 3, + "breaker": "closed" + }, + { + "i": 2, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 1488, + "upstream_calls": 2, + "breaker": "open" + }, + { + "i": 3, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 75, + "upstream_calls": 0, + "breaker": "open" + }, + { + "i": 4, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 33, + "upstream_calls": 0, + "breaker": "open" + }, + { + "i": 5, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 37, + "upstream_calls": 0, + "breaker": "open" + }, + { + "i": 6, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 36, + "upstream_calls": 0, + "breaker": "half_open" + }, + { + "i": 7, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 746, + "upstream_calls": 1, + "breaker": "open" + }, + { + "i": 8, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 39, + "upstream_calls": 0, + "breaker": "open" + }, + { + "i": "恢复后 1", + "status": 200, + "code": "", + "ms": 112, + "upstream_calls": 1, + "breaker": "closed" + }, + { + "i": "恢复后 2", + "status": 200, + "code": "", + "ms": 159, + "upstream_calls": 1, + "breaker": "closed" + }, + { + "i": "恢复后 3", + "status": 200, + "code": "", + "ms": 116, + "upstream_calls": 1, + "breaker": "closed" + } + ] + }, + "env": { + "date": "2026-10-13", + "os": "Windows-11-10.0.26200-SP0", + "cpu": "Intel64 Family 6 Model 186 Stepping 2, GenuineIntel", + "cpu_count": 16, + "python": "3.13.7", + "node": "v22.18.0" + } +} \ No newline at end of file diff --git a/eval/data/load_test_results_v1.json b/eval/data/load_test_results_v1.json new file mode 100644 index 0000000..4a334aa --- /dev/null +++ b/eval/data/load_test_results_v1.json @@ -0,0 +1,394 @@ +{ + "retrieval": { + "cold": [ + { + "concurrency": 1, + "requests": 40, + "ok": 40, + "throughput_rps": 34.28, + "wall_s": 1.17, + "p50_ms": 22, + "p95_ms": 34, + "max_ms": 263, + "status": { + "200": 40 + }, + "routes": {} + }, + { + "concurrency": 4, + "requests": 40, + "ok": 40, + "throughput_rps": 68.74, + "wall_s": 0.58, + "p50_ms": 50, + "p95_ms": 99, + "max_ms": 138, + "status": { + "200": 40 + }, + "routes": {} + }, + { + "concurrency": 16, + "requests": 40, + "ok": 40, + "throughput_rps": 54.57, + "wall_s": 0.73, + "p50_ms": 244, + "p95_ms": 490, + "max_ms": 565, + "status": { + "200": 40 + }, + "routes": {} + } + ], + "warm": [ + { + "concurrency": 1, + "requests": 80, + "ok": 80, + "throughput_rps": 114.35, + "wall_s": 0.7, + "p50_ms": 9, + "p95_ms": 11, + "max_ms": 13, + "status": { + "200": 80 + }, + "routes": {} + }, + { + "concurrency": 4, + "requests": 80, + "ok": 80, + "throughput_rps": 442.86, + "wall_s": 0.18, + "p50_ms": 8, + "p95_ms": 16, + "max_ms": 25, + "status": { + "200": 80 + }, + "routes": {} + }, + { + "concurrency": 16, + "requests": 80, + "ok": 80, + "throughput_rps": 285.01, + "wall_s": 0.28, + "p50_ms": 44, + "p95_ms": 155, + "max_ms": 206, + "status": { + "200": 80 + }, + "routes": {} + } + ], + "cache": { + "size": 120, + "maxsize": 1024, + "hits": 240, + "misses": 120, + "hit_rate": 0.667 + } + }, + "chat": { + "rows": [ + { + "concurrency": 1, + "requests": 12, + "ok": 12, + "throughput_rps": 0.95, + "wall_s": 12.69, + "p50_ms": 1051, + "p95_ms": 1086, + "max_ms": 1100, + "status": { + "200": 12 + }, + "routes": { + "chitchat": 12 + }, + "llm_calls": 12, + "llm_max_in_flight": 1 + }, + { + "concurrency": 8, + "requests": 24, + "ok": 24, + "throughput_rps": 6.52, + "wall_s": 3.68, + "p50_ms": 1196, + "p95_ms": 1323, + "max_ms": 1323, + "status": { + "200": 24 + }, + "routes": { + "chitchat": 24 + }, + "llm_calls": 24, + "llm_max_in_flight": 8 + }, + { + "concurrency": 16, + "requests": 48, + "ok": 48, + "throughput_rps": 7.19, + "wall_s": 6.68, + "p50_ms": 2041, + "p95_ms": 2519, + "max_ms": 2559, + "status": { + "200": 48 + }, + "routes": { + "chitchat": 48 + }, + "llm_calls": 48, + "llm_max_in_flight": 8 + }, + { + "concurrency": 32, + "requests": 96, + "ok": 96, + "throughput_rps": 7.39, + "wall_s": 12.99, + "p50_ms": 4075, + "p95_ms": 4635, + "max_ms": 4775, + "status": { + "200": 96 + }, + "routes": { + "chitchat": 96 + }, + "llm_calls": 96, + "llm_max_in_flight": 8 + } + ], + "llm_concurrency_limit": 8 + }, + "overload": { + "summary": { + "concurrency": 40, + "requests": 40, + "ok": 8, + "throughput_rps": 3.37, + "wall_s": 2.38, + "p50_ms": 2366, + "p95_ms": 2371, + "max_ms": 2371, + "status": { + "200": 8, + "503 SERVER_BUSY": 32 + }, + "routes": { + "chitchat": 8 + } + }, + "rejected_p95_ms": 173 + }, + "rate_limit": { + "rows": [ + { + "i": 1, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 2, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 3, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 4, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 5, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 6, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 7, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 8, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 9, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 10, + "status": 200, + "code": "", + "retry_after": null + }, + { + "i": 11, + "status": 429, + "code": "RATE_LIMITED", + "retry_after": "6" + }, + { + "i": 12, + "status": 429, + "code": "RATE_LIMITED", + "retry_after": "6" + }, + { + "i": 13, + "status": 429, + "code": "RATE_LIMITED", + "retry_after": "6" + }, + { + "i": 14, + "status": 429, + "code": "RATE_LIMITED", + "retry_after": "6" + }, + { + "i": 15, + "status": 429, + "code": "RATE_LIMITED", + "retry_after": "6" + } + ], + "counts": { + "200": 10, + "429 RATE_LIMITED": 5 + } + }, + "breaker": { + "timeline": [ + { + "i": 1, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 1556, + "upstream_calls": 3, + "breaker": "closed" + }, + { + "i": 2, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 1099, + "upstream_calls": 2, + "breaker": "open" + }, + { + "i": 3, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 1531, + "upstream_calls": 0, + "breaker": "open" + }, + { + "i": 4, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 1793, + "upstream_calls": 1, + "breaker": "open" + }, + { + "i": 5, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 1012, + "upstream_calls": 0, + "breaker": "open" + }, + { + "i": 6, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 1684, + "upstream_calls": 0, + "breaker": "half_open" + }, + { + "i": 7, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 1592, + "upstream_calls": 1, + "breaker": "open" + }, + { + "i": 8, + "status": 503, + "code": "UPSTREAM_LLM_UNAVAILABLE", + "ms": 993, + "upstream_calls": 0, + "breaker": "open" + }, + { + "i": "恢复后 1", + "status": 200, + "code": "", + "ms": 114, + "upstream_calls": 1, + "breaker": "closed" + }, + { + "i": "恢复后 2", + "status": 200, + "code": "", + "ms": 114, + "upstream_calls": 1, + "breaker": "closed" + }, + { + "i": "恢复后 3", + "status": 200, + "code": "", + "ms": 110, + "upstream_calls": 1, + "breaker": "closed" + } + ] + }, + "env": { + "date": "2026-10-13", + "os": "Windows-11-10.0.26200-SP0", + "cpu": "Intel64 Family 6 Model 186 Stepping 2, GenuineIntel", + "cpu_count": 16, + "python": "3.13.7", + "node": "v22.18.0" + } +} \ No newline at end of file diff --git a/eval/load_test.md b/eval/load_test.md new file mode 100644 index 0000000..e6ee1a9 --- /dev/null +++ b/eval/load_test.md @@ -0,0 +1,81 @@ +# 压测报告 + +- 运行日期:2026-10-13;单机:Windows-11-10.0.26200-SP0,16 逻辑核;Python 3.13.7,Node v22.18.0 +- 压测客户端与服务在同一台机器上;LLM 为本地模拟服务(固定延迟),**不包含 DeepSeek 的真实延迟和限流** +- 链路:压测客户端 → Node 网关 → agent_service(单进程 uvicorn)→ 模拟 LLM +- 复现:`python eval/load_test.py`(原始数据 `eval/data/load_test_results.json`) + +## 1. 检索接口(本地 CPU:embedding + BM25 + RRF,不重排) + +**首次查询(全部是没查过的问题)** + +| 并发 | 请求 | 成功 | 吞吐 (次/秒) | P50 | P95 | 最大 | +|---|---|---|---|---|---|---| +| 1 | 40 | 40 | 33.78 | 23 ms | 34 ms | 256 ms | +| 4 | 40 | 40 | 66.67 | 53 ms | 86 ms | 102 ms | +| 16 | 40 | 40 | 54.72 | 267 ms | 373 ms | 546 ms | + +**重复查询(命中检索缓存)** + +| 并发 | 请求 | 成功 | 吞吐 (次/秒) | P50 | P95 | 最大 | +|---|---|---|---|---|---|---| +| 1 | 80 | 80 | 111.98 | 9 ms | 11 ms | 42 ms | +| 4 | 80 | 80 | 499.56 | 7 ms | 11 ms | 16 ms | +| 16 | 80 | 80 | 277.67 | 36 ms | 153 ms | 208 ms | + +检索缓存累计:命中 240、未命中 120。 + +## 2. 聊天接口(模拟 LLM 每次 1 秒) + +agent_service 的 LLM 并发上限为 8;网关不限流。 + +| 并发 | 请求 | 成功 | 吞吐 (次/秒) | P50 | P95 | 最大 | LLM 调用数 | 上游最大并发 | 路由 | +|---|---|---|---|---|---|---|---|---|---| +| 1 | 12 | 12 | 0.95 | 1058 ms | 1067 ms | 1094 ms | 12 | 1 | chitchat 12 | +| 8 | 24 | 24 | 6.66 | 1171 ms | 1275 ms | 1276 ms | 24 | 8 | chitchat 24 | +| 16 | 48 | 48 | 7.18 | 2048 ms | 2499 ms | 2515 ms | 48 | 8 | chitchat 48 | +| 32 | 96 | 96 | 7.39 | 4060 ms | 4697 ms | 4848 ms | 96 | 8 | chitchat 96 | + +## 3. 过载保护(同时 40 个聊天请求,模拟 LLM 2 秒,网关重请求并发上限 8) + +- 结果分布:{'200': 8, '503 SERVER_BUSY': 32} +- 被拒绝的请求 P95 返回时间:172 ms(立即告知“当前使用人数较多”,而不是排队到超时) +- 放行请求 P50 2345 ms、P95 2349 ms + +## 4. 限流(同一 IP 连续 15 个重请求,配额 10 次/分钟) + +- 结果分布:{'200': 10, '429 RATE_LIMITED': 5} +- 被限流的响应带 `Retry-After`:6s, 6s, 6s, 6s, 6s + +## 5. LLM 故障与熔断(模拟 LLM 全部返回 500,之后恢复) + +| 请求 | 状态 | 错误码 | 耗时 | 本次打到上游的次数 | 熔断器状态(请求后) | +|---|---|---|---|---|---| +| 1 | 503 | UPSTREAM_LLM_UNAVAILABLE | 2051 ms | 3 | closed | +| 2 | 503 | UPSTREAM_LLM_UNAVAILABLE | 1488 ms | 2 | open | +| 3 | 503 | UPSTREAM_LLM_UNAVAILABLE | 75 ms | 0 | open | +| 4 | 503 | UPSTREAM_LLM_UNAVAILABLE | 33 ms | 0 | open | +| 5 | 503 | UPSTREAM_LLM_UNAVAILABLE | 37 ms | 0 | open | +| 6 | 503 | UPSTREAM_LLM_UNAVAILABLE | 36 ms | 0 | half_open | +| 7 | 503 | UPSTREAM_LLM_UNAVAILABLE | 746 ms | 1 | open | +| 8 | 503 | UPSTREAM_LLM_UNAVAILABLE | 39 ms | 0 | open | +| 恢复后 1 | 200 | — | 112 ms | 1 | closed | +| 恢复后 2 | 200 | — | 159 ms | 1 | closed | +| 恢复后 3 | 200 | — | 116 ms | 1 | closed | + +### 修复记录:熔断打开后仍然很慢 + +第一次压测(`eval/load_test_v1.md`)中,熔断已经打开、上游调用次数为 0 的请求仍然要 **1.0~1.8 秒**才返回。 +原因:熔断器拒绝时抛出的错误码是“暂时不可用(可重试)”,重试逻辑对它做了两次指数退避(0.5s + 1s)。 +修复:熔断拒绝改为单独的 `CircuitOpenError`,重试逻辑遇到它立即放弃(对用户的提示语和错误码不变)。 + +| 熔断打开后的请求 | 修复前 | 修复后 | +|---|---|---| +| 耗时 | 993~1793 ms | 33~75 ms | +| 打到上游的次数 | 0 | 0 | + +## 说明 + +- 吞吐和延迟受本机 CPU 影响,换机器数值会变;这里关注的是**随并发变化的趋势**和**保护机制是否按设计生效**。 +- 聊天吞吐的上限由 LLM 并发闸门决定(约 并发上限 ÷ 单次 LLM 耗时),真实环境下还受 DeepSeek 自身限流影响。 +- 检索是 CPU 密集计算,在线程池中执行;单进程下并发提高后吞吐有限,扩容需要多进程 / 多实例。 diff --git a/eval/load_test.py b/eval/load_test.py new file mode 100644 index 0000000..a734fd3 --- /dev/null +++ b/eval/load_test.py @@ -0,0 +1,323 @@ +"""压测:网关 + agent_service 的吞吐、延迟、限流、过载保护与熔断。 + +LLM 换成本地模拟服务(eval/mock_llm.py,固定延迟),测的是本系统的并发与保护能力, +不包含 DeepSeek 的速度与费用。脚本自己启动 模拟 LLM / agent_service / 网关(独立端口与日志目录), +按场景重启网关以切换限流配置,结束后全部关闭。 + +python eval/load_test.py # 约 3~4 分钟,输出 eval/load_test.md +""" +import asyncio +import json +import os +import platform +import statistics +import subprocess +import sys +import time +from collections import Counter +from datetime import date +from pathlib import Path +from typing import Dict, List, Optional + +import httpx + +ROOT = Path(__file__).resolve().parent.parent +PY = sys.executable +MOCK_PORT, AGENT_PORT, GW_PORT = 8099, 8012, 3010 +MOCK = f"http://127.0.0.1:{MOCK_PORT}" +AGENT = f"http://127.0.0.1:{AGENT_PORT}" +GW = f"http://127.0.0.1:{GW_PORT}" +REPORT = ROOT / "eval" / "load_test.md" +RESULTS = ROOT / "eval" / "data" / "load_test_results.json" +LOG_DIR = ROOT / "logs" / "loadtest" + +CITIES = ["成都", "重庆", "西安", "广州", "厦门", "北京", "上海", "杭州", "南京", "长沙", "昆明", "大理", + "青岛", "苏州", "武汉", "贵阳", "桂林", "拉萨", "哈尔滨", "三亚"] +TOPICS = ["美食推荐", "两天怎么玩", "亲子景点", "夜景去哪看", "小众景点", "住哪里方便", "博物馆", "拍照打卡"] +CHAT_MESSAGES = ["你好", "谢谢你", "你好呀", "好的谢谢", "哈哈好的", "你是谁"] + + +# ---------------- 进程管理 ---------------- +def start(cmd: List[str], cwd: Path, env: Dict[str, str], name: str) -> subprocess.Popen: + LOG_DIR.mkdir(parents=True, exist_ok=True) + out = open(LOG_DIR / f"{name}.out.log", "w", encoding="utf-8") + return subprocess.Popen(cmd, cwd=cwd, env={**os.environ, **env}, stdout=out, stderr=subprocess.STDOUT) + + +def stop(p: Optional[subprocess.Popen]) -> None: + if p and p.poll() is None: + p.terminate() + try: + p.wait(timeout=10) + except subprocess.TimeoutExpired: + p.kill() + + +async def wait_ready(url: str, timeout_s: float = 180) -> None: + deadline = time.monotonic() + timeout_s + async with httpx.AsyncClient() as c: + while time.monotonic() < deadline: + try: + if (await c.get(url, timeout=2)).status_code == 200: + return + except httpx.HTTPError: + pass + await asyncio.sleep(1) + raise RuntimeError(f"{url} not ready after {timeout_s}s") + + +def start_gateway(**limits) -> subprocess.Popen: + env = {"PORT": str(GW_PORT), "AGENT_BASE": AGENT, + "RATE_LIMIT_API_PER_MIN": str(limits.get("api", 0)), + "RATE_LIMIT_HEAVY_PER_MIN": str(limits.get("heavy", 0)), + "HEAVY_MAX_IN_FLIGHT": str(limits.get("in_flight", 0))} + return start(["node", "server.js"], ROOT, env, "gateway") + + +# ---------------- 压测原语 ---------------- +async def one(client: httpx.AsyncClient, method: str, path: str, body: Optional[dict]) -> Dict: + t0 = time.perf_counter() + try: + r = await client.request(method, GW + path, json=body, timeout=120) + code = "" + if r.status_code >= 400: + try: + code = r.json()["error"]["code"] + except Exception: + code = "?" + route = "" + if r.status_code == 200 and path == "/api/chat": + route = r.json().get("route", "") + return {"status": r.status_code, "code": code, "ms": (time.perf_counter() - t0) * 1000, "route": route, + "retry_after": r.headers.get("retry-after")} + except httpx.HTTPError as e: + return {"status": 0, "code": type(e).__name__, "ms": (time.perf_counter() - t0) * 1000, "route": ""} + + +async def run_load(path: str, bodies: List[dict], concurrency: int) -> Dict: + """固定并发:concurrency 个 worker 依次领取请求,直到 bodies 用完。""" + queue = list(bodies) + results: List[Dict] = [] + async with httpx.AsyncClient(limits=httpx.Limits(max_connections=concurrency + 4)) as client: + async def worker(): + while queue: + results.append(await one(client, "POST", path, queue.pop())) + t0 = time.perf_counter() + await asyncio.gather(*[worker() for _ in range(concurrency)]) + wall = time.perf_counter() - t0 + return summarize(results, wall, concurrency) + + +def pct(values: List[float], p: float) -> float: + s = sorted(values) + return s[min(len(s) - 1, max(0, round(p / 100 * (len(s) - 1))))] if s else 0.0 + + +def summarize(results: List[Dict], wall: float, concurrency: int) -> Dict: + ok = [r for r in results if r["status"] == 200] + lat = [r["ms"] for r in ok] + return {"concurrency": concurrency, "requests": len(results), "ok": len(ok), + "throughput_rps": round(len(ok) / wall, 2) if wall else 0, "wall_s": round(wall, 2), + "p50_ms": round(pct(lat, 50)), "p95_ms": round(pct(lat, 95)), "max_ms": round(max(lat)) if lat else 0, + "status": dict(Counter(f"{r['status']} {r['code']}".strip() for r in results)), + "routes": dict(Counter(r["route"] for r in ok if r["route"]))} + + +async def mock(**mode) -> Dict: + async with httpx.AsyncClient() as c: + return (await c.post(MOCK + "/mode", json=mode)).json() + + +async def agent_metrics() -> Dict: + async with httpx.AsyncClient() as c: + return (await c.get(AGENT + "/metrics")).json() + + +# ---------------- 场景 ---------------- +async def scenario_retrieval() -> Dict: + """检索:纯本地 CPU 计算(embedding + BM25 + RRF,不重排)。先全是新查询,再重复查询看缓存。""" + unique = [f"{c}{t}" for t in TOPICS for c in CITIES] # 160 条互不相同 + out = {"cold": [], "warm": []} + i = 0 + for conc in (1, 4, 16): + n = 40 + bodies = [{"query": q, "top_k": 5, "use_rerank": False} for q in unique[i:i + n]] + i += n + out["cold"].append(await run_load("/api/rag/retrieve", bodies, conc)) + hot = unique[:20] # 已经查过 → 缓存命中 + for conc in (1, 4, 16): + bodies = [{"query": hot[k % 20], "top_k": 5, "use_rerank": False} for k in range(80)] + out["warm"].append(await run_load("/api/rag/retrieve", bodies, conc)) + out["cache"] = (await agent_metrics())["caches"]["retrieval"] + return out + + +async def scenario_chat_capacity() -> Dict: + """聊天(闲聊路由):模拟 LLM 每次 1 秒。网关不限流,看 LLM 并发闸门(8)如何决定吞吐。""" + await mock(delay=1.0, fail=False) + rows = [] + for conc in (1, 8, 16, 32): + await mock(reset_stats=True) + n = max(12, conc * 3) + bodies = [{"message": CHAT_MESSAGES[k % len(CHAT_MESSAGES)]} for k in range(n)] + row = await run_load("/api/chat", bodies, conc) + st = await mock() + row["llm_calls"] = st["calls"] + row["llm_max_in_flight"] = st["max_in_flight"] + rows.append(row) + return {"rows": rows, "llm_concurrency_limit": (await agent_metrics())["llm_concurrency"]["limit"]} + + +async def scenario_overload() -> Dict: + """过载保护:同时打进 40 个聊天请求,网关只放行 8 个,其余立即返回 SERVER_BUSY。""" + await mock(delay=2.0, fail=False) + bodies = [{"message": "你好"} for _ in range(40)] + async with httpx.AsyncClient(limits=httpx.Limits(max_connections=50)) as client: + t0 = time.perf_counter() + results = await asyncio.gather(*[one(client, "POST", "/api/chat", b) for b in bodies]) + wall = time.perf_counter() - t0 + rejected = [r["ms"] for r in results if r["code"] == "SERVER_BUSY"] + return {"summary": summarize(results, wall, 40), + "rejected_p95_ms": round(pct(rejected, 95)) if rejected else None} + + +async def scenario_rate_limit() -> Dict: + """限流:同一 IP 连续 15 个重请求,每分钟配额 10。""" + await mock(delay=0.05, fail=False) + rows = [] + async with httpx.AsyncClient() as client: + for k in range(15): + r = await one(client, "POST", "/api/chat", {"message": "你好"}) + rows.append({"i": k + 1, "status": r["status"], "code": r["code"], "retry_after": r["retry_after"]}) + return {"rows": rows, "counts": dict(Counter(f"{r['status']} {r['code']}".strip() for r in rows))} + + +async def scenario_breaker() -> Dict: + """LLM 故障:模拟 LLM 全部返回 500 → 重试 → 熔断后快速失败、不再打上游 → 恢复后半开探测成功。""" + await mock(delay=0.05, fail=True, reset_stats=True) + timeline = [] + async with httpx.AsyncClient() as client: + for k in range(8): + before = (await mock())["calls"] + r = await one(client, "POST", "/api/chat", {"message": "你好"}) + after = (await mock())["calls"] + m = await agent_metrics() + llm_state = next(b["state"] for b in m["breakers"] if b["name"] == "llm") + timeline.append({"i": k + 1, "status": r["status"], "code": r["code"], "ms": round(r["ms"]), + "upstream_calls": after - before, "breaker": llm_state}) + await mock(fail=False) + await asyncio.sleep(6) # 冷却时间配置为 5 秒 + for k in range(3): + before = (await mock())["calls"] + r = await one(client, "POST", "/api/chat", {"message": "你好"}) + after = (await mock())["calls"] + m = await agent_metrics() + llm_state = next(b["state"] for b in m["breakers"] if b["name"] == "llm") + timeline.append({"i": f"恢复后 {k + 1}", "status": r["status"], "code": r["code"], "ms": round(r["ms"]), + "upstream_calls": after - before, "breaker": llm_state}) + return {"timeline": timeline} + + +# ---------------- 主流程 ---------------- +async def main() -> None: + mock_p = agent_p = gw_p = None + results: Dict = {} + try: + mock_p = start([PY, str(ROOT / "eval" / "mock_llm.py"), "--port", str(MOCK_PORT)], ROOT, {}, "mock_llm") + agent_p = start([PY, "app.py"], ROOT / "agent_service", { + "SERVER__PORT": str(AGENT_PORT), "LLM__BASE_URL": MOCK, "LLM_API_KEY": "mock-key", + "LLM__BREAKER_RECOVERY_S": "5", "LOGGING__DIR": str(LOG_DIR), "LOGGING__LEVEL": "WARNING"}, "agent") + await wait_ready(MOCK + "/stats") + await wait_ready(AGENT + "/healthz") + + gw_p = start_gateway() # 容量测试:网关不限流 + await wait_ready(GW + "/api/readyz") + print("retrieval ...", flush=True) + results["retrieval"] = await scenario_retrieval() + print("chat capacity ...", flush=True) + results["chat"] = await scenario_chat_capacity() + stop(gw_p) + + gw_p = start_gateway(in_flight=8) + await wait_ready(GW + "/api/readyz") + print("overload ...", flush=True) + results["overload"] = await scenario_overload() + stop(gw_p) + + gw_p = start_gateway(heavy=10) + await wait_ready(GW + "/api/readyz") + print("rate limit ...", flush=True) + results["rate_limit"] = await scenario_rate_limit() + stop(gw_p) + + gw_p = start_gateway() + await wait_ready(GW + "/api/readyz") + print("breaker ...", flush=True) + results["breaker"] = await scenario_breaker() + finally: + for p in (gw_p, agent_p, mock_p): + stop(p) + results["env"] = {"date": date.today().isoformat(), "os": platform.platform(), "cpu": platform.processor(), + "cpu_count": os.cpu_count(), "python": platform.python_version(), + "node": subprocess.run(["node", "--version"], capture_output=True, text=True).stdout.strip()} + RESULTS.write_text(json.dumps(results, ensure_ascii=False, indent=1), encoding="utf-8") + write_report(results) + + +def write_report(res: Dict) -> None: + env = res["env"] + L = ["# 压测报告", "", + f"- 运行日期:{env['date']};单机:{env['os']},{env['cpu_count']} 逻辑核;Python {env['python']},Node {env['node']}", + "- 压测客户端与服务在同一台机器上;LLM 为本地模拟服务(固定延迟),**不包含 DeepSeek 的真实延迟和限流**", + "- 链路:压测客户端 → Node 网关 → agent_service(单进程 uvicorn)→ 模拟 LLM", + "- 复现:`python eval/load_test.py`(原始数据 `eval/data/load_test_results.json`)", ""] + + def table(rows, extra=()): + head = "| 并发 | 请求 | 成功 | 吞吐 (次/秒) | P50 | P95 | 最大 |" + "".join(f" {h} |" for h, _ in extra) + sep = "|---" * (7 + len(extra)) + "|" + out = [head, sep] + for r in rows: + out.append(f"| {r['concurrency']} | {r['requests']} | {r['ok']} | {r['throughput_rps']} | {r['p50_ms']} ms | " + f"{r['p95_ms']} ms | {r['max_ms']} ms |" + "".join(f" {fn(r)} |" for _, fn in extra)) + return out + + rt = res["retrieval"] + L += ["## 1. 检索接口(本地 CPU:embedding + BM25 + RRF,不重排)", "", + "**首次查询(全部是没查过的问题)**", ""] + table(rt["cold"]) + [ + "", "**重复查询(命中检索缓存)**", ""] + table(rt["warm"]) + [ + "", f"检索缓存累计:命中 {rt['cache']['hits']}、未命中 {rt['cache']['misses']}。", ""] + + ch = res["chat"] + L += ["## 2. 聊天接口(模拟 LLM 每次 1 秒)", "", + f"agent_service 的 LLM 并发上限为 {ch['llm_concurrency_limit']};网关不限流。", ""] + table( + ch["rows"], extra=[("LLM 调用数", lambda r: r["llm_calls"]), + ("上游最大并发", lambda r: r["llm_max_in_flight"]), + ("路由", lambda r: ", ".join(f"{k} {v}" for k, v in r["routes"].items()))]) + [""] + + ov = res["overload"] + s = ov["summary"] + L += ["## 3. 过载保护(同时 40 个聊天请求,模拟 LLM 2 秒,网关重请求并发上限 8)", "", + f"- 结果分布:{s['status']}", + f"- 被拒绝的请求 P95 返回时间:{ov['rejected_p95_ms']} ms(立即告知“当前使用人数较多”,而不是排队到超时)", + f"- 放行请求 P50 {s['p50_ms']} ms、P95 {s['p95_ms']} ms", ""] + + rl = res["rate_limit"] + L += ["## 4. 限流(同一 IP 连续 15 个重请求,配额 10 次/分钟)", "", + f"- 结果分布:{rl['counts']}", + "- 被限流的响应带 `Retry-After`:" + ", ".join(str(r["retry_after"]) + "s" for r in rl["rows"] if r["retry_after"]), ""] + + br = res["breaker"] + L += ["## 5. LLM 故障与熔断(模拟 LLM 全部返回 500,之后恢复)", "", + "| 请求 | 状态 | 错误码 | 耗时 | 本次打到上游的次数 | 熔断器状态(请求后) |", "|---|---|---|---|---|---|"] + for t in br["timeline"]: + L.append(f"| {t['i']} | {t['status']} | {t['code'] or '—'} | {t['ms']} ms | {t['upstream_calls']} | {t['breaker']} |") + L += ["", "## 说明", "", + "- 吞吐和延迟受本机 CPU 影响,换机器数值会变;这里关注的是**随并发变化的趋势**和**保护机制是否按设计生效**。", + "- 聊天吞吐的上限由 LLM 并发闸门决定(约 并发上限 ÷ 单次 LLM 耗时),真实环境下还受 DeepSeek 自身限流影响。", + "- 检索是 CPU 密集计算,在线程池中执行;单进程下并发提高后吞吐有限,扩容需要多进程 / 多实例。"] + REPORT.write_text("\n".join(L) + "\n", encoding="utf-8") + print(f"report -> {REPORT}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/eval/load_test_v1.md b/eval/load_test_v1.md new file mode 100644 index 0000000..eb9721a --- /dev/null +++ b/eval/load_test_v1.md @@ -0,0 +1,70 @@ +# 压测报告 + +- 运行日期:2026-10-13;单机:Windows-11-10.0.26200-SP0,16 逻辑核;Python 3.13.7,Node v22.18.0 +- 压测客户端与服务在同一台机器上;LLM 为本地模拟服务(固定延迟),**不包含 DeepSeek 的真实延迟和限流** +- 链路:压测客户端 → Node 网关 → agent_service(单进程 uvicorn)→ 模拟 LLM +- 复现:`python eval/load_test.py`(原始数据 `eval/data/load_test_results.json`) + +## 1. 检索接口(本地 CPU:embedding + BM25 + RRF,不重排) + +**首次查询(全部是没查过的问题)** + +| 并发 | 请求 | 成功 | 吞吐 (次/秒) | P50 | P95 | 最大 | +|---|---|---|---|---|---|---| +| 1 | 40 | 40 | 34.28 | 22 ms | 34 ms | 263 ms | +| 4 | 40 | 40 | 68.74 | 50 ms | 99 ms | 138 ms | +| 16 | 40 | 40 | 54.57 | 244 ms | 490 ms | 565 ms | + +**重复查询(命中检索缓存)** + +| 并发 | 请求 | 成功 | 吞吐 (次/秒) | P50 | P95 | 最大 | +|---|---|---|---|---|---|---| +| 1 | 80 | 80 | 114.35 | 9 ms | 11 ms | 13 ms | +| 4 | 80 | 80 | 442.86 | 8 ms | 16 ms | 25 ms | +| 16 | 80 | 80 | 285.01 | 44 ms | 155 ms | 206 ms | + +检索缓存累计:命中 240、未命中 120。 + +## 2. 聊天接口(模拟 LLM 每次 1 秒) + +agent_service 的 LLM 并发上限为 8;网关不限流。 + +| 并发 | 请求 | 成功 | 吞吐 (次/秒) | P50 | P95 | 最大 | LLM 调用数 | 上游最大并发 | 路由 | +|---|---|---|---|---|---|---|---|---|---| +| 1 | 12 | 12 | 0.95 | 1051 ms | 1086 ms | 1100 ms | 12 | 1 | chitchat 12 | +| 8 | 24 | 24 | 6.52 | 1196 ms | 1323 ms | 1323 ms | 24 | 8 | chitchat 24 | +| 16 | 48 | 48 | 7.19 | 2041 ms | 2519 ms | 2559 ms | 48 | 8 | chitchat 48 | +| 32 | 96 | 96 | 7.39 | 4075 ms | 4635 ms | 4775 ms | 96 | 8 | chitchat 96 | + +## 3. 过载保护(同时 40 个聊天请求,模拟 LLM 2 秒,网关重请求并发上限 8) + +- 结果分布:{'200': 8, '503 SERVER_BUSY': 32} +- 被拒绝的请求 P95 返回时间:173 ms(立即告知“当前使用人数较多”,而不是排队到超时) +- 放行请求 P50 2366 ms、P95 2371 ms + +## 4. 限流(同一 IP 连续 15 个重请求,配额 10 次/分钟) + +- 结果分布:{'200': 10, '429 RATE_LIMITED': 5} +- 被限流的响应带 `Retry-After`:6s, 6s, 6s, 6s, 6s + +## 5. LLM 故障与熔断(模拟 LLM 全部返回 500,之后恢复) + +| 请求 | 状态 | 错误码 | 耗时 | 本次打到上游的次数 | 熔断器状态(请求后) | +|---|---|---|---|---|---| +| 1 | 503 | UPSTREAM_LLM_UNAVAILABLE | 1556 ms | 3 | closed | +| 2 | 503 | UPSTREAM_LLM_UNAVAILABLE | 1099 ms | 2 | open | +| 3 | 503 | UPSTREAM_LLM_UNAVAILABLE | 1531 ms | 0 | open | +| 4 | 503 | UPSTREAM_LLM_UNAVAILABLE | 1793 ms | 1 | open | +| 5 | 503 | UPSTREAM_LLM_UNAVAILABLE | 1012 ms | 0 | open | +| 6 | 503 | UPSTREAM_LLM_UNAVAILABLE | 1684 ms | 0 | half_open | +| 7 | 503 | UPSTREAM_LLM_UNAVAILABLE | 1592 ms | 1 | open | +| 8 | 503 | UPSTREAM_LLM_UNAVAILABLE | 993 ms | 0 | open | +| 恢复后 1 | 200 | — | 114 ms | 1 | closed | +| 恢复后 2 | 200 | — | 114 ms | 1 | closed | +| 恢复后 3 | 200 | — | 110 ms | 1 | closed | + +## 说明 + +- 吞吐和延迟受本机 CPU 影响,换机器数值会变;这里关注的是**随并发变化的趋势**和**保护机制是否按设计生效**。 +- 聊天吞吐的上限由 LLM 并发闸门决定(约 并发上限 ÷ 单次 LLM 耗时),真实环境下还受 DeepSeek 自身限流影响。 +- 检索是 CPU 密集计算,在线程池中执行;单进程下并发提高后吞吐有限,扩容需要多进程 / 多实例。 diff --git a/eval/mock_llm.py b/eval/mock_llm.py new file mode 100644 index 0000000..06e0a57 --- /dev/null +++ b/eval/mock_llm.py @@ -0,0 +1,60 @@ +"""压测用的模拟 LLM:OpenAI 兼容 /chat/completions,固定延迟后返回一句话。 + +压测要测的是“我们系统”的并发与保护能力,不应把 DeepSeek 的速度和费用算进来; +延迟和故障模式可在运行中通过 POST /mode 切换(用于熔断测试)。 + +python eval/mock_llm.py --port 8099 --delay 1.0 +""" +import argparse +import asyncio + +import uvicorn +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette.routing import Route + +state = {"delay": 1.0, "fail": False, "calls": 0, "in_flight": 0, "max_in_flight": 0} + + +async def completions(request: Request): + await request.body() + state["calls"] += 1 + state["in_flight"] += 1 + state["max_in_flight"] = max(state["max_in_flight"], state["in_flight"]) + try: + await asyncio.sleep(state["delay"]) + if state["fail"]: + return JSONResponse({"error": {"message": "mock upstream failure"}}, status_code=500) + return JSONResponse({ + "choices": [{"message": {"role": "assistant", "content": "你好!想去哪里玩?可以告诉我目的地和天数。"}}], + "usage": {"prompt_tokens": 50, "completion_tokens": 20}, + }) + finally: + state["in_flight"] -= 1 + + +async def mode(request: Request): + body = await request.json() + for k in ("delay", "fail"): + if k in body: + state[k] = body[k] + if body.get("reset_stats"): + state.update(calls=0, max_in_flight=0) + return JSONResponse(state) + + +async def stats(_request: Request): + return JSONResponse(state) + + +app = Starlette(routes=[Route("/chat/completions", completions, methods=["POST"]), + Route("/mode", mode, methods=["POST"]), Route("/stats", stats)]) + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=8099) + ap.add_argument("--delay", type=float, default=1.0) + args = ap.parse_args() + state["delay"] = args.delay + uvicorn.run(app, host="127.0.0.1", port=args.port, log_level="warning") diff --git a/gateway/ops.js b/gateway/ops.js new file mode 100644 index 0000000..f1c85de --- /dev/null +++ b/gateway/ops.js @@ -0,0 +1,140 @@ +// 网关的运维组件:按 IP 的令牌桶限流、重请求并发闸门、进程内指标。 +// 单进程网关,状态放内存即可;多实例部署时限流需要换成 Redis 等共享存储。 + +// ---------------- 令牌桶限流 ---------------- +// 每个 key(IP + 档位)一个桶:容量 = 每分钟配额,按时间匀速补充。允许短时突发,长期速率受限。 +export class RateLimiter { + constructor(perMinute, { now = () => Date.now() } = {}) { + this.capacity = perMinute; + this.refillPerMs = perMinute / 60000; + this.now = now; + this.buckets = new Map(); + this.rejected = 0; + } + + // 返回 { ok, retryAfterS } + take(key) { + if (this.capacity <= 0) return { ok: true, retryAfterS: 0 }; // 0 表示不限流 + const t = this.now(); + let b = this.buckets.get(key); + if (!b) { + b = { tokens: this.capacity, at: t }; + this.buckets.set(key, b); + } + b.tokens = Math.min(this.capacity, b.tokens + (t - b.at) * this.refillPerMs); + b.at = t; + if (b.tokens >= 1) { + b.tokens -= 1; + return { ok: true, retryAfterS: 0 }; + } + this.rejected += 1; + return { ok: false, retryAfterS: Math.ceil((1 - b.tokens) / this.refillPerMs / 1000) }; + } + + // 清理已经补满的桶,避免 IP 越来越多导致内存增长 + sweep() { + const t = this.now(); + for (const [key, b] of this.buckets) { + if (b.tokens + (t - b.at) * this.refillPerMs >= this.capacity) this.buckets.delete(key); + } + } +} + +// ---------------- 并发闸门(过载保护) ---------------- +// 规划 / 聊天一次要几十秒、多次 LLM 调用。超过上限直接返回“繁忙”,而不是让所有请求一起变慢直到超时。 +export class ConcurrencyGate { + constructor(limit) { + this.limit = limit; + this.inFlight = 0; + this.rejected = 0; + } + + tryEnter() { + if (this.limit > 0 && this.inFlight >= this.limit) { + this.rejected += 1; + return null; + } + this.inFlight += 1; + let released = false; + return () => { // 幂等释放:finish 和 close 都可能触发 + if (!released) { + released = true; + this.inFlight -= 1; + } + }; + } +} + +// ---------------- 指标 ---------------- +const WINDOW = 2000; + +export function percentile(values, p) { + if (!values.length) return null; + const s = [...values].sort((a, b) => a - b); + const k = Math.min(s.length - 1, Math.max(0, Math.round((p / 100) * (s.length - 1)))); + return s[k]; +} + +export class GatewayMetrics { + constructor() { + this.startedAt = Date.now(); + this.inFlight = 0; + this.routes = new Map(); // "METHOD route" -> { count, errors4xx, errors5xx, latencies[] } + this.errorCodes = new Map(); + } + + started() { + this.inFlight += 1; + } + + finished(method, route, status, latencyMs, errorCode) { + this.inFlight -= 1; + const key = `${method} ${route}`; + let r = this.routes.get(key); + if (!r) { + r = { count: 0, errors4xx: 0, errors5xx: 0, latencies: [] }; + this.routes.set(key, r); + } + r.count += 1; + if (status >= 500) r.errors5xx += 1; + else if (status >= 400) r.errors4xx += 1; + r.latencies.push(latencyMs); + if (r.latencies.length > WINDOW) r.latencies.shift(); + if (errorCode) this.errorCodes.set(errorCode, (this.errorCodes.get(errorCode) || 0) + 1); + } + + snapshot(extra = {}) { + const routes = {}; + let total = 0; + let errors5xx = 0; + for (const [key, r] of [...this.routes].sort()) { + total += r.count; + errors5xx += r.errors5xx; + routes[key] = { + count: r.count, + errors_4xx: r.errors4xx, + errors_5xx: r.errors5xx, + latency_ms: { + p50: percentile(r.latencies, 50), + p95: percentile(r.latencies, 95), + p99: percentile(r.latencies, 99), + max: r.latencies.length ? Math.max(...r.latencies) : null + } + }; + } + return { + uptime_s: Math.round((Date.now() - this.startedAt) / 1000), + in_flight: this.inFlight, + requests_total: total, + errors_5xx_total: errors5xx, + routes, + error_codes: Object.fromEntries([...this.errorCodes].sort((a, b) => b[1] - a[1])), + ...extra + }; + } +} + +// 只认本机回环地址(含 IPv4 映射到 IPv6 的形式) +export function isLoopback(ip = '') { + return ip === '127.0.0.1' || ip === '::1' || ip === '::ffff:127.0.0.1'; +} diff --git a/gateway/ops.test.js b/gateway/ops.test.js new file mode 100644 index 0000000..9a806d5 --- /dev/null +++ b/gateway/ops.test.js @@ -0,0 +1,63 @@ +// 网关运维组件的单元测试:npm test +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { ConcurrencyGate, GatewayMetrics, RateLimiter, isLoopback, percentile } from './ops.js'; + +test('令牌桶:允许突发到容量,之后按速率补充,并给出 Retry-After', () => { + let now = 0; + const rl = new RateLimiter(10, { now: () => now }); + for (let i = 0; i < 10; i++) assert.equal(rl.take('ip').ok, true); + const denied = rl.take('ip'); + assert.equal(denied.ok, false); + assert.equal(denied.retryAfterS, 6); // 每分钟 10 个 → 6 秒补 1 个 + assert.equal(rl.take('other-ip').ok, true); // 不同客户端互不影响 + now += 6000; + assert.equal(rl.take('ip').ok, true); + assert.equal(rl.rejected, 1); +}); + +test('令牌桶:配额为 0 表示不限流;sweep 清理已补满的桶', () => { + assert.equal(new RateLimiter(0).take('x').ok, true); + let now = 0; + const rl = new RateLimiter(60, { now: () => now }); + rl.take('a'); + now += 60000; + rl.sweep(); + assert.equal(rl.buckets.size, 0); +}); + +test('并发闸门:超过上限拒绝,释放可重复调用但只生效一次', () => { + const gate = new ConcurrencyGate(2); + const r1 = gate.tryEnter(); + const r2 = gate.tryEnter(); + assert.equal(gate.tryEnter(), null); + assert.equal(gate.rejected, 1); + r1(); + r1(); // finish 和 close 都会触发释放 + assert.equal(gate.inFlight, 1); + assert.ok(gate.tryEnter()); + r2(); +}); + +test('指标:按路由统计次数、错误与分位数', () => { + const m = new GatewayMetrics(); + for (const ms of [10, 20, 30, 40]) { + m.started(); + m.finished('POST', '/api/chat', 200, ms); + } + m.started(); + m.finished('POST', '/api/chat', 503, 1, 'SERVER_BUSY'); + const s = m.snapshot({ extra: 1 }); + assert.equal(s.requests_total, 5); + assert.equal(s.errors_5xx_total, 1); + assert.equal(s.in_flight, 0); + assert.equal(s.routes['POST /api/chat'].latency_ms.max, 40); + assert.deepEqual(s.error_codes, { SERVER_BUSY: 1 }); + assert.equal(s.extra, 1); + assert.equal(percentile([], 50), null); +}); + +test('isLoopback 只认本机地址', () => { + assert.ok(isLoopback('127.0.0.1') && isLoopback('::1') && isLoopback('::ffff:127.0.0.1')); + assert.ok(!isLoopback('172.18.0.1') && !isLoopback('192.168.1.5')); +}); diff --git a/output-map.html b/output-map.html index 0978877..3d61767 100644 --- a/output-map.html +++ b/output-map.html @@ -295,7 +295,29 @@ } - +
@@ -343,7 +365,7 @@

操作提示