From 117def259cba3bc8afb3897d1a0f12ca2ebac432 Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 10:58:41 +0800 Subject: [PATCH 01/36] docs(design): add 0.10.0 persistence design (PG primary / SQLite fallback via sqlx Any) --- design/persistence.md | 376 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 376 insertions(+) create mode 100644 design/persistence.md diff --git a/design/persistence.md b/design/persistence.md new file mode 100644 index 0000000..f5fef46 --- /dev/null +++ b/design/persistence.md @@ -0,0 +1,376 @@ +# 持久化设计（0.10.0） + +> 状态：草案 v1（杜衡，2026-09-03） +> 范围：ReviewEngine 0.10.0 数据库持久化。本文只做设计，不含实现代码；SQL 为建表草案，可直接誊入 `migrations/`。 + +## 1. 目标与已拍板决策 + +以下决策已拍板，本文不重新论证，只在既有约束内做落地设计： + +1. **部署形态**：PG 为主、SQLite 兜底。有 `DATABASE_URL` 走 PostgreSQL；无则内嵌 SQLite，默认路径 `~/.config/review-engine/review.db`。 +2. **访问层**：sqlx 0.8，运行时 `Any` 池（features: `runtime-tokio`, `postgres`, `sqlite`, `migrate`, `chrono`, `uuid`, `json`）。 +3. **评论回流**：GitLab Note webhook 实时入库为主，评审前主动拉取 notes API 兜底。 +4. **配置入库**：git 平台 / LLM 实例配置从 `ui-state.toml` 搬进数据库，含一次性透明迁移。 + +0.10.0 要解决的具体问题： + +- 重启后评审历史丢失（TaskStore 纯内存）。 +- 配置持久化依赖单个 TOML 文件，无并发写保护、无历史语义。 +- MR 讨论（人类评论 + 历史评审结论）不进评审上下文，二次评审重复劳动。 +- LLM API key 目前明文落盘（`persist.rs:27-29` 明写的威胁模型例外），借入库一并收进加密边界。 + +## 2. 现状结论（已核实） + +### 2.1 任务存储 + +- `src/server/task_queue.rs:116` `TaskStore`：`HashMap` + broadcast SSE；`TaskEntry` 字段见 68-84 行（`task_id/state/created_at/started_at/completed_at/result/error/request/source_meta/progress/expert_name`）。 +- reaper：每 300 s 清 `completed_at` 超 30 分钟的条目（135-149 行），手动路径 `cleanup_expired()`（160-167 行）。 +- 状态机：`Pending → Running → (Completed | Failed)`，`Cancelled` 终态且 `update` 对其早退（256-261 行）。`retry` 仅允许 `Failed → Pending`（451-475 行）。 +- 结果以 `serde_json::Value`（序列化的 `ReviewOutput`，`src/models/finding.rs:143`）挂 `result`；`ReviewOutput` 含 `reports: Vec`、`aggregated: Option`、`consolidated: Option`。 + +### 2.2 API 投影 + +- `src/server/api/review/task.rs:24-46` `task_to_status`、48-104 `build_review_detail`、106-124 `build_review_list_item`：`TaskEntry → API 响应` 的唯一转换点，入库后改造面集中在这三个函数与 `handlers.rs` 的 `list_reviews`（298 行）/`get_review`（207 行）。 +- 分页参数结构 `ListParams` 已含 `status/page/per_page/q/project/repository/date_from/date_to`（task.rs:155-165），0.10.0 不需要新增参数，只需要换数据源。 +- **搭车 bug 确认**：task.rs:75 `raw_comment` 只取 `output.aggregated.markdown`；团队评审 `aggregated=None` 时详情「完整评论」tab 空态。可用的 fallback 是 `output.consolidated.assessment.tl_dr`（`src/models/mod.rs:102`，`ConsolidatedReport` 结构见 `src/team/lead_consolidator.rs:62-80`）。 + +### 2.3 配置持久化 + +- `src/server/api/config/persist.rs`：`UiStateFile` 四区段（`ui` 投影 / `llm: Vec` / `git_platforms: Vec` / `gitlab: PersistedGitlabConfig`，52-78 行）。`PUT /api/v1/config` 热生效 + 落盘；启动经同一 `apply_ui_config` 回放（366-375 行）。 +- **env/CLI 来源值永不落盘**：`UiStateEnvOverrides`（346-361 行）+ `from_applied` 的 `is_env_derived_llm` / `strip_env_value` 过滤（93-178 行）。此原则入库后必须原样保留——入库只是换 `save_ui_state` 的落点，过滤逻辑不动。 +- git 凭据已加密（`encrypt_ui_state`，231-242 行）；**LLM API key 明文**（`llm` 区段不在加密范围，27-29 行注释明写）。 + +### 2.4 加密边界 + +- `src/config/secrets.rs`：ChaCha20-Poly1305，`enc:` 前缀 + 配置目录 `secrets.key`（32 字节，0600，原子写）。`decrypt_secret` 对无 `enc:` 前缀的值透传（126-128 行），天然兼容遗留明文。 +- 入库后加解密仍只在持久化边界发生，密钥文件位置不变（沿用 `key_path_for`，40-45 行）。 + +### 2.5 Webhook + +- **更正任务描述的一处事实**：Note Hook 处理器已存在——`handle_note_hook`（`src/server/gitlab/hooks.rs:408`）目前用于 `/review`、`/describe` 命令触发评审，含 allowlist 门禁与 URL 重写。0.10.0 的新工作不是"新增 Note 事件类型"，而是**在既有处理器里加 note 入库**，并在评审 worker 侧消费。 +- notes API 能力已具备：`list_discussions`（`src/git_provider/gitlab/client.rs:587`）、`post_note`（595）、`get_current_user_id`（144，回流自噬过滤要用）。 + +### 2.6 其他挂点 + +- `AppState`（`src/server/state.rs:208`）已有 `Option>` 挂可插拔组件的先例（`task_store: Option>` 216 行、`feedback_store` 239 行）。DB handle 沿用同一模式：`pub db: Option>`。 +- `TaskStore::new()` 会被无 tokio runtime 的同步单测经 `AppState::new()` 触达（task_queue.rs:129-134 注释），DB 注入不能破坏这条路径——用 `Option` + setter，默认 `None` 即 0.9 行为。 +- `Cargo.toml` 当前无 sqlx 依赖；`async-trait`、`chrono`、`uuid`、`serde_json` 均已在依赖树中。 + +## 3. Schema 定稿 + +单目录 `migrations/`，首版一个文件 `0001_init.sql` 建全部 7 表。sqlx `migrate!()` 宏内嵌，`Migrator::run(&pool)` 启动时执行。 + +### 3.1 方言差异点（设计约束，先于 DDL） + +`Any` 池双后端共用同一套 SQL，以下约束逐条对应后面的 DDL 写法： + +| 主题 | PG | SQLite | 本文的取舍 | +|---|---|---|---| +| 占位符 | 原生 `$1..$n` | `?` | **统一写 `?`**。Any 驱动内部为 PG 做翻译；写 `$1` 在 SQLite 端直接报错。（落地验证点 A，见 §11） | +| upsert | `ON CONFLICT ... DO UPDATE/NOTHING` | 同语法（≥3.24） | 两端一致，直接用；sqlx 内置 libsqlite3 版本远高于此 | +| `RETURNING` | 支持 | ≥3.35 支持 | **一律不用**。主键全部由 Rust 侧生成（UUID v4），写后无需回读；避免 Any 下两端 decode 行为差异 | +| JSON 列 | 原生 JSONB | TEXT | **DDL 用 TEXT，绑定用 `String`**：store 层 `serde_json::to_string` 后按 TEXT 绑定，读出再 `from_str`。若声明 PG JSONB 列而 SQLite 是 TEXT，`serde_json::Value` 在 PG 端会按 JSONB 编码、绑到 TEXT 列报类型错——应用层序列化是唯一两头都稳的做法 | +| 布尔 | 原生 BOOL | 0/1 | DDL `BOOLEAN`，sqlx Any 的 `bool` 编解码两端兼容 | +| 时间戳 | TIMESTAMPTZ | 无原生类型（NUMERIC 亲和） | DDL `TIMESTAMP`；**值一律 Rust 侧 chrono 生成**，不写 `CURRENT_TIMESTAMP` 默认值，两端时间戳格式由应用层统一 | +| 模糊搜索 | `ILIKE` | `LIKE` 仅 ASCII 不敏感 | 统一 `LOWER(col) LIKE LOWER(?)`，行为两端一致 | +| 外键 | 默认启用 | 需 `PRAGMA foreign_keys=ON` | SQLite 连接串带 `?...` 参数或建池后执行 PRAGMA（见 §4.3） | +| 自增主键 | SERIAL/IDENTITY | AUTOINCREMENT | **都不用**：全部自然键/UUID 文本主键，绕开方言差异 | + +### 3.2 建表 SQL 草案（`migrations/0001_init.sql`） + +```sql +-- ── 评审任务（TaskEntry 的持久投影）── +CREATE TABLE reviews ( + task_id TEXT PRIMARY KEY, -- UUID v4, Rust 侧生成 + state TEXT NOT NULL, -- pending|running|completed|failed|cancelled + source_meta TEXT NOT NULL DEFAULT '{}', -- SourceMeta JSON + -- 从 source_meta 物化的过滤列：分页过滤要走索引，JSON 文本抽取两端写法不同， + -- 写穿时由 Rust 同步维护，读路径不碰 JSON 抽取函数。 + project TEXT, + repository TEXT, + request TEXT, -- 序列化 ReviewRequest（无凭据，见 task.rs:175-178） + result TEXT, -- ReviewOutput JSON + error TEXT, + progress INTEGER, -- 0-100，仅终态时快照；进行中的实时进度不入库 + created_at TIMESTAMP NOT NULL, + started_at TIMESTAMP, + completed_at TIMESTAMP +); +CREATE INDEX idx_reviews_created_at ON reviews (created_at DESC); +CREATE INDEX idx_reviews_state ON reviews (state); +CREATE INDEX idx_reviews_project ON reviews (project); + +-- ── 专家子报告（从 ReviewOutput.reports 拆行，便于按专家查询）── +CREATE TABLE expert_reports ( + task_id TEXT NOT NULL REFERENCES reviews(task_id) ON DELETE CASCADE, + expert_name TEXT NOT NULL, + report TEXT NOT NULL, -- ExpertReport JSON + duration_ms INTEGER, -- 首版可为 NULL：TaskEntry 目前不记 per-expert 耗时， + -- 需执行器补计时后再填充（见 §5.4 注意点） + created_at TIMESTAMP NOT NULL, + PRIMARY KEY (task_id, expert_name) +); + +-- ── MR 讨论（Note webhook + notes API 兜底共用的幂等存储）── +CREATE TABLE mr_discussions ( + platform TEXT NOT NULL, -- GitPlatformConfig.name（实例级隔离） + project TEXT NOT NULL, -- path_with_namespace + mr_iid BIGINT NOT NULL, + note_id BIGINT NOT NULL, + author TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL, + created_at TIMESTAMP NOT NULL, -- note 的创建时间，非入库时间 + ingested_at TIMESTAMP NOT NULL, -- 入库时间，排序兜底 + PRIMARY KEY (platform, project, mr_iid, note_id) -- 幂等键 +); +CREATE INDEX idx_mr_discussions_mr ON mr_discussions (platform, project, mr_iid, created_at); + +-- ── 注入上下文（支撑 LLM 前缀缓存复用）── +CREATE TABLE review_contexts ( + task_id TEXT NOT NULL REFERENCES reviews(task_id) ON DELETE CASCADE, + kind TEXT NOT NULL, -- 'mr_discussions' | 未来扩展 + content TEXT NOT NULL, -- 渲染后的上下文本（前缀稳定） + content_hash TEXT NOT NULL, -- sha256 hex；同 MR 二次评审 hash 相同即复用 + token_estimate INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMP NOT NULL, + PRIMARY KEY (task_id, kind) +); +CREATE INDEX idx_review_contexts_hash ON review_contexts (content_hash); + +-- ── git 平台实例（ui-state.toml 的 [[git_platforms]] 区段入库）── +CREATE TABLE git_platforms ( + id TEXT PRIMARY KEY, -- UUID v4；业务合并键仍是 name（与内存模型一致） + name TEXT NOT NULL UNIQUE, + type TEXT NOT NULL DEFAULT 'gitlab', + base_url TEXT NOT NULL DEFAULT '', + internal_base_url TEXT NOT NULL DEFAULT '', + token TEXT NOT NULL DEFAULT '', -- enc: 加密 + webhook_secret TEXT NOT NULL DEFAULT '', -- enc: 加密 + webhook_signing_secret TEXT NOT NULL DEFAULT '', -- enc: 加密 + enabled BOOLEAN NOT NULL DEFAULT TRUE, + raw TEXT NOT NULL DEFAULT '{}', -- 扩展兜底：allowed_projects 等未列化字段 + updated_at TIMESTAMP NOT NULL +); + +-- ── LLM 实例（[[llm]] 区段入库；api_key 顺带收进加密边界）── +CREATE TABLE llm_providers ( + id TEXT PRIMARY KEY, -- UUID v4 + provider TEXT NOT NULL, -- 对齐 LLMConfig.provider（brief 中的 "name"） + model TEXT NOT NULL DEFAULT '', + api_base TEXT NOT NULL DEFAULT '', + api_key TEXT NOT NULL DEFAULT '', -- enc: 加密（新增：0.9 明文落盘） + max_tokens INTEGER NOT NULL DEFAULT 4096, + temperature REAL NOT NULL DEFAULT 0.7, + raw TEXT NOT NULL DEFAULT '{}', -- 扩展兜底：disable_thinking 等 + updated_at TIMESTAMP NOT NULL +); +CREATE UNIQUE INDEX idx_llm_providers_provider ON llm_providers (provider); + +-- ── 应用设置（ui 投影 / legacy gitlab 字段 / rules / advanced 等）── +CREATE TABLE app_settings ( + key TEXT PRIMARY KEY, -- 如 'ui'、'gitlab'、'rules'、'advanced' + value TEXT NOT NULL, -- JSON + updated_at TIMESTAMP NOT NULL +); +``` + +说明： + +- legacy `gitlab` 三个凭据（`PersistedGitlabConfig`）进 `app_settings`（key=`gitlab`，值 JSON，三个字段均 `enc:`），不开新表——它是遗留域，未来会被 `git_platforms` 吸收。 +- `git_platforms.id` / `llm_providers.id` 用 UUID 而非自增，原因见 §3.1 自增主键行。 +- `reviews.request` 沿用现有约定：序列化的是无凭据 `ReviewRequest`，token 永不入库（task.rs:175-178 注释承诺的语义，入库后不变）。 + +## 4. Rust 抽象层设计 + +### 4.1 模块结构 + +``` +src/store/ + mod.rs — SqlxStore::connect(url) / ::connect_default()、方言探测、测试 helper（new_in_memory） + traits.rs — ReviewStore / ConfigStore / DiscussionStore 三个 trait + sqlx.rs — SqlxStore { pool: AnyPool } 及三个 trait 的实现；所有 SQL 集中在此文件 + rows.rs — 行结构 ⇄ 领域结构（TaskEntry/UiStateFile/…）的编解码；enc: 加解密边界在此 +migrations/ + 0001_init.sql +``` + +`src/lib.rs` 加 `pub mod store;`。`AppState` 加 `pub db: Option>`（沿用 `task_store` 的 Option 先例，state.rs:216）。 + +### 4.2 trait 取舍：三个域 trait，一个实现 + +**推荐**：`ReviewStore`（reviews / expert_reports / review_contexts）、`ConfigStore`（git_platforms / llm_providers / app_settings）、`DiscussionStore`（mr_discussions）三个 trait，由同一个 `SqlxStore` 实现，共享一个 `AnyPool` 和 §3.1 的方言封装。 + +理由： + +- 三类调用方天然不相交：task_queue 只碰评审域、config put/replay 只碰配置域、note hook / worker 只碰讨论域。按域拆分后每个调用方只见自己的方法面，单测 mock 面最小。 +- 一个 `SqlxStore` 实现三者，避免了"每表一个 Repo"的样板爆炸（7 表 7 trait 没有收益）。 +- 项目已有 `async-trait` 依赖（Cargo.toml:103），trait object 的装箱开销不在热路径上（热路径仍是内存 HashMap + SSE，见 §5）。 + +**否决的备选**： + +- **单一大 `Store` trait**：任何一域加方法都动全局接口；mock 一个域要实现全部方法，测试成本高。否决。 +- **不用 trait、调用方直接依赖具体 `SqlxStore`**：这是最简方案，差点入选。否决原因是两处调用方（`PUT /config` 持久化、note hook 幂等入库）的单测需要注入假实现来断言"写库被调用且内容正确"，若绑死具体类型就只能起真 DB。In-memory SQLite 能缓解但消不掉（连接池时序、加解密边界都要真跑），保留 trait 的成本很低。 +- **每表一个 Repo trait**：过度碎片化，否决。 + +### 4.3 `sqlx::Any` 双后端可行性 + +结论：可行，但必须守住 §3.1 的封装纪律。落地要点： + +- 连接：`AnyPoolOptions` + `sqlx::any::install_default_drivers()`；`DATABASE_URL` 存在且以 `postgres://`/`postgresql://` 开头 → PG；否则 SQLite。**`DATABASE_URL` 设置了但连接失败 → 启动显式报错退出，绝不静默落 SQLite**（数据写到意外的地方比启动失败更难收拾，见 §9）。 +- SQLite URL 组装：默认 `sqlite://{config_dir}/review.db?mode=rwc`，`config_dir` 沿用 `resolve_ui_state_path` 的同套解析（persist.rs:214-225）；建池后执行 `PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON; PRAGMA busy_timeout=5000;`。 +- 迁移：`sqlx::migrate!("./migrations")` + `Migrator::run(&pool)`。**AnyPool 上的 migrate 支持需要落地时先跑 smoke test 确认**（验证点 A）：建空库跑 `0001_init.sql`，再跑一次确认幂等跳过。 +- 测试 helper：`SqlxStore::new_in_memory()` 用 `sqlite::memory:` + `max_connections(1)`（连接池 >1 时每个连接是独立的内存库，这是 SQLite 内存模式的经典坑），供不写文件的单元测试使用。 + +## 5. TaskStore 写穿方案 + +### 5.1 原则 + +内存仍是热路径与 SSE 的唯一来源；DB 是历史的唯一来源。状态迁移**同步 await 写库**（保证重启恢复语义正确），写库失败不阻塞评审。 + +### 5.2 逐方法写穿点 + +`TaskStore` 新增 `db: Option>`（构造后 setter 注入，`None` 即 0.9 纯内存行为，同步单测路径不受影响）： + +| 方法（task_queue.rs 行号） | 写库动作 | 说明 | +|---|---|---| +| `create_with_request`（179） | INSERT reviews（state=pending） | | +| `start`（212） | UPDATE state=running, started_at | | +| `set_progress`（229） | **不写库** | 高频事件，进度对历史无价值；终态写时快照一次 `progress` 即可 | +| `fill_source_meta`（310） | UPDATE source_meta + 物化 project/repository | 每任务至多一次，值得写 | +| `update` 终态分支（249） | UPDATE state/result/error/completed_at/progress + 逐条 INSERT expert_reports | Cancelled 早退分支（259-261）不写 | +| `delete`（428，cancel 语义） | UPDATE state=cancelled, completed_at | | +| `retry`（451） | UPDATE state=pending, error=NULL, completed_at=NULL | | + +写库失败处理：记 `tracing::error!` + 继续。终态写失败做一次立即重试，仍失败则放过——历史页少一条可接受，评审本身不能死。 + +### 5.3 重启恢复语义 + +- 启动时（migrate 之后、HTTP 监听之前）执行：`UPDATE reviews SET state='failed', error='interrupted: server restarted', completed_at=? WHERE state IN ('pending','running')`。 +- **取舍：复用 `failed` 而非新增 `interrupted` 状态**。新增 `TaskState::Interrupted` 会涟漪到 `task_status_str`、SSE 事件映射、前端 StatusBadge 颜色表（design.md §6.1），而收益只是列表上一个标签差异；`error` 文案已能表达原因。前端若想区分，可读 `error` 前缀。 +- 中断任务**不自动重新入队**：自动重跑会消耗 LLM 配额且可能重复评论 MR；由用户在历史页手工 retry（`retry` 允许 `Failed → Pending`，interrupted 落库为 failed，天然可 retry）。 +- 终态从库读：历史列表/详情直接查 DB（写穿保证 DB 含进行中任务），内存从空启动，无需回填。 + +### 5.4 reaper 与持久化的关系 + +- 30 分钟 reaper **保持原样、只清内存**（task_queue.rs:135-149），不删库。队列监控/SSE 的视图不变。 +- 注意点：`expert_reports.duration_ms` 首版允许 NULL——`TaskEntry` 不记 per-expert 耗时，执行器补计时是独立小改动，不阻塞本方案。 + +## 6. 配置迁移方案 + +### 6.1 启动序列（严格按序） + +1. 解析 DB URL（§4.3）→ 建池 → `Migrator::run` → **失败即退出非零**（§9）。 +2. 恢复语义扫尾（§5.3 的 interrupted UPDATE）。 +3. **一次性导入**：`git_platforms`、`llm_providers`、`app_settings` 三表合计为空 且 `ui-state.toml` 存在 → 走现有 `load_ui_state`（persist.rs:310，含解密）读入 → 经 `rows.rs` 加密边界写库（git 凭据 + LLM key 全部 `enc:`）→ `std::fs::rename("ui-state.toml", "ui-state.toml.migrated")`。**备份不删**。 + - 导入失败：记 error、**不改名原文件**、回退到现有 `load_and_apply_ui_state` 文件回放路径继续启动——迁移失败不能让用户丢配置。 + - 导入成功的判定要保守：三表全部写入完成才 rename；任何一步失败整体回滚（单事务包裹整个导入）。 +4. 之后经同一 `apply_ui_config` 路径从 DB 回放（替换 `load_and_apply_ui_state` 的数据源，回放逻辑本身不动——热/冷语义一致性是现有设计的优点，保留）。 + +### 6.2 PUT /config 语义 + +- 热生效路径（`apply_ui_config`）完全不变。 +- 持久化落点从 `save_ui_state`（文件）换成 `ConfigStore::save_*`（库）。**`UiStateFile::from_applied` 的 env 过滤逻辑原样复用**（persist.rs:93-178）：env/CLI 来源值永不入库，与永不落盘同一原则。 +- 落库失败：返回 500（与今天 `save_ui_state` 失败一致），不静默吞掉。 +- `secrets.key` 位置不变（配置目录下）；`rows.rs` 用 `load_or_create_key` 拿同一把钥匙。PG 部署时 key 文件仍在 server 本地配置目录——这是本地对称加密的既有威胁模型，本文不扩大也不缩小它。 + +### 6.3 优先级矩阵（逐配置域） + +| 配置域 | config.toml | DB（ui-state 迁入） | env/CLI | +|---|---|---|---| +| legacy gitlab 凭据（token/webhook_secret/signing_secret） | 仅作初始种子 | **权威源**；空时才用 env 兜底并记 deprecation warn | fallback-only（语义同今天，persist.rs:390-436） | +| LLM provider 列表 | 初始种子 | 覆盖 config.toml | **整体胜出**（`llm_from_env` 时 DB 的 llm 区段不回放，同 persist.rs:443-472） | +| git_platforms | 无来源（待核实：`config/resolver/` 是否承载 platforms，实现前确认） | **唯一权威** | 无来源 | +| ui 投影（rules / advanced / URL / 模型选择） | 初始种子 | 回放覆盖种子 | 无 | + +迁移完成后 `UiStateEnvOverrides` 机制保留原名原义，只是过滤的落点从文件换成库。 + +## 7. Note webhook 入库 + 评审前注入 + +### 7.1 入库（挂在既有 `handle_note_hook`，hooks.rs:408） + +在解析之后、命令判断之前插入入库逻辑（命令评论也是讨论历史的一部分，同样入库）： + +- **payload 关键字段**：`object_kind`（须为 `"note"`）、`object_attributes.id`（note_id）、`object_attributes.noteable_type`（须为 `"MergeRequest"`，Commit/Issue/Snippet note 忽略）、`object_attributes.note`（body）、`object_attributes.created_at`、`user.username`/`user.name`（author）、`merge_request.iid`（缺失时回退 `object_attributes.url` 尾部解析，复用 `mr_iid_from_url`，hooks.rs:394）、`project.path_with_namespace`。platform 取匹配到的 `GitPlatformConfig.name`，未匹配用 `"default"`。 +- **幂等**：主键 `(platform, project, mr_iid, note_id)`，`ON CONFLICT DO UPDATE SET body=excluded.body, author=excluded.author`——webhook 重投自然去重，note 被编辑则更新。 +- **回流自噬防护（必须做）**：本服务自己 `post_note`/`post_comment` 发的评审报告也会触发 Note hook。不入库规则：(a) `object_attributes.note` 以本服务报告固定前缀开头；(b) 或 author id 等于 `get_current_user_id()`（client.rs:144）的结果（启动时解析一次并缓存）。两条件任一命中即跳过入库（命令触发的 `/review` note 除外——那是用户意图）。实现时确认 (a) 的报告前缀常量位置。 +- 系统 note（`object_attributes.system=true`，如 "added 1 commit"）**入库但打标意义不大**——按已拍板 schema 无 `system` 列，决策：**跳过 system note**，它们是噪音不是讨论。 + +### 7.2 评审前注入（worker 侧） + +在评审流水线取 diff 之后、专家执行之前（`resolve.rs` / `run_review_common` 路径）： + +1. 按 `(platform, project, mr_iid)` 查 `mr_discussions`。 +2. **兜底**：查询结果为空（或该 MR 从未见过）→ 调 notes API（`list_discussions`，client.rs:587）拉全量 → upsert 入库 → 用拉取结果。 +3. **组织成追加式上下文**：按 `(created_at, note_id)` 升序渲染确定性模板，固定头部（如 `## MR Discussion History`）+ 逐条 `- [author @ created_at]: body`；body 截断上限（建议 2000 字符/条）防爆 context。**前缀稳定是硬要求**：同一 MR 历史不变时渲染输出逐字节相同，`content_hash` 相同 → LLM 前缀缓存命中；新评论只追加在尾部。 +4. 渲染结果 + hash 写入 `review_contexts`（`ON CONFLICT (task_id, kind) DO UPDATE`）；hash 相同的后续任务可直接复用渲染文本。 +5. **降级**：DB 不可用、notes API 失败、渲染超限——全部只记 warn，评审继续，不带讨论上下文。评论注入是增强，不是评审的前置条件。 + +## 8. API / 前端影响面 + +### 8.1 后端 + +- `list_reviews`（handlers.rs:298）：数据源从 `TaskStore.list`（内存）换为 DB 查询。**参数与响应 shape 不变**（`ListParams` 已齐，task.rs:155-165）：`page` 默认 1、`per_page` 默认 20、上限 100；`q` 用 `LOWER(source_meta) LIKE LOWER(?)`（§3.1）；`project`/`repository` 走物化列等值；`date_from/to` 走 `created_at` 范围；`COUNT(*)` 出 total。进行中任务 DB 已有（写穿），无需内存合并。 +- `get_review`（handlers.rs:207)：改读 DB；若该 task_id 恰在内存中（进行中），叠加实时 `progress`/`expert_name` 两个字段后返回。`task_to_status`/`build_review_detail`/`build_review_list_item` 三个投影函数改为接受"DB 行结构"，签名变化收敛在 `api/review/task.rs` 一个文件。 +- 新增（如评审前注入需要暴露）：无。Note 数据 0.10.0 不开查询 API。 + +### 8.2 前端 + +- 历史页（`/history`，design.md §2）：沿用服务端分页 + `ElPagination`（total 已有），每页 20。首版不做无限滚动。 +- 若要滚动加载（可选增强）：IntersectionObserver 哨兵 div + page 累加 append 到列表；filter/q 变化时重置 page=1 并清空已加载；SSE 的 `review.completed` 事件触发第一页刷新而非整表重载（配合 design.md §7.5 的 flash-border）。 +- 详情页：无结构变化（字段不变），但历史记录现在重启后仍在，注意加载态/404 处理走既有约定（design.md §10）。 + +### 8.3 搭车修复：团队评审详情空态 + +`build_review_detail`（task.rs:75）`raw_comment` fallback 链改为： + +``` +output.aggregated.map(|a| a.markdown) + .or_else(|| output.consolidated.map(|c| c.assessment.tl_dr)) + .filter(|s| !s.is_empty()) +``` + +`tl_dr` 字段已确认存在（`src/models/mod.rs:102`）。加一条 `aggregated=None + consolidated=Some` 的单测。 + +## 9. 风险与回退 + +| 风险 | 行为 | 回退 | +|---|---|---| +| `DATABASE_URL` 已设但 PG 连不上 | **启动显式报错退出**，绝不静默落 SQLite（数据写到意外的库比不起服务更糟） | 修好连接或显式去掉 `DATABASE_URL` 走 SQLite | +| SQLite 文件不可写（权限/只读盘） | 同样显式报错退出；提供逃生门 `REVIEW_DISABLE_DB=1`（或 `--no-db`）降级为 0.9 纯内存模式，启动时 warn 一条"持久化已禁用" | 设逃生门即回到 0.9 行为 | +| migrate 失败（SQL 写错、库损坏） | 退出非零，不启动 HTTP；DB 未被业务写入 | 0.9.x 二进制不读库，直接回退部署无副作用 | +| ui-state 导入中途失败 | 单事务回滚，原文件**不改名**，回退文件回放路径继续启动 | 下次启动重试导入（幂等：三表为空才触发） | +| `secrets.key` 丢失 | DB 中 `enc:` 值无法解，启动报错指明重录（沿用 secrets.rs 现有错误文案风格） | Web UI 重新录入凭据保存即可 | +| 写穿失败（内存成功、库失败） | error 日志 + 终态一次重试；评审不阻塞 | 历史页少记录，无其他影响 | +| 回滚到 0.9.x | — | DB 文件/PG 表原样保留无害；`ui-state.toml.migrated` 手工改回 `ui-state.toml` 即恢复旧配置源 | + +## 10. 实施清单（依赖序，可逐项验收） + +1. **[祁远]** `Cargo.toml` 加 sqlx 0.8（指定 features）；`src/store/` 骨架 + `migrations/0001_init.sql`；`SqlxStore::connect/new_in_memory` + migrate 接线。**验收**：验证点 A（Any 占位符翻译 + AnyPool migrate smoke test）通过，SQLite 内存库建表成功。 +2. **[祁远]** `rows.rs` 加密边界 + `ConfigStore` 实现（§3.2 三张配置表 + §6.2 保存路径）。**验收**：配置 PUT→库→重启回放 round-trip 单测绿；LLM key 在库里是 `enc:`。 +3. **[祁远]** 一次性导入（§6.1 第 3 步，单事务 + rename 备份 + 失败回退）。**验收**：老 `ui-state.toml`（含明文 LLM key）启动一次后：库里有数据、文件改名、GET /config 行为不变、env 覆盖矩阵（§6.3）逐行单测。 +4. **[梁序]** `ReviewStore` + TaskStore 写穿（§5.2）+ 重启恢复（§5.3）。**验收**：跑一个评审 → kill -9 → 重启 → 该任务在库里是 failed/interrupted 文案；完成的评审重启后历史可查。 +5. **[梁序]** `list_reviews`/`get_review` 读库 + 投影函数签名收敛（§8.1）。**验收**：分页/过滤参数行为与 0.9 一致（同参数响应 shape 不变）。 +6. **[梁序]** Note hook 入库（§7.1，含自噬防护）+ worker 注入（§7.2）。**验收**：发 note webhook → 库里有行；重投不重复；编辑则更新；二次评审的 prompt 前缀逐字节稳定（hash 相同）。 +7. **[沈一帆]** 前端历史页适配（§8.2）+ 详情空态修复的 UI 确认。**验收**：重启后历史页有数据；团队评审详情「完整评论」tab 非空。 +8. **[梁序]** `build_review_detail` fallback 链（§8.3）+ 单测。 +9. 全量：fmt / clippy / test 绿；PG 与 SQLite 双后端各跑一遍验收清单。 + +依赖关系：1 → 2,3,4；4 → 5,6；2,3 与 4,5 可并行；6 依赖 1 即可起步（`DiscussionStore` 独立），注入部分依赖 4 的 worker 改造对齐。 + +## 11. 待验证点（实现前确认，不确定处不猜） + +- **验证点 A**：sqlx 0.8 `Any` 驱动的 `?` 占位符 PG 翻译行为、以及 `Migrator` 在 `AnyPool` 上的行为（含 `_sqlx_migrations` 锁表在 SQLite 上的表现）。方法：步骤 1 的 smoke test，双后端各跑。 +- **验证点 B**：`config/resolver/` 是否从 config.toml 承载 `git_platforms`（§6.3 表中标注待核实）。方法：`Grep "git_platforms" src/config/`。 +- **验证点 C**：评审报告的固定前缀常量位置（§7.1 自噬防护条件 a）。方法：`Grep` publisher/output 模块的报告头部模板。 +- **验证点 D**：`Any` 驱动下 `chrono::DateTime` 绑到 SQLite `TIMESTAMP` 列的存储格式与排序正确性（字典序 = 时间序是分页 `ORDER BY created_at` 的前提）。方法：步骤 4 的 round-trip 测试里断言排序。 + +## 12. 验收标准清单 + +- [ ] `cargo fmt --check` / `cargo clippy` / `cargo test` 全绿 +- [ ] PG 与 SQLite 双后端：评审完成后历史落库、可查 +- [ ] 重启 server 后历史列表/详情仍可查（§5.3） +- [ ] `ui-state.toml` 迁移后：配置热生效不变、密钥可用（git token 解密、LLM key 解密且库里为 `enc:`）、原文件备份为 `.migrated` +- [ ] Note Hook 入库：重投幂等、编辑更新、自身评论不入库 +- [ ] 二次评审注入：prompt 中含讨论历史前缀，同 MR 无新评论时 `content_hash` 相同（前缀缓存可命中） +- [ ] 团队评审（`aggregated=null`）详情「完整评论」tab 不再空态 +- [ ] `DATABASE_URL` 指向不可达 PG 时启动显式报错（不静默落 SQLite） From 55903d51b9f2d2f93ec475af843ac3041f8f326b Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 10:58:41 +0800 Subject: [PATCH 02/36] feat(store): add sqlx Any-pool skeleton, 0001_init migration, and smoke tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sqlx 0.8 with default-features=false; adds `any` and `macros` on top of the design-doc feature list: the Any driver and the migrate!() macro are otherwise not compiled in. - migrations/0001_init.sql: 7 tables per design/persistence.md §3.2, with one deviation proven by smoke test — timestamp columns are TEXT carrying fixed-width RFC 3339 UTC strings, because the Any driver has no chrono Type impls and SQLite refuses String decode from TIMESTAMP-typed columns. - src/store/mod.rs: SqlxStore::connect (PG/SQLite URL discrimination), connect_default (sqlite://{config_dir}/review.db?mode=rwc), new_in_memory (max_connections(1)), migrate (embedded via migrate!()). SQLite pools get WAL / foreign_keys / busy_timeout pragmas. - Verification point A/D smoke tests: schema creation + idempotent re-run, `?` placeholder INSERT/SELECT round trip, timestamp round trip with ORDER BY correctness. PG side is #[ignore]d behind DATABASE_URL. --- Cargo.lock | 678 ++++++++++++++++++++++++++++++++++++++- Cargo.toml | 9 + migrations/0001_init.sql | 103 ++++++ src/lib.rs | 3 + src/store/mod.rs | 306 ++++++++++++++++++ src/store/rows.rs | 4 + src/store/sqlx.rs | 4 + src/store/traits.rs | 6 + 8 files changed, 1111 insertions(+), 2 deletions(-) create mode 100644 migrations/0001_init.sql create mode 100644 src/store/mod.rs create mode 100644 src/store/rows.rs create mode 100644 src/store/sqlx.rs create mode 100644 src/store/traits.rs diff --git a/Cargo.lock b/Cargo.lock index c51d8fa..84f0d74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -131,6 +137,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -223,6 +238,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bit-set" version = "0.5.3" @@ -243,6 +264,9 @@ name = "bitflags" version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +dependencies = [ + "serde_core", +] [[package]] name = "block-buffer" @@ -270,6 +294,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.0" @@ -393,6 +423,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "convert_case" version = "0.10.0" @@ -417,6 +453,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -426,6 +477,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.22" @@ -488,6 +548,17 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.8" @@ -534,6 +605,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] @@ -558,6 +630,12 @@ dependencies = [ "litrs", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -569,6 +647,9 @@ name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] [[package]] name = "equivalent" @@ -586,6 +667,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + [[package]] name = "fancy-regex" version = "0.13.0" @@ -629,12 +731,29 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -705,6 +824,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.32" @@ -827,12 +957,32 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -851,6 +1001,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + [[package]] name = "hmac" version = "0.12.1" @@ -1115,7 +1274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", ] [[package]] @@ -1215,6 +1374,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "libc" @@ -1222,6 +1384,35 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.9.3", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -1282,6 +1473,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.2" @@ -1378,12 +1579,47 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.6", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1391,6 +1627,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1421,6 +1658,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1439,7 +1682,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] @@ -1454,6 +1697,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1466,6 +1718,39 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "poly1305" version = "0.8.0" @@ -1768,6 +2053,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d678d17679829e73d371e96880897e98fee2ded7acc0a50bdf8af2affa4b2fe5" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.12.4" @@ -1871,6 +2165,7 @@ dependencies = [ "serde_json", "serde_yaml_ng", "sha2", + "sqlx", "subtle", "tar", "tempfile", @@ -1903,6 +2198,26 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -2141,6 +2456,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -2198,6 +2524,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -2215,6 +2551,9 @@ name = "smallvec" version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] [[package]] name = "socket2" @@ -2226,12 +2565,238 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap", + "log", + "memchr", + "once_cell", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64", + "bitflags", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.6", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64", + "bitflags", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.6", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror", + "tracing", + "url", + "uuid", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2644,12 +3209,33 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -2726,6 +3312,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2766,6 +3358,12 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.126" @@ -2863,6 +3461,16 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2953,6 +3561,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -2989,6 +3606,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -3022,6 +3654,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -3034,6 +3672,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -3046,6 +3690,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -3070,6 +3720,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -3082,6 +3738,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -3094,6 +3756,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -3106,6 +3774,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index 420a226..de5fb27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,6 +102,15 @@ futures = "0.3" # Async trait support async-trait = "0.1" +# Database access: sqlx 0.8 with the Any pool so one code path serves both +# PostgreSQL (primary, via DATABASE_URL) and embedded SQLite (fallback). +# Placeholders are written as `?` everywhere; the Any driver translates them +# for Postgres. See design/persistence.md §3.1 for the dialect rules. +# Two additions on top of the design-doc list, both because of +# `default-features = false`: `any` (the Any driver is otherwise not compiled +# in) and `macros` (`sqlx::migrate!` is gated on it). +sqlx = { version = "0.8", default-features = false, features = ["any", "runtime-tokio", "postgres", "sqlite", "migrate", "macros", "chrono", "uuid", "json"] } + # Token counting for LLM context window management tiktoken-rs = "0.7" diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 0000000..cf93048 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,103 @@ +-- 0.10.0 持久化首版：7 表。方言约束见 design/persistence.md §3.1： +-- 占位符统一 `?`；不用 RETURNING（主键 Rust 侧 UUID）；JSON 列一律 TEXT； +-- 布尔用 BOOLEAN。 +-- 时间戳列：与设计文档 §3.2 草案的 TIMESTAMP 不同，一律 TEXT，存 Rust 侧 +-- chrono 生成的固定宽度 RFC 3339 UTC 串（2026-09-03T10:00:00.000000Z）。 +-- 原因（验证点 A/D 落地结论）：sqlx Any 驱动没有 chrono 的 Type 实现， +-- 且 SQLite 端拒绝对声明类型为 TIMESTAMP/Datetime 的列做 String 解码； +-- 固定宽度 UTC 串字典序 == 时间序，ORDER BY / 范围过滤语义不变。 +-- PG 端 TEXT 列天然接受该串，无需 CAST。 + +-- ── 评审任务（TaskEntry 的持久投影）── +CREATE TABLE reviews ( + task_id TEXT PRIMARY KEY, -- UUID v4, Rust 侧生成 + state TEXT NOT NULL, -- pending|running|completed|failed|cancelled + source_meta TEXT NOT NULL DEFAULT '{}', -- SourceMeta JSON + -- 从 source_meta 物化的过滤列：分页过滤要走索引，JSON 文本抽取两端写法不同， + -- 写穿时由 Rust 同步维护，读路径不碰 JSON 抽取函数。 + project TEXT, + repository TEXT, + request TEXT, -- 序列化 ReviewRequest（无凭据，见 task.rs:175-178） + result TEXT, -- ReviewOutput JSON + error TEXT, + progress INTEGER, -- 0-100，仅终态时快照；进行中的实时进度不入库 + created_at TEXT NOT NULL, + started_at TEXT, + completed_at TEXT +); +CREATE INDEX idx_reviews_created_at ON reviews (created_at DESC); +CREATE INDEX idx_reviews_state ON reviews (state); +CREATE INDEX idx_reviews_project ON reviews (project); + +-- ── 专家子报告（从 ReviewOutput.reports 拆行，便于按专家查询）── +CREATE TABLE expert_reports ( + task_id TEXT NOT NULL REFERENCES reviews(task_id) ON DELETE CASCADE, + expert_name TEXT NOT NULL, + report TEXT NOT NULL, -- ExpertReport JSON + duration_ms INTEGER, -- 首版可为 NULL：TaskEntry 目前不记 per-expert 耗时， + -- 需执行器补计时后再填充（见 §5.4 注意点） + created_at TEXT NOT NULL, + PRIMARY KEY (task_id, expert_name) +); + +-- ── MR 讨论（Note webhook + notes API 兜底共用的幂等存储）── +CREATE TABLE mr_discussions ( + platform TEXT NOT NULL, -- GitPlatformConfig.name（实例级隔离） + project TEXT NOT NULL, -- path_with_namespace + mr_iid BIGINT NOT NULL, + note_id BIGINT NOT NULL, + author TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL, + created_at TEXT NOT NULL, -- note 的创建时间，非入库时间 + ingested_at TEXT NOT NULL, -- 入库时间，排序兜底 + PRIMARY KEY (platform, project, mr_iid, note_id) -- 幂等键 +); +CREATE INDEX idx_mr_discussions_mr ON mr_discussions (platform, project, mr_iid, created_at); + +-- ── 注入上下文（支撑 LLM 前缀缓存复用）── +CREATE TABLE review_contexts ( + task_id TEXT NOT NULL REFERENCES reviews(task_id) ON DELETE CASCADE, + kind TEXT NOT NULL, -- 'mr_discussions' | 未来扩展 + content TEXT NOT NULL, -- 渲染后的上下文本（前缀稳定） + content_hash TEXT NOT NULL, -- sha256 hex；同 MR 二次评审 hash 相同即复用 + token_estimate INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + PRIMARY KEY (task_id, kind) +); +CREATE INDEX idx_review_contexts_hash ON review_contexts (content_hash); + +-- ── git 平台实例（ui-state.toml 的 [[git_platforms]] 区段入库）── +CREATE TABLE git_platforms ( + id TEXT PRIMARY KEY, -- UUID v4；业务合并键仍是 name（与内存模型一致） + name TEXT NOT NULL UNIQUE, + type TEXT NOT NULL DEFAULT 'gitlab', + base_url TEXT NOT NULL DEFAULT '', + internal_base_url TEXT NOT NULL DEFAULT '', + token TEXT NOT NULL DEFAULT '', -- enc: 加密 + webhook_secret TEXT NOT NULL DEFAULT '', -- enc: 加密 + webhook_signing_secret TEXT NOT NULL DEFAULT '', -- enc: 加密 + enabled BOOLEAN NOT NULL DEFAULT TRUE, + raw TEXT NOT NULL DEFAULT '{}', -- 扩展兜底：allowed_projects 等未列化字段 + updated_at TEXT NOT NULL +); + +-- ── LLM 实例（[[llm]] 区段入库；api_key 顺带收进加密边界）── +CREATE TABLE llm_providers ( + id TEXT PRIMARY KEY, -- UUID v4 + provider TEXT NOT NULL, -- 对齐 LLMConfig.provider（brief 中的 "name"） + model TEXT NOT NULL DEFAULT '', + api_base TEXT NOT NULL DEFAULT '', + api_key TEXT NOT NULL DEFAULT '', -- enc: 加密（新增：0.9 明文落盘） + max_tokens INTEGER NOT NULL DEFAULT 4096, + temperature REAL NOT NULL DEFAULT 0.7, + raw TEXT NOT NULL DEFAULT '{}', -- 扩展兜底：disable_thinking 等 + updated_at TEXT NOT NULL +); +CREATE UNIQUE INDEX idx_llm_providers_provider ON llm_providers (provider); + +-- ── 应用设置（ui 投影 / legacy gitlab 字段 / rules / advanced 等）── +CREATE TABLE app_settings ( + key TEXT PRIMARY KEY, -- 如 'ui'、'gitlab'、'rules'、'advanced' + value TEXT NOT NULL, -- JSON + updated_at TEXT NOT NULL +); diff --git a/src/lib.rs b/src/lib.rs index e41e015..8c5791f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,6 +37,9 @@ pub mod publisher; pub mod repo; pub mod scoring; pub mod server; +/// Persistence layer (0.10.0): sqlx `Any` pool serving PostgreSQL and SQLite +/// from one code path. See `design/persistence.md`. +pub mod store; pub mod team; pub mod tokenizer; diff --git a/src/store/mod.rs b/src/store/mod.rs new file mode 100644 index 0000000..d5d6191 --- /dev/null +++ b/src/store/mod.rs @@ -0,0 +1,306 @@ +//! Persistence layer (0.10.0). +//! +//! One `sqlx::AnyPool` serves both PostgreSQL (primary, via `DATABASE_URL`) +//! and embedded SQLite (fallback). All SQL lives behind this module; the +//! dialect rules of `design/persistence.md` §3.1 apply everywhere: +//! +//! - placeholders are always `?` (the Any driver translates them for PG); +//! - no `RETURNING` (primary keys are Rust-side UUIDs); +//! - JSON columns are `TEXT`, serialized/deserialized by the store layer; +//! - timestamps are generated by chrono on the Rust side, never by DDL +//! defaults. NOTE (finding from verification point A/D): the `Any` driver +//! has no `Type` impls for chrono types — only bool/int/float/str/blob +//! are supported — and SQLite refuses `String` decode from a column +//! declared `TIMESTAMP`. Timestamp columns are therefore declared `TEXT` +//! and carry fixed-width RFC 3339 UTC strings +//! (`2026-09-03T10:00:00.000000Z`), encoded/decoded by the store layer. +//! Fixed-width UTC `Z` formatting keeps lexicographic order == +//! chronological order, which `ORDER BY created_at` pagination relies on. +//! +//! This step ships only the connection/migration skeleton plus the +//! verification-point-A smoke tests. Domain traits (`ReviewStore` / +//! `ConfigStore` / `DiscussionStore`) and row codecs land in later steps. + +pub mod rows; +pub mod sqlx; +pub mod traits; + +use std::path::Path; + +use anyhow::{Context, Result}; +use chrono::{DateTime, SecondsFormat, Utc}; + +/// Embedded migrations (compiled in via `sqlx::migrate!`). +static MIGRATOR: ::sqlx::migrate::Migrator = ::sqlx::migrate!("./migrations"); + +/// SQLx store backed by an `Any` pool (PostgreSQL or SQLite). +/// +/// Business methods are added in later steps behind the domain traits in +/// [`traits`]; this type currently owns pool construction, SQLite pragmas, +/// and migrations. +#[derive(Debug, Clone)] +pub struct SqlxStore { + pool: ::sqlx::AnyPool, +} + +impl SqlxStore { + /// Connect to the database identified by `url`. + /// + /// URL discrimination (design/persistence.md §4.3): + /// - `postgres://` / `postgresql://` → PostgreSQL; + /// - anything else → SQLite. + /// + /// For SQLite, after the pool is built the connection pragmas required by + /// the schema are applied: WAL journal, foreign keys on, 5 s busy timeout. + pub async fn connect(url: &str) -> Result { + ::sqlx::any::install_default_drivers(); + let is_postgres = url.starts_with("postgres://") || url.starts_with("postgresql://"); + let pool = ::sqlx::any::AnyPoolOptions::new().connect(url).await.with_context(|| { + format!( + "failed to connect to database ({url_scheme})", + url_scheme = scheme_of(url) + ) + })?; + if !is_postgres { + apply_sqlite_pragmas(&pool).await?; + } + Ok(Self { pool }) + } + + /// Connect to the default embedded SQLite database under `config_dir` + /// (`sqlite://{config_dir}/review.db?mode=rwc`, created on demand). + pub async fn connect_default(config_dir: &Path) -> Result { + std::fs::create_dir_all(config_dir) + .with_context(|| format!("failed to create config dir {}", config_dir.display()))?; + let url = format!("sqlite://{}/review.db?mode=rwc", config_dir.display()); + Self::connect(&url).await + } + + /// In-memory SQLite store for unit tests. + /// + /// `max_connections(1)` is mandatory: with a larger pool each connection + /// would be an independent in-memory database. + pub async fn new_in_memory() -> Result { + ::sqlx::any::install_default_drivers(); + let pool = ::sqlx::any::AnyPoolOptions::new() + .max_connections(1) + .connect("sqlite::memory:") + .await + .context("failed to open in-memory sqlite database")?; + apply_sqlite_pragmas(&pool).await?; + Ok(Self { pool }) + } + + /// Apply the embedded migrations (idempotent — already-applied + /// migrations are skipped based on the `_sqlx_migrations` ledger). + pub async fn migrate(&self) -> Result<()> { + MIGRATOR.run(&self.pool).await.context("database migration failed")?; + Ok(()) + } + + /// Access the underlying pool (used by trait implementations in + /// [`sqlx`] and by tests). + pub fn pool(&self) -> &::sqlx::AnyPool { + &self.pool + } +} + +/// Scheme prefix of a database URL, for error messages that must not leak +/// credentials embedded in the URL. +fn scheme_of(url: &str) -> &str { + url.split("://").next().unwrap_or(url) +} + +async fn apply_sqlite_pragmas(pool: &::sqlx::AnyPool) -> Result<()> { + // Executed as separate statements: Any queries are single-statement. + for pragma in [ + "PRAGMA journal_mode=WAL;", + "PRAGMA foreign_keys=ON;", + "PRAGMA busy_timeout=5000;", + ] { + ::sqlx::query(pragma) + .execute(pool) + .await + .with_context(|| format!("failed to apply {pragma}"))?; + } + Ok(()) +} + +/// Encode a timestamp for a `TEXT` timestamp column. The Any driver has no +/// chrono `Type` impls, so values cross as fixed-width RFC 3339 UTC strings +/// (`2026-09-03T10:00:00.000000Z`); lexicographic order == chronological +/// order, which `ORDER BY created_at` pagination relies on. +// Used by tests now; the trait implementations (later steps) are the real +// consumers, hence the allow to keep non-test builds warning-free. +#[allow(dead_code)] +pub(crate) fn encode_ts(ts: &DateTime) -> String { + ts.to_rfc3339_opts(SecondsFormat::Micros, true) +} + +/// Decode a timestamp produced by [`encode_ts`]. +#[allow(dead_code)] +pub(crate) fn decode_ts(s: &str) -> Result> { + Ok(DateTime::parse_from_rfc3339(s) + .with_context(|| format!("invalid RFC 3339 timestamp in database: {s:?}"))? + .with_timezone(&Utc)) +} + +#[cfg(test)] +mod tests { + use super::*; + use ::sqlx::Row; + use chrono::TimeZone; + + /// 验证点 A(a): in-memory SQLite + migrate creates the schema, and a + /// second migrate run is an idempotent no-op. + #[tokio::test] + async fn migrate_creates_schema_and_is_idempotent() { + let store = SqlxStore::new_in_memory().await.unwrap(); + store.migrate().await.unwrap(); + + let tables: Vec = ::sqlx::query( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + ) + .fetch_all(store.pool()) + .await + .unwrap() + .into_iter() + .map(|row| row.get::(0)) + .collect(); + for expected in [ + "app_settings", + "expert_reports", + "git_platforms", + "llm_providers", + "mr_discussions", + "review_contexts", + "reviews", + ] { + assert!( + tables.iter().any(|t| t == expected), + "missing table {expected}, got {tables:?}" + ); + } + + // Second run: must succeed and apply nothing new. + store.migrate().await.unwrap(); + let applied: i64 = ::sqlx::query_scalar("SELECT COUNT(*) FROM _sqlx_migrations") + .fetch_one(store.pool()) + .await + .unwrap(); + assert_eq!(applied, 1, "only 0001_init should be recorded"); + } + + /// 验证点 A(b): `?` placeholder INSERT + SELECT round trip on SQLite + /// through the Any driver. + #[tokio::test] + async fn question_mark_placeholders_round_trip() { + let store = SqlxStore::new_in_memory().await.unwrap(); + store.migrate().await.unwrap(); + + ::sqlx::query( + "INSERT INTO reviews (task_id, state, source_meta, project, repository, created_at) \ + VALUES (?, ?, ?, ?, ?, ?)", + ) + .bind("00000000-0000-0000-0000-0000000000ab") + .bind("pending") + .bind("{}") + .bind("group/proj") + .bind("proj") + .bind(encode_ts(&Utc::now())) + .execute(store.pool()) + .await + .unwrap(); + + let (task_id, state, project): (String, String, Option) = + ::sqlx::query_as("SELECT task_id, state, project FROM reviews WHERE task_id = ?") + .bind("00000000-0000-0000-0000-0000000000ab") + .fetch_one(store.pool()) + .await + .unwrap(); + assert_eq!(task_id, "00000000-0000-0000-0000-0000000000ab"); + assert_eq!(state, "pending"); + assert_eq!(project.as_deref(), Some("group/proj")); + } + + /// 验证点 D: timestamps (chrono-generated, RFC 3339 TEXT because the Any + /// driver has no chrono `Type` impls) round-trip losslessly and + /// `ORDER BY created_at` returns chronological order. + #[tokio::test] + async fn datetime_binding_round_trip_and_ordering() { + let store = SqlxStore::new_in_memory().await.unwrap(); + store.migrate().await.unwrap(); + + let t1 = Utc.with_ymd_and_hms(2026, 9, 3, 10, 0, 0).unwrap(); + let t2 = Utc.with_ymd_and_hms(2026, 9, 3, 10, 0, 1).unwrap(); + let t3 = Utc.with_ymd_and_hms(2026, 9, 3, 9, 59, 59).unwrap(); + + // Insert out of order on purpose. + for (id, ts) in [("b", t2), ("a", t1), ("c", t3)] { + ::sqlx::query("INSERT INTO reviews (task_id, state, created_at) VALUES (?, ?, ?)") + .bind(id) + .bind("completed") + .bind(encode_ts(&ts)) + .execute(store.pool()) + .await + .unwrap(); + } + + // The stored representation itself must be the RFC 3339 UTC string — + // lexicographic order on it is the pagination premise. + let raw: String = ::sqlx::query_scalar("SELECT created_at FROM reviews WHERE task_id = 'a'") + .fetch_one(store.pool()) + .await + .unwrap(); + assert_eq!(raw, "2026-09-03T10:00:00.000000Z"); + + let rows: Vec<(String, String)> = + ::sqlx::query_as("SELECT task_id, created_at FROM reviews ORDER BY created_at") + .fetch_all(store.pool()) + .await + .unwrap(); + let ids: Vec<&str> = rows.iter().map(|(id, _)| id.as_str()).collect(); + assert_eq!(ids, vec!["c", "a", "b"], "chronological order expected"); + // Round-trip fidelity: decoded value must equal the original. + assert_eq!(decode_ts(&rows[1].1).unwrap(), t1); + } + + /// 验证点 A(a) on the PG side: requires a live PostgreSQL via + /// `DATABASE_URL`. Run explicitly with: + /// `DATABASE_URL=postgres://... cargo test store -- --ignored` + /// + /// Verifies: migrate on an AnyPool against PG (incl. the + /// `_sqlx_migrations` ledger), idempotent re-run, and `?` placeholder + /// translation. Timestamps are TEXT columns bound as RFC 3339 strings + /// (Any driver constraint), so the read-back is a plain String round trip. + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing at a scratch PostgreSQL"] + async fn migrate_on_postgres_smoke() { + let url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set"); + let store = SqlxStore::connect(&url).await.unwrap(); + store.migrate().await.unwrap(); + // Idempotent second run. + store.migrate().await.unwrap(); + + // `?` placeholders must be translated by the Any driver. + let now = Utc::now(); + ::sqlx::query("INSERT INTO reviews (task_id, state, created_at) VALUES (?, ?, ?)") + .bind("pg-smoke-0001") + .bind("pending") + .bind(encode_ts(&now)) + .execute(store.pool()) + .await + .unwrap(); + let back: String = ::sqlx::query_scalar("SELECT created_at FROM reviews WHERE task_id = ?") + .bind("pg-smoke-0001") + .fetch_one(store.pool()) + .await + .unwrap(); + assert_eq!(decode_ts(&back).unwrap(), now); + ::sqlx::query("DELETE FROM reviews WHERE task_id = ?") + .bind("pg-smoke-0001") + .execute(store.pool()) + .await + .unwrap(); + } +} diff --git a/src/store/rows.rs b/src/store/rows.rs new file mode 100644 index 0000000..6900755 --- /dev/null +++ b/src/store/rows.rs @@ -0,0 +1,4 @@ +//! Row structure ⇄ domain structure codecs (`TaskEntry` / `UiStateFile` / …). +//! The `enc:` encryption boundary lives here (design/persistence.md §4.1). +//! +//! Placeholder module — codecs land in later steps. diff --git a/src/store/sqlx.rs b/src/store/sqlx.rs new file mode 100644 index 0000000..72a1eb0 --- /dev/null +++ b/src/store/sqlx.rs @@ -0,0 +1,4 @@ +//! `SqlxStore` implementations of the domain traits in [`crate::store::traits`]. +//! All SQL lives in this file (design/persistence.md §4.1). +//! +//! Placeholder module — implementations land in later steps. diff --git a/src/store/traits.rs b/src/store/traits.rs new file mode 100644 index 0000000..3b2c164 --- /dev/null +++ b/src/store/traits.rs @@ -0,0 +1,6 @@ +//! Domain traits: `ReviewStore` (reviews / expert_reports / review_contexts), +//! `ConfigStore` (git_platforms / llm_providers / app_settings), +//! `DiscussionStore` (mr_discussions). +//! +//! Placeholder module — trait definitions land in later steps +//! (design/persistence.md §4.2). From 7ae9f8179d847a472ba9218bff16d4ff7f850aeb Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 11:16:11 +0800 Subject: [PATCH 03/36] feat(store): ConfigStore implementation with enc: encryption boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - traits.rs: ConfigStore — whole-set load/replace for git_platforms and llm_providers (aligned with UiStateFile semantics), legacy gitlab via the app_settings 'gitlab' JSON row, arbitrary app_settings JSON get/upsert, and config_tables_empty (the §6.1 one-shot import trigger). - rows.rs: GitPlatformConfig / LLMConfig / PersistedGitlabConfig codecs. This is the enc: boundary: token / webhook_secret / webhook_signing_secret / api_key are ChaCha20-Poly1305 encrypted at rest (LLM api_key newly inside the boundary — 0.9 persisted it plaintext); empty stays empty; values without the enc: prefix read back as legacy plaintext. LLM list order is round-tripped via a position marker in raw JSON (first entry is the fallback primary provider). - sqlx.rs: SqlxStore impl; ? placeholders, no RETURNING, RFC 3339 TEXT timestamps, replace-in-transaction semantics. - SqlxStore now carries the secrets key: connect() resolves it via resolve_ui_state_path + key_path_for, connect_default(config_dir) uses {config_dir}/secrets.key, new_in_memory() uses an ephemeral key. - Migration deviation (verified by smoke test): git_platforms.enabled is INTEGER 0/1 instead of BOOLEAN — the Any driver cannot decode SQLite columns declared BOOLEAN (only Null/Int4/Integer/Float/Blob/Text pass the SQLite→Any type mapping). --- migrations/0001_init.sql | 18 +- src/store/mod.rs | 51 +++-- src/store/rows.rs | 179 +++++++++++++++- src/store/sqlx.rs | 446 ++++++++++++++++++++++++++++++++++++++- src/store/traits.rs | 60 +++++- 5 files changed, 726 insertions(+), 28 deletions(-) diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql index cf93048..48ff07f 100644 --- a/migrations/0001_init.sql +++ b/migrations/0001_init.sql @@ -1,12 +1,13 @@ -- 0.10.0 持久化首版：7 表。方言约束见 design/persistence.md §3.1： --- 占位符统一 `?`；不用 RETURNING（主键 Rust 侧 UUID）；JSON 列一律 TEXT； --- 布尔用 BOOLEAN。 +-- 占位符统一 `?`；不用 RETURNING（主键 Rust 侧 UUID）；JSON 列一律 TEXT。 -- 时间戳列：与设计文档 §3.2 草案的 TIMESTAMP 不同，一律 TEXT，存 Rust 侧 -- chrono 生成的固定宽度 RFC 3339 UTC 串（2026-09-03T10:00:00.000000Z）。 --- 原因（验证点 A/D 落地结论）：sqlx Any 驱动没有 chrono 的 Type 实现， --- 且 SQLite 端拒绝对声明类型为 TIMESTAMP/Datetime 的列做 String 解码； --- 固定宽度 UTC 串字典序 == 时间序，ORDER BY / 范围过滤语义不变。 --- PG 端 TEXT 列天然接受该串，无需 CAST。 +-- 布尔列：同理不用 BOOLEAN，一律 INTEGER 0/1。 +-- 原因（验证点 A/D 落地结论）：sqlx Any 驱动对 SQLite 只认 +-- Null/Int4/Integer/Float/Blob/Text 五类声明类型，BOOLEAN / TIMESTAMP +-- 列读不出来，chrono/bool 也没有 Type 实现；固定宽度 UTC 串 +-- 字典序 == 时间序，ORDER BY / 范围过滤语义不变。PG 端 TEXT / INTEGER +-- 列天然接受这些值，无需 CAST。 -- ── 评审任务（TaskEntry 的持久投影）── CREATE TABLE reviews ( @@ -76,7 +77,10 @@ CREATE TABLE git_platforms ( token TEXT NOT NULL DEFAULT '', -- enc: 加密 webhook_secret TEXT NOT NULL DEFAULT '', -- enc: 加密 webhook_signing_secret TEXT NOT NULL DEFAULT '', -- enc: 加密 - enabled BOOLEAN NOT NULL DEFAULT TRUE, + -- 设计文档 §3.1 写的是 BOOLEAN，但 Any 驱动无法解码 SQLite 声明类型为 + -- Bool 的列（验证点 A 实证；Any 只认 Null/Int4/Integer/Float/Blob/Text）， + -- 故用 INTEGER 0/1，绑定侧转 bool。 + enabled INTEGER NOT NULL DEFAULT 1, raw TEXT NOT NULL DEFAULT '{}', -- 扩展兜底：allowed_projects 等未列化字段 updated_at TEXT NOT NULL ); diff --git a/src/store/mod.rs b/src/store/mod.rs index d5d6191..4e7f180 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -17,9 +17,11 @@ //! Fixed-width UTC `Z` formatting keeps lexicographic order == //! chronological order, which `ORDER BY created_at` pagination relies on. //! -//! This step ships only the connection/migration skeleton plus the -//! verification-point-A smoke tests. Domain traits (`ReviewStore` / -//! `ConfigStore` / `DiscussionStore`) and row codecs land in later steps. +//! Shipped so far: connection/migration skeleton, verification-point-A/D +//! smoke tests, and the configuration domain ([`traits::ConfigStore`] +//! implemented on [`SqlxStore`] in [`sqlx`], row codecs + the `enc:` +//! boundary in [`rows`]). `ReviewStore` / `DiscussionStore` land in later +//! steps. pub mod rows; pub mod sqlx; @@ -35,12 +37,14 @@ static MIGRATOR: ::sqlx::migrate::Migrator = ::sqlx::migrate!("./migrations"); /// SQLx store backed by an `Any` pool (PostgreSQL or SQLite). /// -/// Business methods are added in later steps behind the domain traits in -/// [`traits`]; this type currently owns pool construction, SQLite pragmas, -/// and migrations. +/// Besides pool construction / SQLite pragmas / migrations, the store holds +/// the at-rest encryption key (`secrets.key`, per config dir): the `enc:` +/// boundary lives in [`rows`], and PG deployments still read the key from +/// the server-local config dir (design/persistence.md §6.2). #[derive(Debug, Clone)] pub struct SqlxStore { pool: ::sqlx::AnyPool, + pub(crate) key: [u8; 32], } impl SqlxStore { @@ -52,7 +56,18 @@ impl SqlxStore { /// /// For SQLite, after the pool is built the connection pragmas required by /// the schema are applied: WAL journal, foreign keys on, 5 s busy timeout. + /// The secrets key resolves via the standard config-dir resolution + /// (`persist::resolve_ui_state_path` → `key_path_for`). pub async fn connect(url: &str) -> Result { + let state_path = crate::server::api::config::persist::resolve_ui_state_path() + .context("cannot resolve the config dir for the secrets key")?; + let key = crate::config::secrets::load_or_create_key(&crate::config::secrets::key_path_for(&state_path))?; + Self::connect_with_key(url, key).await + } + + /// Connect with an explicit secrets key (used by [`Self::connect`] and + /// available to tests that need a stable key without a config dir). + pub async fn connect_with_key(url: &str, key: [u8; 32]) -> Result { ::sqlx::any::install_default_drivers(); let is_postgres = url.starts_with("postgres://") || url.starts_with("postgresql://"); let pool = ::sqlx::any::AnyPoolOptions::new().connect(url).await.with_context(|| { @@ -64,23 +79,31 @@ impl SqlxStore { if !is_postgres { apply_sqlite_pragmas(&pool).await?; } - Ok(Self { pool }) + Ok(Self { pool, key }) } /// Connect to the default embedded SQLite database under `config_dir` - /// (`sqlite://{config_dir}/review.db?mode=rwc`, created on demand). + /// (`sqlite://{config_dir}/review.db?mode=rwc`, created on demand); the + /// secrets key is `{config_dir}/secrets.key`, created on first use. pub async fn connect_default(config_dir: &Path) -> Result { std::fs::create_dir_all(config_dir) .with_context(|| format!("failed to create config dir {}", config_dir.display()))?; + let key = crate::config::secrets::load_or_create_key( + &config_dir.join(crate::config::secrets::SECRETS_KEY_FILE_NAME), + )?; let url = format!("sqlite://{}/review.db?mode=rwc", config_dir.display()); - Self::connect(&url).await + Self::connect_with_key(&url, key).await } /// In-memory SQLite store for unit tests. /// /// `max_connections(1)` is mandatory: with a larger pool each connection - /// would be an independent in-memory database. + /// would be an independent in-memory database. Uses an ephemeral random + /// key — enough for round-trip tests, but nothing persists between + /// instances. pub async fn new_in_memory() -> Result { + let mut key = [0u8; 32]; + rand::RngCore::fill_bytes(&mut rand::rngs::OsRng, &mut key); ::sqlx::any::install_default_drivers(); let pool = ::sqlx::any::AnyPoolOptions::new() .max_connections(1) @@ -88,7 +111,7 @@ impl SqlxStore { .await .context("failed to open in-memory sqlite database")?; apply_sqlite_pragmas(&pool).await?; - Ok(Self { pool }) + Ok(Self { pool, key }) } /// Apply the embedded migrations (idempotent — already-applied @@ -130,14 +153,12 @@ async fn apply_sqlite_pragmas(pool: &::sqlx::AnyPool) -> Result<()> { /// chrono `Type` impls, so values cross as fixed-width RFC 3339 UTC strings /// (`2026-09-03T10:00:00.000000Z`); lexicographic order == chronological /// order, which `ORDER BY created_at` pagination relies on. -// Used by tests now; the trait implementations (later steps) are the real -// consumers, hence the allow to keep non-test builds warning-free. -#[allow(dead_code)] pub(crate) fn encode_ts(ts: &DateTime) -> String { ts.to_rfc3339_opts(SecondsFormat::Micros, true) } -/// Decode a timestamp produced by [`encode_ts`]. +/// Decode a timestamp produced by [`encode_ts`]. Used by tests and by the +/// review-domain codecs (later steps). #[allow(dead_code)] pub(crate) fn decode_ts(s: &str) -> Result> { Ok(DateTime::parse_from_rfc3339(s) diff --git a/src/store/rows.rs b/src/store/rows.rs index 6900755..8bf216f 100644 --- a/src/store/rows.rs +++ b/src/store/rows.rs @@ -1,4 +1,177 @@ -//! Row structure ⇄ domain structure codecs (`TaskEntry` / `UiStateFile` / …). -//! The `enc:` encryption boundary lives here (design/persistence.md §4.1). +//! Row structure ⇄ domain structure codecs for the configuration domain. //! -//! Placeholder module — codecs land in later steps. +//! This module is the `enc:` encryption boundary (design/persistence.md +//! §4.1): domain values are live plaintext, row values are the at-rest form. +//! Encrypted at rest: `git_platforms.token / webhook_secret / +//! webhook_signing_secret`, `llm_providers.api_key` (newly inside the +//! boundary — 0.9 stored it plaintext), and each field of the legacy +//! `gitlab` settings JSON. Empty strings stay empty (never encrypted). +//! Values read back WITHOUT the `enc:` prefix are legacy plaintext and pass +//! through unchanged (`decrypt_secret`'s existing semantics). + +use anyhow::{Context, Result}; +use serde_json::{json, Value}; + +use crate::config::secrets::{decrypt_secret, encrypt_secret}; +use crate::models::{GitPlatformConfig, LLMConfig}; +use crate::server::api::config::persist::PersistedGitlabConfig; + +/// At-rest form of one `git_platforms` row. +#[derive(Debug)] +pub(crate) struct GitPlatformRow { + pub id: String, + pub name: String, + pub platform_type: String, + pub base_url: String, + pub internal_base_url: String, + pub token: String, + pub webhook_secret: String, + pub webhook_signing_secret: String, + pub enabled: bool, + /// JSON fallback bag for non-columnized fields (`allowed_projects`). + pub raw: String, + pub updated_at: String, +} + +/// At-rest form of one `llm_providers` row. +#[derive(Debug)] +pub(crate) struct LlmProviderRow { + pub id: String, + pub provider: String, + pub model: String, + pub api_base: String, + pub api_key: String, + pub max_tokens: i64, + pub temperature: f64, + /// JSON fallback bag: `disable_thinking`, plus `position` — the list + /// index, because provider order is semantically meaningful (first entry + /// is the fallback primary) and the table has no sequence column. + pub raw: String, + pub updated_at: String, +} + +fn encrypt_non_empty(value: &str, key: &[u8; 32]) -> Result { + if value.is_empty() { + Ok(String::new()) + } else { + encrypt_secret(value, key) + } +} + +pub(crate) fn git_platform_to_row( + platform: &GitPlatformConfig, + id: String, + updated_at: String, + key: &[u8; 32], +) -> Result { + // `enabled` has no domain counterpart yet (GitPlatformConfig carries no + // such field); the column is future-proofing and always written TRUE. + let raw = if platform.allowed_projects.is_empty() { + json!({}) + } else { + json!({ "allowed_projects": platform.allowed_projects }) + }; + Ok(GitPlatformRow { + id, + name: platform.name.clone(), + platform_type: platform.platform_type.clone(), + base_url: platform.base_url.clone(), + internal_base_url: platform.internal_base_url.clone(), + token: encrypt_non_empty(&platform.token, key)?, + webhook_secret: encrypt_non_empty(&platform.webhook_secret, key)?, + webhook_signing_secret: encrypt_non_empty(&platform.webhook_signing_secret, key)?, + enabled: true, + raw: raw.to_string(), + updated_at, + }) +} + +pub(crate) fn git_platform_from_row(row: GitPlatformRow, key: &[u8; 32]) -> Result { + let raw: Value = serde_json::from_str(&row.raw) + .with_context(|| format!("git_platforms row {:?} has invalid raw JSON", row.name))?; + let allowed_projects = raw + .get("allowed_projects") + .and_then(Value::as_array) + .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_string)).collect()) + .unwrap_or_default(); + Ok(GitPlatformConfig { + name: row.name, + platform_type: row.platform_type, + base_url: row.base_url, + internal_base_url: row.internal_base_url, + token: decrypt_secret(&row.token, key)?, + webhook_secret: decrypt_secret(&row.webhook_secret, key)?, + webhook_signing_secret: decrypt_secret(&row.webhook_signing_secret, key)?, + allowed_projects, + }) +} + +pub(crate) fn llm_to_row( + config: &LLMConfig, + position: usize, + id: String, + updated_at: String, + key: &[u8; 32], +) -> Result { + let mut raw = json!({ "position": position as i64 }); + if let Some(disable_thinking) = config.disable_thinking { + raw["disable_thinking"] = json!(disable_thinking); + } + Ok(LlmProviderRow { + id, + provider: config.provider.clone(), + model: config.model.clone(), + api_base: config.api_base.clone(), + api_key: encrypt_non_empty(&config.api_key, key)?, + max_tokens: i64::from(config.max_tokens), + temperature: f64::from(config.temperature), + raw: raw.to_string(), + updated_at, + }) +} + +/// List position recorded by [`llm_to_row`]; `None` for rows written by +/// other means (sorts after positioned rows). +pub(crate) fn llm_row_position(row: &LlmProviderRow) -> Option { + serde_json::from_str::(&row.raw).ok()?.get("position")?.as_i64() +} + +pub(crate) fn llm_from_row(row: LlmProviderRow, key: &[u8; 32]) -> Result { + let raw: Value = serde_json::from_str(&row.raw) + .with_context(|| format!("llm_providers row {:?} has invalid raw JSON", row.provider))?; + let disable_thinking = raw.get("disable_thinking").and_then(Value::as_bool); + Ok(LLMConfig { + provider: row.provider, + model: row.model, + api_key: decrypt_secret(&row.api_key, key)?, + api_base: row.api_base, + max_tokens: u32::try_from(row.max_tokens) + .with_context(|| format!("llm_providers.max_tokens out of range: {}", row.max_tokens))?, + temperature: row.temperature as f32, + disable_thinking, + }) +} + +/// Legacy GitLab credentials ⇄ the `app_settings` row at key `gitlab`. +/// Each field is individually `enc:`-encrypted inside the JSON (§3.2 note). +pub(crate) fn legacy_gitlab_to_value(gitlab: &PersistedGitlabConfig, key: &[u8; 32]) -> Result { + Ok(json!({ + "token": encrypt_non_empty(&gitlab.token, key)?, + "webhook_secret": encrypt_non_empty(&gitlab.webhook_secret, key)?, + "webhook_signing_secret": encrypt_non_empty(&gitlab.webhook_signing_secret, key)?, + })) +} + +pub(crate) fn legacy_gitlab_from_value(value: &Value, key: &[u8; 32]) -> Result { + let field = |name: &str| -> Result { + match value.get(name).and_then(Value::as_str) { + Some(s) => decrypt_secret(s, key), + None => Ok(String::new()), + } + }; + Ok(PersistedGitlabConfig { + token: field("token")?, + webhook_secret: field("webhook_secret")?, + webhook_signing_secret: field("webhook_signing_secret")?, + }) +} diff --git a/src/store/sqlx.rs b/src/store/sqlx.rs index 72a1eb0..2b3d8ef 100644 --- a/src/store/sqlx.rs +++ b/src/store/sqlx.rs @@ -1,4 +1,448 @@ //! `SqlxStore` implementations of the domain traits in [`crate::store::traits`]. //! All SQL lives in this file (design/persistence.md §4.1). //! -//! Placeholder module — implementations land in later steps. +//! Dialect discipline (§3.1): `?` placeholders only, no `RETURNING`, JSON as +//! bound `String`, timestamps via `encode_ts` / `decode_ts` (RFC 3339 TEXT). +//! NOTE: this file is itself named `sqlx.rs` — the sibling module shadows +//! the extern crate lexically, so every reference to the real sqlx crate +//! must use the absolute `::sqlx::` path. + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use chrono::Utc; + +use crate::models::{GitPlatformConfig, LLMConfig}; +use crate::server::api::config::persist::PersistedGitlabConfig; + +use super::rows; +use super::traits::ConfigStore; +use super::{encode_ts, SqlxStore}; + +const LEGACY_GITLAB_KEY: &str = "gitlab"; + +#[async_trait] +impl ConfigStore for SqlxStore { + async fn load_git_platforms(&self) -> Result> { + let rows = ::sqlx::query_as::< + _, + ( + String, + String, + String, + String, + String, + String, + String, + String, + i64, + String, + String, + ), + >( + "SELECT id, name, type, base_url, internal_base_url, token, webhook_secret, \ + webhook_signing_secret, enabled, raw, updated_at FROM git_platforms ORDER BY name", + ) + .fetch_all(self.pool()) + .await + .context("failed to load git_platforms")?; + rows.into_iter() + .map( + |( + id, + name, + platform_type, + base_url, + internal_base_url, + token, + webhook_secret, + webhook_signing_secret, + enabled, + raw, + updated_at, + )| { + rows::git_platform_from_row( + rows::GitPlatformRow { + id, + name, + platform_type, + base_url, + internal_base_url, + token, + webhook_secret, + webhook_signing_secret, + // Any driver cannot decode SQLite BOOLEAN-declared + // columns; the column is INTEGER 0/1. + enabled: enabled != 0, + raw, + updated_at, + }, + &self.key, + ) + }, + ) + .collect() + } + + async fn replace_git_platforms(&self, platforms: &[GitPlatformConfig]) -> Result<()> { + let now = encode_ts(&Utc::now()); + let mut tx = self.pool().begin().await.context("begin replace_git_platforms")?; + ::sqlx::query("DELETE FROM git_platforms") + .execute(&mut *tx) + .await + .context("clear git_platforms")?; + for platform in platforms { + let row = rows::git_platform_to_row(platform, uuid::Uuid::new_v4().to_string(), now.clone(), &self.key)?; + ::sqlx::query( + "INSERT INTO git_platforms (id, name, type, base_url, internal_base_url, token, \ + webhook_secret, webhook_signing_secret, enabled, raw, updated_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(&row.id) + .bind(&row.name) + .bind(&row.platform_type) + .bind(&row.base_url) + .bind(&row.internal_base_url) + .bind(&row.token) + .bind(&row.webhook_secret) + .bind(&row.webhook_signing_secret) + .bind(i64::from(row.enabled)) + .bind(&row.raw) + .bind(&row.updated_at) + .execute(&mut *tx) + .await + .with_context(|| format!("insert git_platform {:?}", platform.name))?; + } + tx.commit().await.context("commit replace_git_platforms")?; + Ok(()) + } + + async fn load_llm_providers(&self) -> Result> { + let rows = ::sqlx::query_as::<_, (String, String, String, String, String, i64, f64, String, String)>( + "SELECT id, provider, model, api_base, api_key, max_tokens, temperature, raw, \ + updated_at FROM llm_providers ORDER BY provider", + ) + .fetch_all(self.pool()) + .await + .context("failed to load llm_providers")?; + let mut rows: Vec = rows + .into_iter() + .map( + |(id, provider, model, api_base, api_key, max_tokens, temperature, raw, updated_at)| { + rows::LlmProviderRow { + id, + provider, + model, + api_base, + api_key, + max_tokens, + temperature, + raw, + updated_at, + } + }, + ) + .collect(); + // Stable sort by the recorded list position; rows without one keep + // their deterministic `provider` order at the tail. + rows.sort_by_key(|r| rows::llm_row_position(r).unwrap_or(i64::MAX)); + rows.into_iter().map(|r| rows::llm_from_row(r, &self.key)).collect() + } + + async fn replace_llm_providers(&self, providers: &[LLMConfig]) -> Result<()> { + let now = encode_ts(&Utc::now()); + let mut tx = self.pool().begin().await.context("begin replace_llm_providers")?; + ::sqlx::query("DELETE FROM llm_providers") + .execute(&mut *tx) + .await + .context("clear llm_providers")?; + for (position, config) in providers.iter().enumerate() { + let row = rows::llm_to_row( + config, + position, + uuid::Uuid::new_v4().to_string(), + now.clone(), + &self.key, + )?; + ::sqlx::query( + "INSERT INTO llm_providers (id, provider, model, api_base, api_key, max_tokens, \ + temperature, raw, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(&row.id) + .bind(&row.provider) + .bind(&row.model) + .bind(&row.api_base) + .bind(&row.api_key) + .bind(row.max_tokens) + .bind(row.temperature) + .bind(&row.raw) + .bind(&row.updated_at) + .execute(&mut *tx) + .await + .with_context(|| format!("insert llm_provider {:?}", config.provider))?; + } + tx.commit().await.context("commit replace_llm_providers")?; + Ok(()) + } + + async fn load_legacy_gitlab(&self) -> Result { + match self.load_setting(LEGACY_GITLAB_KEY).await? { + Some(value) => rows::legacy_gitlab_from_value(&value, &self.key), + None => Ok(PersistedGitlabConfig::default()), + } + } + + async fn save_legacy_gitlab(&self, gitlab: &PersistedGitlabConfig) -> Result<()> { + let value = rows::legacy_gitlab_to_value(gitlab, &self.key)?; + self.save_setting(LEGACY_GITLAB_KEY, &value).await + } + + async fn load_setting(&self, key: &str) -> Result> { + let raw: Option = ::sqlx::query_scalar("SELECT value FROM app_settings WHERE key = ?") + .bind(key) + .fetch_optional(self.pool()) + .await + .with_context(|| format!("failed to load app_setting {key:?}"))?; + raw.map(|s| serde_json::from_str(&s).with_context(|| format!("app_setting {key:?} holds invalid JSON"))) + .transpose() + } + + async fn save_setting(&self, key: &str, value: &serde_json::Value) -> Result<()> { + // Upsert syntax is shared by PG and SQLite (≥3.24); no RETURNING. + ::sqlx::query( + "INSERT INTO app_settings (key, value, updated_at) VALUES (?, ?, ?) \ + ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at", + ) + .bind(key) + .bind(value.to_string()) + .bind(encode_ts(&Utc::now())) + .execute(self.pool()) + .await + .with_context(|| format!("failed to save app_setting {key:?}"))?; + Ok(()) + } + + async fn config_tables_empty(&self) -> Result { + let (gp, lp, st): (i64, i64, i64) = ::sqlx::query_as( + "SELECT (SELECT COUNT(*) FROM git_platforms), \ + (SELECT COUNT(*) FROM llm_providers), \ + (SELECT COUNT(*) FROM app_settings)", + ) + .fetch_one(self.pool()) + .await + .context("failed to count config tables")?; + Ok(gp == 0 && lp == 0 && st == 0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::decode_ts; + + async fn fresh_store() -> SqlxStore { + let store = SqlxStore::new_in_memory().await.unwrap(); + store.migrate().await.unwrap(); + store + } + + fn sample_platforms() -> Vec { + vec![ + GitPlatformConfig { + name: "internal".into(), + platform_type: "gitlab".into(), + base_url: "https://gitlab.internal.example".into(), + internal_base_url: "http://gitlab.svc:8080".into(), + token: "glpat-internal-token".into(), + webhook_secret: "wh-internal".into(), + webhook_signing_secret: "whsec_internal".into(), + allowed_projects: vec!["group/a".into(), "group/b".into()], + }, + GitPlatformConfig { + name: "public".into(), + platform_type: "gitlab".into(), + base_url: "https://gitlab.com".into(), + ..Default::default() + }, + ] + } + + fn llm_eq(a: &LLMConfig, b: &LLMConfig) -> bool { + // LLMConfig has no PartialEq (custom Debug masks the key); compare + // field by field. + a.provider == b.provider + && a.model == b.model + && a.api_key == b.api_key + && a.api_base == b.api_base + && a.max_tokens == b.max_tokens + && a.temperature == b.temperature + && a.disable_thinking == b.disable_thinking + } + + #[tokio::test] + async fn git_platforms_round_trip_with_encrypted_secrets() { + let store = fresh_store().await; + let platforms = sample_platforms(); + store.replace_git_platforms(&platforms).await.unwrap(); + + // At rest: every secret column of the populated entry is `enc:`-prefixed. + let (token, wh, whs, raw): (String, String, String, String) = ::sqlx::query_as( + "SELECT token, webhook_secret, webhook_signing_secret, raw FROM git_platforms \ + WHERE name = 'internal'", + ) + .fetch_one(store.pool()) + .await + .unwrap(); + assert!(token.starts_with("enc:"), "token not encrypted: {token}"); + assert!(wh.starts_with("enc:"), "webhook_secret not encrypted"); + assert!(whs.starts_with("enc:"), "webhook_signing_secret not encrypted"); + assert!(!token.contains("glpat-internal-token")); + let raw_json: serde_json::Value = serde_json::from_str(&raw).unwrap(); + assert_eq!(raw_json["allowed_projects"], serde_json::json!(["group/a", "group/b"])); + + // Read back: field-level equality, deterministic name order. + let loaded = store.load_git_platforms().await.unwrap(); + let mut expected = platforms.clone(); + expected.sort_by(|a, b| a.name.cmp(&b.name)); + assert_eq!(loaded, expected); + + // Replace semantics: second replace swaps the whole set atomically. + store.replace_git_platforms(&platforms[1..]).await.unwrap(); + let loaded = store.load_git_platforms().await.unwrap(); + assert_eq!(loaded, vec![platforms[1].clone()]); + } + + #[tokio::test] + async fn git_platforms_legacy_plaintext_passes_through() { + let store = fresh_store().await; + store.replace_git_platforms(&sample_platforms()).await.unwrap(); + // Simulate a legacy / hand-written plaintext secret in the DB. + ::sqlx::query("UPDATE git_platforms SET token = 'plain-legacy-token' WHERE name = 'internal'") + .execute(store.pool()) + .await + .unwrap(); + let loaded = store.load_git_platforms().await.unwrap(); + let internal = loaded.iter().find(|p| p.name == "internal").unwrap(); + assert_eq!(internal.token, "plain-legacy-token"); + } + + #[tokio::test] + async fn llm_providers_round_trip_with_encrypted_api_key_and_order() { + let store = fresh_store().await; + let providers = vec![ + LLMConfig { + provider: "openai".into(), + model: "gpt-5".into(), + api_key: "sk-live-key".into(), + api_base: "https://api.openai.com/v1".into(), + max_tokens: 8192, + temperature: 0.3, + disable_thinking: None, + }, + LLMConfig { + provider: "deepseek".into(), + model: "deepseek-v4-flash".into(), + api_key: "ds-key".into(), + api_base: "https://api.deepseek.com".into(), + max_tokens: 4096, + temperature: 0.7, + disable_thinking: Some(true), + }, + ]; + store.replace_llm_providers(&providers).await.unwrap(); + + // At rest: api_key is `enc:`-prefixed (newly inside the encryption + // boundary — 0.9 stored it plaintext). + let (api_key, raw): (String, String) = + ::sqlx::query_as("SELECT api_key, raw FROM llm_providers WHERE provider = 'openai'") + .fetch_one(store.pool()) + .await + .unwrap(); + assert!(api_key.starts_with("enc:"), "api_key not encrypted: {api_key}"); + assert!(!api_key.contains("sk-live-key")); + assert_eq!(serde_json::from_str::(&raw).unwrap()["position"], 0); + + // Read back: order preserved (openai first), field-level equality. + let loaded = store.load_llm_providers().await.unwrap(); + assert_eq!(loaded.len(), 2); + assert!(llm_eq(&loaded[0], &providers[0]), "entry 0 mismatch: {loaded:?}"); + assert!(llm_eq(&loaded[1], &providers[1]), "entry 1 mismatch: {loaded:?}"); + + // Legacy plaintext api_key passes through on read. + ::sqlx::query("UPDATE llm_providers SET api_key = 'plain-legacy-key' WHERE provider = 'openai'") + .execute(store.pool()) + .await + .unwrap(); + let loaded = store.load_llm_providers().await.unwrap(); + assert_eq!(loaded[0].api_key, "plain-legacy-key"); + } + + #[tokio::test] + async fn legacy_gitlab_round_trip_with_encrypted_fields() { + let store = fresh_store().await; + + // Missing row → all-empty default. + let loaded = store.load_legacy_gitlab().await.unwrap(); + assert_eq!(loaded.token, ""); + assert_eq!(loaded.webhook_secret, ""); + assert_eq!(loaded.webhook_signing_secret, ""); + + let gitlab = PersistedGitlabConfig { + token: "glpat-legacy".into(), + webhook_secret: "wh-legacy".into(), + webhook_signing_secret: String::new(), + }; + store.save_legacy_gitlab(&gitlab).await.unwrap(); + + // At rest: every non-empty field inside the JSON is `enc:`-prefixed; + // empty stays empty. + let raw: String = ::sqlx::query_scalar("SELECT value FROM app_settings WHERE key = 'gitlab'") + .fetch_one(store.pool()) + .await + .unwrap(); + let value: serde_json::Value = serde_json::from_str(&raw).unwrap(); + assert!(value["token"].as_str().unwrap().starts_with("enc:")); + assert!(value["webhook_secret"].as_str().unwrap().starts_with("enc:")); + assert_eq!(value["webhook_signing_secret"], ""); + assert!(!raw.contains("glpat-legacy")); + + let loaded = store.load_legacy_gitlab().await.unwrap(); + assert_eq!(loaded.token, "glpat-legacy"); + assert_eq!(loaded.webhook_secret, "wh-legacy"); + assert_eq!(loaded.webhook_signing_secret, ""); + } + + #[tokio::test] + async fn app_settings_arbitrary_json_round_trip() { + let store = fresh_store().await; + + assert_eq!(store.load_setting("ui").await.unwrap(), None); + + let ui = serde_json::json!({ + "rules": {"maxFindings": 50}, + "advanced": {"parallelExperts": 4}, + "nested": {"list": [1, 2, 3], "flag": true} + }); + store.save_setting("ui", &ui).await.unwrap(); + assert_eq!(store.load_setting("ui").await.unwrap(), Some(ui)); + + // Upsert overwrites. + let updated = serde_json::json!({"rules": {"maxFindings": 20}}); + store.save_setting("ui", &updated).await.unwrap(); + assert_eq!(store.load_setting("ui").await.unwrap(), Some(updated)); + + // updated_at is a decodable RFC 3339 timestamp. + let ts: String = ::sqlx::query_scalar("SELECT updated_at FROM app_settings WHERE key = 'ui'") + .fetch_one(store.pool()) + .await + .unwrap(); + decode_ts(&ts).unwrap(); + } + + #[tokio::test] + async fn config_tables_empty_flag() { + let store = fresh_store().await; + assert!(store.config_tables_empty().await.unwrap()); + store.save_setting("ui", &serde_json::json!({})).await.unwrap(); + assert!(!store.config_tables_empty().await.unwrap()); + } +} diff --git a/src/store/traits.rs b/src/store/traits.rs index 3b2c164..1d7a3c3 100644 --- a/src/store/traits.rs +++ b/src/store/traits.rs @@ -2,5 +2,61 @@ //! `ConfigStore` (git_platforms / llm_providers / app_settings), //! `DiscussionStore` (mr_discussions). //! -//! Placeholder module — trait definitions land in later steps -//! (design/persistence.md §4.2). +//! Only `ConfigStore` is defined so far (step 2 of the 0.10.0 persistence +//! rollout); the other two land with their implementations. +//! +//! Semantics follow `UiStateFile` (design/persistence.md §4.2, §6.2): each +//! domain is read and replaced AS A WHOLE — `PUT /config` resolves the full +//! intended set in memory first, then persists it atomically. Per-row partial +//! updates are deliberately not offered. + +use anyhow::Result; +use async_trait::async_trait; + +use crate::models::{GitPlatformConfig, LLMConfig}; +use crate::server::api::config::persist::PersistedGitlabConfig; + +/// Persistence boundary for UI-managed configuration. +/// +/// All values crossing this trait are LIVE (plaintext) domain values; the +/// `enc:` at-rest encryption happens inside the store implementation +/// (`rows.rs`), keyed by the per-config-dir `secrets.key`. +#[async_trait] +pub trait ConfigStore: Send + Sync { + /// All configured git platform instances (live secrets, deterministic + /// order by `name`). + async fn load_git_platforms(&self) -> Result>; + + /// Atomically replace the whole git platform set (mirrors how + /// `PUT /config` resolves and persists the full list). Empty slice = + /// clear the table. + async fn replace_git_platforms(&self, platforms: &[GitPlatformConfig]) -> Result<()>; + + /// All configured LLM providers (live API keys). List order is + /// preserved: the first entry is the fallback primary provider + /// (`sync_llm_projection`, persist.rs), so order is round-tripped via a + /// `position` marker in the row's `raw` JSON. + async fn load_llm_providers(&self) -> Result>; + + /// Atomically replace the whole LLM provider set. + async fn replace_llm_providers(&self, providers: &[LLMConfig]) -> Result<()>; + + /// Legacy GitLab credentials (app_settings key `gitlab`). Missing row = + /// all-empty default. + async fn load_legacy_gitlab(&self) -> Result; + + /// Persist legacy GitLab credentials. All three fields are individually + /// `enc:`-encrypted inside the stored JSON (§3.2 note). + async fn save_legacy_gitlab(&self, gitlab: &PersistedGitlabConfig) -> Result<()>; + + /// Arbitrary JSON setting from `app_settings` (e.g. the `ui` projection). + async fn load_setting(&self, key: &str) -> Result>; + + /// Upsert an arbitrary JSON setting. + async fn save_setting(&self, key: &str, value: &serde_json::Value) -> Result<()>; + + /// True when git_platforms + llm_providers + app_settings are all empty + /// — the trigger condition for the one-shot `ui-state.toml` import + /// (design/persistence.md §6.1 step 3). + async fn config_tables_empty(&self) -> Result; +} From 105d3f2f914c6addb793357bb6bbe0a89ed60573 Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 11:17:28 +0800 Subject: [PATCH 04/36] docs(design): align persistence.md dialect table with Any-driver smoke-test findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - §3.1 布尔行: BOOLEAN 证伪, DDL 改 INTEGER 0/1 (Any 无法解码 SQLite Bool 声明列) - §3.2 enabled 列同步改 INTEGER (时间戳 TEXT 化与 §11 验证点 A 勾选为杜衡并行修订) --- design/persistence.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/design/persistence.md b/design/persistence.md index f5fef46..1116dc3 100644 --- a/design/persistence.md +++ b/design/persistence.md @@ -70,12 +70,14 @@ | upsert | `ON CONFLICT ... DO UPDATE/NOTHING` | 同语法（≥3.24） | 两端一致，直接用；sqlx 内置 libsqlite3 版本远高于此 | | `RETURNING` | 支持 | ≥3.35 支持 | **一律不用**。主键全部由 Rust 侧生成（UUID v4），写后无需回读；避免 Any 下两端 decode 行为差异 | | JSON 列 | 原生 JSONB | TEXT | **DDL 用 TEXT，绑定用 `String`**：store 层 `serde_json::to_string` 后按 TEXT 绑定，读出再 `from_str`。若声明 PG JSONB 列而 SQLite 是 TEXT，`serde_json::Value` 在 PG 端会按 JSONB 编码、绑到 TEXT 列报类型错——应用层序列化是唯一两头都稳的做法 | -| 布尔 | 原生 BOOL | 0/1 | DDL `BOOLEAN`，sqlx Any 的 `bool` 编解码两端兼容 | -| 时间戳 | TIMESTAMPTZ | 无原生类型（NUMERIC 亲和） | DDL `TIMESTAMP`；**值一律 Rust 侧 chrono 生成**，不写 `CURRENT_TIMESTAMP` 默认值，两端时间戳格式由应用层统一 | +| 布尔 | 原生 BOOL | 0/1 | **DDL 用 `INTEGER` 存 0/1**，绑定侧转 `bool`。原方案 `BOOLEAN` 被证伪：Any 驱动对 SQLite 只认 Null/Int4/Integer/Float/Blob/Text 五类声明类型，`BOOLEAN` 列读出直接报错（验证点 A 实测） | +| 时间戳 | TEXT | TEXT | **DDL 用 TEXT**，存 Rust 侧 chrono 生成的固定宽度 RFC 3339 UTC 串（如 `2026-09-03T10:00:00.000000Z`），**字典序 == 时间序**。原因（验证点 A/D 落地结论）：sqlx 0.8 Any 驱动没有 chrono 的 `Type` 实现，且 SQLite 端拒绝对声明类型为 `TIMESTAMP` 的列做 String 解码（smoke test 实测）；PG 端 TEXT 列无需 CAST | | 模糊搜索 | `ILIKE` | `LIKE` 仅 ASCII 不敏感 | 统一 `LOWER(col) LIKE LOWER(?)`，行为两端一致 | | 外键 | 默认启用 | 需 `PRAGMA foreign_keys=ON` | SQLite 连接串带 `?...` 参数或建池后执行 PRAGMA（见 §4.3） | | 自增主键 | SERIAL/IDENTITY | AUTOINCREMENT | **都不用**：全部自然键/UUID 文本主键，绕开方言差异 | +补充：`uuid` 与 `chrono` 同样无 `Type` 实现，UUID 主键按 TEXT 绑定（值仍由 Rust 侧生成，同 `RETURNING` 行约定），无影响。 + ### 3.2 建表 SQL 草案（`migrations/0001_init.sql`） ```sql @@ -92,9 +94,9 @@ CREATE TABLE reviews ( result TEXT, -- ReviewOutput JSON error TEXT, progress INTEGER, -- 0-100，仅终态时快照；进行中的实时进度不入库 - created_at TIMESTAMP NOT NULL, - started_at TIMESTAMP, - completed_at TIMESTAMP + created_at TEXT NOT NULL, + started_at TEXT, + completed_at TEXT ); CREATE INDEX idx_reviews_created_at ON reviews (created_at DESC); CREATE INDEX idx_reviews_state ON reviews (state); @@ -107,7 +109,7 @@ CREATE TABLE expert_reports ( report TEXT NOT NULL, -- ExpertReport JSON duration_ms INTEGER, -- 首版可为 NULL：TaskEntry 目前不记 per-expert 耗时， -- 需执行器补计时后再填充（见 §5.4 注意点） - created_at TIMESTAMP NOT NULL, + created_at TEXT NOT NULL, PRIMARY KEY (task_id, expert_name) ); @@ -119,8 +121,8 @@ CREATE TABLE mr_discussions ( note_id BIGINT NOT NULL, author TEXT NOT NULL DEFAULT '', body TEXT NOT NULL, - created_at TIMESTAMP NOT NULL, -- note 的创建时间，非入库时间 - ingested_at TIMESTAMP NOT NULL, -- 入库时间，排序兜底 + created_at TEXT NOT NULL, -- note 的创建时间，非入库时间 + ingested_at TEXT NOT NULL, -- 入库时间，排序兜底 PRIMARY KEY (platform, project, mr_iid, note_id) -- 幂等键 ); CREATE INDEX idx_mr_discussions_mr ON mr_discussions (platform, project, mr_iid, created_at); @@ -132,7 +134,7 @@ CREATE TABLE review_contexts ( content TEXT NOT NULL, -- 渲染后的上下文本（前缀稳定） content_hash TEXT NOT NULL, -- sha256 hex；同 MR 二次评审 hash 相同即复用 token_estimate INTEGER NOT NULL DEFAULT 0, - created_at TIMESTAMP NOT NULL, + created_at TEXT NOT NULL, PRIMARY KEY (task_id, kind) ); CREATE INDEX idx_review_contexts_hash ON review_contexts (content_hash); @@ -147,9 +149,9 @@ CREATE TABLE git_platforms ( token TEXT NOT NULL DEFAULT '', -- enc: 加密 webhook_secret TEXT NOT NULL DEFAULT '', -- enc: 加密 webhook_signing_secret TEXT NOT NULL DEFAULT '', -- enc: 加密 - enabled BOOLEAN NOT NULL DEFAULT TRUE, + enabled INTEGER NOT NULL DEFAULT 1, -- 布尔列用 INTEGER 0/1，见 §3.1 布尔行 raw TEXT NOT NULL DEFAULT '{}', -- 扩展兜底：allowed_projects 等未列化字段 - updated_at TIMESTAMP NOT NULL + updated_at TEXT NOT NULL ); -- ── LLM 实例（[[llm]] 区段入库；api_key 顺带收进加密边界）── @@ -162,7 +164,7 @@ CREATE TABLE llm_providers ( max_tokens INTEGER NOT NULL DEFAULT 4096, temperature REAL NOT NULL DEFAULT 0.7, raw TEXT NOT NULL DEFAULT '{}', -- 扩展兜底：disable_thinking 等 - updated_at TIMESTAMP NOT NULL + updated_at TEXT NOT NULL ); CREATE UNIQUE INDEX idx_llm_providers_provider ON llm_providers (provider); @@ -170,7 +172,7 @@ CREATE UNIQUE INDEX idx_llm_providers_provider ON llm_providers (provider); CREATE TABLE app_settings ( key TEXT PRIMARY KEY, -- 如 'ui'、'gitlab'、'rules'、'advanced' value TEXT NOT NULL, -- JSON - updated_at TIMESTAMP NOT NULL + updated_at TEXT NOT NULL ); ``` @@ -359,10 +361,10 @@ output.aggregated.map(|a| a.markdown) ## 11. 待验证点（实现前确认，不确定处不猜） -- **验证点 A**：sqlx 0.8 `Any` 驱动的 `?` 占位符 PG 翻译行为、以及 `Migrator` 在 `AnyPool` 上的行为（含 `_sqlx_migrations` 锁表在 SQLite 上的表现）。方法：步骤 1 的 smoke test，双后端各跑。 +- **验证点 A**（✅ 已验证，sqlx 0.8.6 smoke test）：`?` 占位符 PG 翻译、`Migrator` 在 `AnyPool` 上的行为均正常；SQLite 侧通过，PG 侧留 `#[ignore]` 入口待有实例时跑。附带结论：Any 驱动无 chrono/uuid 的 `Type` 实现，SQLite 拒绝对 `TIMESTAMP` 声明列做 String 解码——时间戳/uuid 一律 TEXT 绑定（§3.1 已按此定稿）。 - **验证点 B**：`config/resolver/` 是否从 config.toml 承载 `git_platforms`（§6.3 表中标注待核实）。方法：`Grep "git_platforms" src/config/`。 - **验证点 C**：评审报告的固定前缀常量位置（§7.1 自噬防护条件 a）。方法：`Grep` publisher/output 模块的报告头部模板。 -- **验证点 D**：`Any` 驱动下 `chrono::DateTime` 绑到 SQLite `TIMESTAMP` 列的存储格式与排序正确性（字典序 = 时间序是分页 `ORDER BY created_at` 的前提）。方法：步骤 4 的 round-trip 测试里断言排序。 +- **验证点 D**（✅ 已验证）：通过。固定宽度 RFC 3339 UTC 串按 TEXT 存储，字典序 == 时间序，`ORDER BY created_at` 排序正确（分页前提成立）。 ## 12. 验收标准清单 From c18bc9d8ef8038fae38e5986fc214625f84388ee Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 11:35:50 +0800 Subject: [PATCH 05/36] feat(config): one-shot ui-state.toml import, DB-backed replay, PUT to DB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Startup sequence (design/persistence.md §6.1, strict order, cli/app.rs): 1. bootstrap_database(): REVIEW_DISABLE_DB=1 bypasses to 0.9 behaviour (warn); DATABASE_URL set-but-unreachable is a hard startup error, never a silent SQLite fallback (§9); no DATABASE_URL → embedded SQLite at {config_dir}/review.db. migrate() failure propagates (exit non-zero). 2. (placeholder comment) §5.3 interrupted sweep — 梁序 step 4. 3. import_ui_state_into_db(): triggers only when all three config tables are empty AND ui-state.toml exists; the whole import is ONE transaction (ConfigStore::save_ui_state), renamed to ui-state.toml.migrated only after every table is written; any error keeps the file and falls back to the file replay path. 4. load_and_apply_ui_state_from_db(): DB rows reassemble into a UiStateFile and go through the SAME replay_payload + apply_ui_config path, so env precedence and masked projections are literally shared with the file path. Empty DB → Ok(false) → file replay fallback. PUT /config (§6.2): state.db set → ConfigStore::save_ui_state (single tx); else the 0.9 file path. UiStateFile::from_applied env filtering is reused unchanged — env/CLI values are never persisted anywhere. Persist failure returns 500, same as a file-write failure today. AppState gains pub db: Option> (None default; the sync AppState::new() test path is untouched). Tests: import happy path (plaintext-LLM-key legacy file → tables populated, file renamed, all four secret columns enc: at rest, DB replay ≡ file replay); env precedence matrix against the DB source (env LLM wins wholesale; gitlab env is fallback-only / DB wins when set); failed import (dup UNIQUE name) rolls back completely and keeps the file; put_config_persists_to_db_instead_of_file; REVIEW_DISABLE_DB flag parsing. --- src/cli/app.rs | 66 ++++- src/server/api/config/persist.rs | 421 ++++++++++++++++++++++++++++++- src/server/api/config/put.rs | 25 +- src/server/state.rs | 6 + src/store/sqlx.rs | 185 +++++++++----- src/store/traits.rs | 14 +- 6 files changed, 631 insertions(+), 86 deletions(-) diff --git a/src/cli/app.rs b/src/cli/app.rs index 71cab47..c6b195f 100644 --- a/src/cli/app.rs +++ b/src/cli/app.rs @@ -279,7 +279,47 @@ pub async fn run() -> Result<()> { llm_from_env: !env_llm_entries.is_empty(), llm_entries: env_llm_entries, }); + // 0.10.0 persistence (design/persistence.md §6.1, strict order): + // 1) resolve DB URL → pool → migrate (failure aborts startup; + // REVIEW_DISABLE_DB=1 bypasses to 0.9 behaviour); + // 2) TODO(梁序, step 4): §5.3 interrupted sweep — UPDATE reviews + // SET state='failed', error='interrupted: server restarted', + // completed_at=? WHERE state IN ('pending','running') goes + // here, after migrate and before the config replay; + // 3) one-shot ui-state.toml import (single transaction; failure + // keeps the file and falls back to the file replay below); + // 4) replay the DB state through the same apply_ui_config path. + app_state.db = review_engine::server::api::config::persist::bootstrap_database() + .await? + .map(Arc::new); let state = Arc::new(app_state); + let mut config_replayed = false; + if let Some(store) = state.db.clone() { + let overrides = state.ui_state_env.clone().unwrap_or_default(); + if let Some(path) = state.ui_state_path.clone() { + match review_engine::server::api::config::persist::import_ui_state_into_db(&store, &path).await { + Ok(true) => {} + Ok(false) => {} + Err(e) => tracing::error!( + "ui-state.toml import failed: {e:#}; the file is untouched, \ + falling back to the file replay path" + ), + } + } + match review_engine::server::api::config::persist::load_and_apply_ui_state_from_db( + &state, &store, &overrides, + ) + .await + { + Ok(applied) => { + config_replayed = applied; + if applied { + tracing::info!("applied UI state from the database"); + } + } + Err(e) => tracing::warn!("failed to replay UI state from the database: {e:#}"), + } + } let dispatcher = review_engine::server::dispatcher::MrDispatcher::persistent(); let mut handlers: Vec> = vec![]; let gitlab_token = gitlab_token_opt.clone().unwrap_or_default(); @@ -301,16 +341,22 @@ pub async fn run() -> Result<()> { // `--gitlab-token` / `GITLAB_TOKEN` must land there regardless // of webhook setup. review_engine::server::gitlab::init_gitlab_runtime(&gitlab_handler); - // Load the persisted UI state (ui-state.toml) and apply it - // through the same code path as PUT /config, so hot-apply and - // cold-start semantics are identical. A missing/corrupt file - // never blocks startup — it just reverts to config.toml/env. - if let Some(path) = state.ui_state_path.clone() { - let overrides = state.ui_state_env.clone().unwrap_or_default(); - match review_engine::server::api::config::persist::load_and_apply_ui_state(&state, &path, &overrides) { - Ok(true) => tracing::info!("applied persisted UI state from {}", path.display()), - Ok(false) => {} - Err(e) => tracing::warn!("failed to load persisted UI state from {}: {e:#}", path.display()), + // Load the persisted UI state and apply it through the same code + // path as PUT /config, so hot-apply and cold-start semantics are + // identical. When the DB replay above already applied (or the DB + // is active but empty after a failed import), the file replay is + // the fallback. A missing/corrupt file never blocks startup — it + // just reverts to config.toml/env. + if !config_replayed { + if let Some(path) = state.ui_state_path.clone() { + let overrides = state.ui_state_env.clone().unwrap_or_default(); + match review_engine::server::api::config::persist::load_and_apply_ui_state( + &state, &path, &overrides, + ) { + Ok(true) => tracing::info!("applied persisted UI state from {}", path.display()), + Ok(false) => {} + Err(e) => tracing::warn!("failed to load persisted UI state from {}: {e:#}", path.display()), + } } } // Mount /webhook/gitlab unconditionally: verification is resolved diff --git a/src/server/api/config/persist.rs b/src/server/api/config/persist.rs index 5285b1e..67b6b83 100644 --- a/src/server/api/config/persist.rs +++ b/src/server/api/config/persist.rs @@ -31,11 +31,14 @@ use std::path::{Path, PathBuf}; +use anyhow::Context as _; use serde::{Deserialize, Serialize}; use crate::config::secrets::{self, ENC_PREFIX}; use crate::models::{GitPlatformConfig, LLMConfig}; use crate::server::AppState; +use crate::store::traits::ConfigStore; +use crate::store::SqlxStore; use super::put::AppliedConfig; use super::types::{UiConfig, UiGitLabConfig, UiGitPlatformConfig, UiLlmProviderConfig, API_KEY_MASK}; @@ -367,10 +370,139 @@ pub fn load_and_apply_ui_state(state: &AppState, path: &Path, overrides: &UiStat let Some(file) = load_ui_state(path)? else { return Ok(false); }; - let payload = replay_payload(&file, overrides); + apply_replay(state, &file, overrides, &path.display().to_string())?; + Ok(true) +} + +/// Shared tail of both replay paths: build the PUT-equivalent payload and +/// push it through `apply_ui_config`. `source_desc` only feeds error text. +fn apply_replay( + state: &AppState, + file: &UiStateFile, + overrides: &UiStateEnvOverrides, + source_desc: &str, +) -> anyhow::Result<()> { + let payload = replay_payload(file, overrides); super::put::apply_ui_config(state, &payload).map_err(|(status, axum::Json(body))| { - anyhow::anyhow!("failed to apply {} (HTTP {}): {}", path.display(), status, body) + anyhow::anyhow!("failed to apply {source_desc} (HTTP {status}): {body}") })?; + Ok(()) +} + +// ── 0.10.0 database-backed persistence (design/persistence.md §6) ── + +/// Suffix of the post-import backup: `ui-state.toml` → +/// `ui-state.toml.migrated` (kept, never deleted — §6.1). +pub const MIGRATED_SUFFIX: &str = ".migrated"; + +/// `ui-state.toml` → `ui-state.toml.migrated`. +pub fn migrated_path(path: &Path) -> PathBuf { + let mut os = path.as_os_str().to_owned(); + os.push(MIGRATED_SUFFIX); + PathBuf::from(os) +} + +/// True when `REVIEW_DISABLE_DB` requests the 0.9 escape hatch (§9). +/// Pure function over the env value for testability. +pub fn db_disabled_flag(value: Option<&str>) -> bool { + matches!( + value.map(|v| v.trim().to_ascii_lowercase()), + Some(v) if v == "1" || v == "true" || v == "yes" + ) +} + +/// Startup step 1 (§6.1): resolve the DB URL, build the pool, run +/// migrations. `Ok(None)` = persistence disabled (escape hatch, or no +/// config dir resolvable — the 0.9 "persistence off" case); an unreachable +/// database is a hard startup error, NEVER a silent SQLite fallback (§9). +pub async fn bootstrap_database() -> anyhow::Result> { + if db_disabled_flag(std::env::var("REVIEW_DISABLE_DB").ok().as_deref()) { + tracing::warn!("persistence disabled via REVIEW_DISABLE_DB — running with 0.9 in-memory + file behaviour"); + return Ok(None); + } + let store = match std::env::var("DATABASE_URL") { + Ok(url) if !url.is_empty() => SqlxStore::connect(&url).await.with_context(|| { + "DATABASE_URL is set but the database is unreachable; \ + refusing to silently fall back to embedded SQLite (fix the connection, \ + unset DATABASE_URL, or set REVIEW_DISABLE_DB=1 to bypass persistence)" + .to_string() + })?, + _ => { + let Some(state_path) = resolve_ui_state_path() else { + tracing::warn!("no config dir resolvable — persistence disabled (0.9 behaviour)"); + return Ok(None); + }; + let dir = state_path + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + SqlxStore::connect_default(&dir).await? + } + }; + store.migrate().await?; + Ok(Some(store)) +} + +/// Startup step 3 (§6.1): one-shot import of `ui-state.toml` into the DB. +/// +/// Triggers only when the three config tables are completely empty AND the +/// file exists. The whole import is ONE transaction (`save_ui_state`), so a +/// mid-import failure rolls back cleanly; the file is renamed to +/// `.migrated` only after every table has been written. Any error leaves +/// the file untouched so the caller falls back to the file replay path and +/// the next startup retries the import. +pub async fn import_ui_state_into_db(store: &SqlxStore, path: &Path) -> anyhow::Result { + if !store.config_tables_empty().await? { + return Ok(false); + } + let Some(file) = load_ui_state(path)? else { + return Ok(false); + }; + store.save_ui_state(&file).await?; + match std::fs::rename(path, migrated_path(path)) { + Ok(()) => tracing::info!( + "imported ui-state.toml into the database; backup at {}", + migrated_path(path).display() + ), + // The data is safely in the DB and DB replay wins from here on; a + // failed rename only means the backup was not created. + Err(e) => tracing::warn!("import succeeded but renaming {} failed: {e}", path.display()), + } + Ok(true) +} + +/// Startup step 4 (§6.1): replay the UI state from the DB through the SAME +/// `apply_ui_config` path as `PUT /config`. The DB rows are reassembled into +/// a [`UiStateFile`] so the replay payload builder (env precedence, masked +/// projections) is literally shared with the file path. `Ok(false)` = DB +/// holds no configuration at all (caller falls back to the file replay). +pub async fn load_and_apply_ui_state_from_db( + state: &AppState, + store: &SqlxStore, + overrides: &UiStateEnvOverrides, +) -> anyhow::Result { + let ui: Option = store + .load_setting("ui") + .await? + .map(|v| serde_json::from_value(v).context("app_settings row 'ui' is not a valid UiConfig")) + .transpose()?; + let file = UiStateFile { + ui, + llm: store.load_llm_providers().await?, + git_platforms: store.load_git_platforms().await?, + gitlab: store.load_legacy_gitlab().await?, + }; + let gitlab = &file.gitlab; + let empty = file.ui.is_none() + && file.llm.is_empty() + && file.git_platforms.is_empty() + && gitlab.token.is_empty() + && gitlab.webhook_secret.is_empty() + && gitlab.webhook_signing_secret.is_empty(); + if empty { + return Ok(false); + } + apply_replay(state, &file, overrides, "the database UI state")?; Ok(true) } @@ -1430,4 +1562,289 @@ webhook_secret = "legacy-wh-plain" assert_eq!(llm[0].api_key, "tp-REALKEY"); assert_eq!(llm[0].api_base, "https://token-plan-cn.xiaomimimo.com/v1"); } + + // ── 0.10.0: DB-backed persistence (import / DB replay / escape hatch) ── + + async fn fresh_db() -> SqlxStore { + let store = SqlxStore::new_in_memory().await.unwrap(); + store.migrate().await.unwrap(); + store + } + + /// (a) One-shot import: an old ui-state.toml (plaintext LLM key on disk, + /// encrypted git token) imports into empty config tables; the file is + /// renamed to .migrated; every secret column at rest is `enc:`; and the + /// DB replay produces the same effective configuration as the 0.9 file + /// replay. + #[tokio::test] + async fn import_then_db_replay_matches_file_replay() { + let _lock = RUNTIME_TEST_LOCK.lock().await; + let _guard = RuntimeGuard::new(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(UI_STATE_FILE_NAME); + // save_ui_state writes the legacy on-disk shape: git secrets enc:, + // the LLM api_key PLAINTEXT (0.9 threat model, persist.rs:27-29). + save_ui_state(&path, &sample_state_file()).unwrap(); + let on_disk = std::fs::read_to_string(&path).unwrap(); + assert!( + on_disk.contains("sk-live"), + "precondition: 0.9 stores the LLM key in plaintext" + ); + + let store = fresh_db().await; + assert!(import_ui_state_into_db(&store, &path).await.unwrap()); + + assert!(!path.exists(), "original file must be renamed away"); + assert!(migrated_path(&path).exists(), "backup must be kept"); + assert!(!store.config_tables_empty().await.unwrap()); + + // At rest: all four secret columns are enc:-prefixed — the LLM key + // included (newly inside the encryption boundary). + let api_key: String = sqlx::query_scalar("SELECT api_key FROM llm_providers") + .fetch_one(store.pool()) + .await + .unwrap(); + assert!( + api_key.starts_with("enc:"), + "api_key must be encrypted at rest: {api_key}" + ); + assert!(!api_key.contains("sk-live")); + let token: String = sqlx::query_scalar("SELECT token FROM git_platforms") + .fetch_one(store.pool()) + .await + .unwrap(); + assert!(token.starts_with("enc:"), "platform token must stay encrypted"); + let gitlab_raw: String = sqlx::query_scalar("SELECT value FROM app_settings WHERE key = 'gitlab'") + .fetch_one(store.pool()) + .await + .unwrap(); + let gitlab_json: serde_json::Value = serde_json::from_str(&gitlab_raw).unwrap(); + assert!(gitlab_json["token"].as_str().unwrap().starts_with("enc:")); + assert!(gitlab_json["webhook_secret"].as_str().unwrap().starts_with("enc:")); + assert_eq!(gitlab_json["webhook_signing_secret"], ""); + + // Replay equivalence: DB replay vs 0.9 file replay (from the backup) + // must land the same effective configuration. + let state_db = Arc::new(fresh_state(vec![])); + assert!( + load_and_apply_ui_state_from_db(&state_db, &store, &UiStateEnvOverrides::default()) + .await + .unwrap() + ); + let state_file = Arc::new(fresh_state(vec![])); + assert!(load_and_apply_ui_state(&state_file, &migrated_path(&path), &UiStateEnvOverrides::default()).unwrap()); + let db_llm = state_db.llm_configs.read().unwrap().clone(); + let file_llm = state_file.llm_configs.read().unwrap().clone(); + assert_eq!(db_llm.len(), 1); + assert_eq!(db_llm[0].api_key, "sk-live"); + assert_eq!(db_llm[0].api_key, file_llm[0].api_key); + assert_eq!( + state_db.git_platforms.read().unwrap().clone(), + state_file.git_platforms.read().unwrap().clone() + ); + assert_eq!( + serde_json::to_value(&*state_db.ui_config.read().unwrap()).unwrap(), + serde_json::to_value(&*state_file.ui_config.read().unwrap()).unwrap(), + "GET /config projection must be identical for DB and file replay" + ); + } + + /// (b) env precedence matrix against the DB source: env-seeded LLM list + /// wins wholesale (DB llm section NOT replayed); legacy gitlab env/CLI + /// values are fallback-only (used only when the DB value is empty). + #[tokio::test] + async fn db_replay_env_precedence_matrix() { + let _lock = RUNTIME_TEST_LOCK.lock().await; + let _guard = RuntimeGuard::new(); + let env_llm = LLMConfig { + provider: "openai".to_string(), + model: "gpt-env".to_string(), + api_key: "sk-env".to_string(), + api_base: "https://api.openai.com/v1".to_string(), + max_tokens: 4096, + temperature: 0.7, + disable_thinking: None, + }; + + // Row 1: llm_from_env — DB llm entries must not touch the runtime. + let store = fresh_db().await; + store + .replace_llm_providers(&[LLMConfig { + provider: "openai".to_string(), + model: "gpt-db".to_string(), + api_key: "sk-db".to_string(), + api_base: String::new(), + max_tokens: 4096, + temperature: 0.7, + disable_thinking: None, + }]) + .await + .unwrap(); + let state = Arc::new(fresh_state(vec![env_llm.clone()])); + let overrides = UiStateEnvOverrides { + llm_from_env: true, + llm_entries: vec![env_llm.clone()], + ..Default::default() + }; + assert!(load_and_apply_ui_state_from_db(&state, &store, &overrides) + .await + .unwrap()); + let llm = state.llm_configs.read().unwrap().clone(); + assert_eq!(llm.len(), 1); + assert_eq!(llm[0].api_key, "sk-env", "env provider list wins wholesale over the DB"); + assert_eq!(llm[0].model, "gpt-env"); + + // Row 2: DB gitlab empty → env/CLI fills in (fallback-only). + *crate::server::gitlab::gitlab_runtime().write().unwrap() = crate::server::gitlab::GitLabRuntimeConfig { + webhook_secret: String::new(), + signing_secret: None, + signing_key: None, + token: String::new(), + }; + let state2 = Arc::new(fresh_state(vec![])); + let overrides2 = UiStateEnvOverrides { + gitlab_token: Some("glpat-env".to_string()), + gitlab_webhook_secret: Some("wh-env".to_string()), + ..Default::default() + }; + assert!(load_and_apply_ui_state_from_db(&state2, &store, &overrides2) + .await + .unwrap()); + let rt = crate::server::gitlab::gitlab_runtime().read().unwrap().clone(); + assert_eq!(rt.token, "glpat-env", "env fallback fills an empty DB token"); + assert_eq!(rt.webhook_secret, "wh-env"); + + // Row 3: DB gitlab set → DB is authoritative, env is ignored. + store + .save_legacy_gitlab(&PersistedGitlabConfig { + token: "glpat-db".to_string(), + webhook_secret: "wh-db".to_string(), + webhook_signing_secret: String::new(), + }) + .await + .unwrap(); + *crate::server::gitlab::gitlab_runtime().write().unwrap() = crate::server::gitlab::GitLabRuntimeConfig { + webhook_secret: String::new(), + signing_secret: None, + signing_key: None, + token: String::new(), + }; + let state3 = Arc::new(fresh_state(vec![])); + assert!(load_and_apply_ui_state_from_db(&state3, &store, &overrides2) + .await + .unwrap()); + let rt = crate::server::gitlab::gitlab_runtime().read().unwrap().clone(); + assert_eq!(rt.token, "glpat-db", "DB token must beat the env override"); + assert_eq!(rt.webhook_secret, "wh-db"); + } + + /// (c) A mid-import failure rolls back the whole transaction: no partial + /// rows, the file is NOT renamed, and the file replay fallback still + /// works. Injected fault: two git platforms with the same `name` violate + /// the UNIQUE constraint on the second insert. + #[tokio::test] + async fn failed_import_rolls_back_and_keeps_file() { + let _lock = RUNTIME_TEST_LOCK.lock().await; + let _guard = RuntimeGuard::new(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(UI_STATE_FILE_NAME); + let mut file = sample_state_file(); + file.git_platforms.push(GitPlatformConfig { + name: "testbed".to_string(), // duplicate of the first entry + base_url: "https://dup.example.com".to_string(), + ..Default::default() + }); + save_ui_state(&path, &file).unwrap(); + + let store = fresh_db().await; + let err = import_ui_state_into_db(&store, &path).await.unwrap_err(); + assert!( + format!("{err:#}").contains("testbed"), + "error must name the failing row: {err:#}" + ); + assert!(path.exists(), "failed import must NOT rename the file"); + assert!(!migrated_path(&path).exists()); + assert!( + store.config_tables_empty().await.unwrap(), + "the transaction must roll back completely (no partial import)" + ); + + // Fallback: the file replay path still applies the (valid parts of + // the) configuration — same as a corrupt-DB 0.9 startup. + let state = Arc::new(fresh_state(vec![])); + assert!(load_and_apply_ui_state(&state, &path, &UiStateEnvOverrides::default()).unwrap()); + assert_eq!(state.llm_configs.read().unwrap()[0].api_key, "sk-live"); + } + + /// PUT /config with a DB attached persists to the DB (not to the file) + /// and stores the resolved live key encrypted. + #[tokio::test] + async fn put_config_persists_to_db_instead_of_file() { + let _lock = RUNTIME_TEST_LOCK.lock().await; + let _guard = RuntimeGuard::new(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(UI_STATE_FILE_NAME); + let store = fresh_db().await; + + let mut state = fresh_state(vec![]); + state.ui_state_path = Some(path.clone()); + state.db = Some(Arc::new(store.clone())); + let state = Arc::new(state); + let resp = crate::server::api::config::put_config( + axum::extract::State(state.clone()), + axum::Json(serde_json::json!({ + "llm": { + "openaiApiKey": "sk-live-db", + "apiBaseUrl": "https://api.openai.com/v1", + "defaultModel": "gpt-4o" + }, + "gitlab": { "apiToken": "glpat-db", "webhookSecret": "wh-db" } + })), + ) + .await + .into_response(); + assert_eq!(resp.status(), axum::http::StatusCode::OK); + + assert!(!path.exists(), "no ui-state.toml may be written when the DB is active"); + let llm = store.load_llm_providers().await.unwrap(); + assert_eq!(llm.len(), 1); + assert_eq!( + llm[0].api_key, "sk-live-db", + "masked/resolved live key must land in the DB" + ); + let at_rest: String = sqlx::query_scalar("SELECT api_key FROM llm_providers") + .fetch_one(store.pool()) + .await + .unwrap(); + assert!(at_rest.starts_with("enc:")); + let gitlab = store.load_legacy_gitlab().await.unwrap(); + assert_eq!(gitlab.token, "glpat-db"); + assert_eq!(gitlab.webhook_secret, "wh-db"); + + // And a fresh state replays it back from the DB. + let state2 = Arc::new(fresh_state(vec![])); + assert!( + load_and_apply_ui_state_from_db(&state2, &store, &UiStateEnvOverrides::default()) + .await + .unwrap() + ); + assert_eq!(state2.llm_configs.read().unwrap()[0].api_key, "sk-live-db"); + } + + /// (d) The escape hatch: REVIEW_DISABLE_DB parsing. Behavioural 0.9 + /// equivalence is structural — `db = None` routes everything through the + /// file path, which the tests above (and every pre-existing persist test) + /// exercise. + #[test] + fn db_disabled_flag_parsing() { + assert!(db_disabled_flag(Some("1"))); + assert!(db_disabled_flag(Some("true"))); + assert!(db_disabled_flag(Some(" TRUE "))); + assert!(db_disabled_flag(Some("yes"))); + assert!(!db_disabled_flag(None)); + assert!(!db_disabled_flag(Some(""))); + assert!(!db_disabled_flag(Some("0"))); + assert!(!db_disabled_flag(Some("no"))); + assert!(!db_disabled_flag(Some("random"))); + } } diff --git a/src/server/api/config/put.rs b/src/server/api/config/put.rs index 151441f..17ac268 100644 --- a/src/server/api/config/put.rs +++ b/src/server/api/config/put.rs @@ -6,6 +6,7 @@ use base64::Engine; use std::sync::Arc; use crate::server::AppState; +use crate::store::traits::ConfigStore; use super::is_blank_or_masked; use super::types::{UiConfig, UiGitLabConfig, UiGitPlatformConfig, API_KEY_MASK}; @@ -482,16 +483,32 @@ pub async fn put_config( }; // Write-through persistence: everything the UI manages (llm, gitlab - // legacy fields, gitPlatforms, rules, advanced…) lands in - // `ui-state.toml` so a restart keeps it. The in-memory update above has + // legacy fields, gitPlatforms, rules, advanced…) is persisted so a + // restart keeps it — to the database when `state.db` is set (0.10.0, + // §6.2), otherwise to `ui-state.toml` (0.9 behaviour, also the + // REVIEW_DISABLE_DB escape hatch). The in-memory update above has // already been applied either way; a persist failure is surfaced as a // 500 so a silently-non-persistent deployment cannot go unnoticed. // - // The file is built from the REQUEST-RESOLVED sets (`applied`), never + // The snapshot is built from the REQUEST-RESOLVED sets (`applied`), never // from the effective runtime state: the runtime may additionally carry // env-derived entries (env wins at runtime), and persisting those would // leak env secrets to disk and resurrect them on a clean-env restart. - if let Some(path) = &state.ui_state_path { + // The env filter (`UiStateFile::from_applied`) is identical for both + // sinks: env/CLI values are never persisted anywhere. + if let Some(db) = &state.db { + let snapshot = super::persist::UiStateFile::from_applied(&applied, state.ui_state_env.as_ref()); + if let Err(e) = db.save_ui_state(&snapshot).await { + tracing::error!(error = %e, "failed to persist config to the database"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({ + "error": format!("config applied in memory but failed to persist to the database: {e}") + })), + ) + .into_response(); + } + } else if let Some(path) = &state.ui_state_path { let snapshot = super::persist::UiStateFile::from_applied(&applied, state.ui_state_env.as_ref()); if let Err(e) = super::persist::save_ui_state(path, &snapshot) { tracing::error!(path = %path.display(), error = %e, "failed to persist ui-state.toml"); diff --git a/src/server/state.rs b/src/server/state.rs index 93a2211..9a9ed8d 100644 --- a/src/server/state.rs +++ b/src/server/state.rs @@ -237,6 +237,11 @@ pub struct AppState { pub ui_state_env: Option, /// Finding feedback store for user verdicts (optional). pub feedback_store: Option>, + /// Persistent database handle (0.10.0; PG primary / SQLite fallback). + /// `None` = 0.9 behaviour (pure in-memory + ui-state.toml file), used by + /// tests, embedded use, and the `REVIEW_DISABLE_DB=1` escape hatch. Set + /// after pool + migrate succeed at startup, before the config replay. + pub db: Option>, /// Self-upgrade single-flight store + GitHub check cache + install method. pub upgrade: UpgradeStore, /// In-memory models.dev catalog cache (24h TTL enforced by handlers). @@ -265,6 +270,7 @@ impl AppState { ui_state_path: None, ui_state_env: None, feedback_store: None, + db: None, upgrade: UpgradeStore::new(), catalog: CatalogStore::new(), } diff --git a/src/store/sqlx.rs b/src/store/sqlx.rs index 2b3d8ef..588de8b 100644 --- a/src/store/sqlx.rs +++ b/src/store/sqlx.rs @@ -12,13 +12,102 @@ use async_trait::async_trait; use chrono::Utc; use crate::models::{GitPlatformConfig, LLMConfig}; -use crate::server::api::config::persist::PersistedGitlabConfig; +use crate::server::api::config::persist::{PersistedGitlabConfig, UiStateFile}; use super::rows; use super::traits::ConfigStore; use super::{encode_ts, SqlxStore}; const LEGACY_GITLAB_KEY: &str = "gitlab"; +const UI_KEY: &str = "ui"; + +type AnyTx<'a> = ::sqlx::Transaction<'a, ::sqlx::Any>; + +/// DELETE + re-INSERT the whole git_platforms set inside `tx`. +async fn replace_git_platforms_in(tx: &mut AnyTx<'_>, platforms: &[GitPlatformConfig], key: &[u8; 32]) -> Result<()> { + let now = encode_ts(&Utc::now()); + ::sqlx::query("DELETE FROM git_platforms") + .execute(&mut **tx) + .await + .context("clear git_platforms")?; + for platform in platforms { + let row = rows::git_platform_to_row(platform, uuid::Uuid::new_v4().to_string(), now.clone(), key)?; + ::sqlx::query( + "INSERT INTO git_platforms (id, name, type, base_url, internal_base_url, token, \ + webhook_secret, webhook_signing_secret, enabled, raw, updated_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(&row.id) + .bind(&row.name) + .bind(&row.platform_type) + .bind(&row.base_url) + .bind(&row.internal_base_url) + .bind(&row.token) + .bind(&row.webhook_secret) + .bind(&row.webhook_signing_secret) + .bind(i64::from(row.enabled)) + .bind(&row.raw) + .bind(&row.updated_at) + .execute(&mut **tx) + .await + .with_context(|| format!("insert git_platform {:?}", platform.name))?; + } + Ok(()) +} + +/// DELETE + re-INSERT the whole llm_providers set inside `tx`. +async fn replace_llm_providers_in(tx: &mut AnyTx<'_>, providers: &[LLMConfig], key: &[u8; 32]) -> Result<()> { + let now = encode_ts(&Utc::now()); + ::sqlx::query("DELETE FROM llm_providers") + .execute(&mut **tx) + .await + .context("clear llm_providers")?; + for (position, config) in providers.iter().enumerate() { + let row = rows::llm_to_row(config, position, uuid::Uuid::new_v4().to_string(), now.clone(), key)?; + ::sqlx::query( + "INSERT INTO llm_providers (id, provider, model, api_base, api_key, max_tokens, \ + temperature, raw, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(&row.id) + .bind(&row.provider) + .bind(&row.model) + .bind(&row.api_base) + .bind(&row.api_key) + .bind(row.max_tokens) + .bind(row.temperature) + .bind(&row.raw) + .bind(&row.updated_at) + .execute(&mut **tx) + .await + .with_context(|| format!("insert llm_provider {:?}", config.provider))?; + } + Ok(()) +} + +/// Upsert one app_settings row inside `tx`. Syntax is shared by PG and +/// SQLite (≥3.24); no RETURNING. +async fn upsert_setting_in(tx: &mut AnyTx<'_>, key: &str, value: &serde_json::Value) -> Result<()> { + ::sqlx::query( + "INSERT INTO app_settings (key, value, updated_at) VALUES (?, ?, ?) \ + ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at", + ) + .bind(key) + .bind(value.to_string()) + .bind(encode_ts(&Utc::now())) + .execute(&mut **tx) + .await + .with_context(|| format!("failed to save app_setting {key:?}"))?; + Ok(()) +} + +async fn delete_setting_in(tx: &mut AnyTx<'_>, key: &str) -> Result<()> { + ::sqlx::query("DELETE FROM app_settings WHERE key = ?") + .bind(key) + .execute(&mut **tx) + .await + .with_context(|| format!("failed to delete app_setting {key:?}"))?; + Ok(()) +} #[async_trait] impl ConfigStore for SqlxStore { @@ -84,34 +173,8 @@ impl ConfigStore for SqlxStore { } async fn replace_git_platforms(&self, platforms: &[GitPlatformConfig]) -> Result<()> { - let now = encode_ts(&Utc::now()); let mut tx = self.pool().begin().await.context("begin replace_git_platforms")?; - ::sqlx::query("DELETE FROM git_platforms") - .execute(&mut *tx) - .await - .context("clear git_platforms")?; - for platform in platforms { - let row = rows::git_platform_to_row(platform, uuid::Uuid::new_v4().to_string(), now.clone(), &self.key)?; - ::sqlx::query( - "INSERT INTO git_platforms (id, name, type, base_url, internal_base_url, token, \ - webhook_secret, webhook_signing_secret, enabled, raw, updated_at) \ - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - ) - .bind(&row.id) - .bind(&row.name) - .bind(&row.platform_type) - .bind(&row.base_url) - .bind(&row.internal_base_url) - .bind(&row.token) - .bind(&row.webhook_secret) - .bind(&row.webhook_signing_secret) - .bind(i64::from(row.enabled)) - .bind(&row.raw) - .bind(&row.updated_at) - .execute(&mut *tx) - .await - .with_context(|| format!("insert git_platform {:?}", platform.name))?; - } + replace_git_platforms_in(&mut tx, platforms, &self.key).await?; tx.commit().await.context("commit replace_git_platforms")?; Ok(()) } @@ -149,37 +212,8 @@ impl ConfigStore for SqlxStore { } async fn replace_llm_providers(&self, providers: &[LLMConfig]) -> Result<()> { - let now = encode_ts(&Utc::now()); let mut tx = self.pool().begin().await.context("begin replace_llm_providers")?; - ::sqlx::query("DELETE FROM llm_providers") - .execute(&mut *tx) - .await - .context("clear llm_providers")?; - for (position, config) in providers.iter().enumerate() { - let row = rows::llm_to_row( - config, - position, - uuid::Uuid::new_v4().to_string(), - now.clone(), - &self.key, - )?; - ::sqlx::query( - "INSERT INTO llm_providers (id, provider, model, api_base, api_key, max_tokens, \ - temperature, raw, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", - ) - .bind(&row.id) - .bind(&row.provider) - .bind(&row.model) - .bind(&row.api_base) - .bind(&row.api_key) - .bind(row.max_tokens) - .bind(row.temperature) - .bind(&row.raw) - .bind(&row.updated_at) - .execute(&mut *tx) - .await - .with_context(|| format!("insert llm_provider {:?}", config.provider))?; - } + replace_llm_providers_in(&mut tx, providers, &self.key).await?; tx.commit().await.context("commit replace_llm_providers")?; Ok(()) } @@ -207,17 +241,30 @@ impl ConfigStore for SqlxStore { } async fn save_setting(&self, key: &str, value: &serde_json::Value) -> Result<()> { - // Upsert syntax is shared by PG and SQLite (≥3.24); no RETURNING. - ::sqlx::query( - "INSERT INTO app_settings (key, value, updated_at) VALUES (?, ?, ?) \ - ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at", - ) - .bind(key) - .bind(value.to_string()) - .bind(encode_ts(&Utc::now())) - .execute(self.pool()) - .await - .with_context(|| format!("failed to save app_setting {key:?}"))?; + let mut tx = self.pool().begin().await.context("begin save_setting")?; + upsert_setting_in(&mut tx, key, value).await?; + tx.commit().await.context("commit save_setting")?; + Ok(()) + } + + async fn save_ui_state(&self, state: &UiStateFile) -> Result<()> { + let mut tx = self.pool().begin().await.context("begin save_ui_state")?; + replace_git_platforms_in(&mut tx, &state.git_platforms, &self.key).await?; + replace_llm_providers_in(&mut tx, &state.llm, &self.key).await?; + let gitlab = &state.gitlab; + if gitlab.token.is_empty() && gitlab.webhook_secret.is_empty() && gitlab.webhook_signing_secret.is_empty() { + // Unset is unset: an all-empty legacy gitlab value removes the row + // instead of storing an empty JSON shell. + delete_setting_in(&mut tx, LEGACY_GITLAB_KEY).await?; + } else { + let value = rows::legacy_gitlab_to_value(gitlab, &self.key)?; + upsert_setting_in(&mut tx, LEGACY_GITLAB_KEY, &value).await?; + } + if let Some(ui) = &state.ui { + let value = serde_json::to_value(ui).context("serialize ui projection")?; + upsert_setting_in(&mut tx, UI_KEY, &value).await?; + } + tx.commit().await.context("commit save_ui_state")?; Ok(()) } diff --git a/src/store/traits.rs b/src/store/traits.rs index 1d7a3c3..8de9b35 100644 --- a/src/store/traits.rs +++ b/src/store/traits.rs @@ -14,7 +14,7 @@ use anyhow::Result; use async_trait::async_trait; use crate::models::{GitPlatformConfig, LLMConfig}; -use crate::server::api::config::persist::PersistedGitlabConfig; +use crate::server::api::config::persist::{PersistedGitlabConfig, UiStateFile}; /// Persistence boundary for UI-managed configuration. /// @@ -55,6 +55,18 @@ pub trait ConfigStore: Send + Sync { /// Upsert an arbitrary JSON setting. async fn save_setting(&self, key: &str, value: &serde_json::Value) -> Result<()>; + /// Atomically persist a whole [`UiStateFile`] snapshot in ONE + /// transaction: git_platforms + llm_providers are replaced wholesale, the + /// legacy `gitlab` settings row is upserted (an all-empty value deletes + /// the row — unset is unset), and the `ui` projection is upserted when + /// present (`None` leaves any existing `ui` row untouched). + /// + /// Used by the `PUT /config` save path (§6.2) and by the one-shot + /// ui-state.toml import (§6.1): for the import, all-or-nothing is a hard + /// requirement — a partial import must roll back so the next startup can + /// retry against the still-present file. + async fn save_ui_state(&self, state: &UiStateFile) -> Result<()>; + /// True when git_platforms + llm_providers + app_settings are all empty /// — the trigger condition for the one-shot `ui-state.toml` import /// (design/persistence.md §6.1 step 3). From 480205ad5b5b82ad2e3d78bd9f2ee2d8eeb99e3c Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 11:59:33 +0800 Subject: [PATCH 06/36] feat(store): ReviewStore trait + TaskStore write-through + restart recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.10.0 persistence step 4 (design/persistence.md §5): - store/traits.rs: ReviewStore — create / mark_started / fill_source_meta / complete (terminal, single tx: reviews UPDATE + expert_reports replace) / mark_cancelled / mark_retry / mark_interrupted (startup sweep). - store/sqlx.rs: SqlxStore impl; store/rows.rs: TaskEntry⇄reviews and ReviewOutput.reports⇄expert_reports codecs (state strings reuse the API projection mapping task_status_str so DB/SSE vocabulary cannot drift). - server/task_queue.rs: TaskStore gains db: Option> + set_db; write-through on create_with_request/start/fill_source_meta/ update(terminal)/delete/retry per §5.2; set_progress stays memory-only; the cancelled early-return writes nothing. DB writes are synchronously awaited AFTER the in-memory lock is released; failures log and continue, terminal writes retry once. db=None is exactly 0.9 behaviour. - cli/app.rs: after migrate, mark_interrupted() sweeps stale pending/running rows to failed('interrupted: server restarted') and the DB handle is injected into the task store. - reaper untouched (memory-only). Tests: full lifecycle row assertions, expert_reports split, failing-store injection (path unblocked, terminal retried exactly once), interrupted sweep state coverage, codec round-trip, cancel/retry write-through, db=None 0.9 parity. fmt/clippy/test green (1592 passed, 0 failed). --- src/cli/app.rs | 26 ++- src/server/task_queue.rs | 459 +++++++++++++++++++++++++++++++++++---- src/store/mod.rs | 5 +- src/store/rows.rs | 146 +++++++++++++ src/store/sqlx.rs | 301 ++++++++++++++++++++++++- src/store/traits.rs | 47 +++- 6 files changed, 931 insertions(+), 53 deletions(-) diff --git a/src/cli/app.rs b/src/cli/app.rs index c6b195f..f2c1d69 100644 --- a/src/cli/app.rs +++ b/src/cli/app.rs @@ -282,16 +282,34 @@ pub async fn run() -> Result<()> { // 0.10.0 persistence (design/persistence.md §6.1, strict order): // 1) resolve DB URL → pool → migrate (failure aborts startup; // REVIEW_DISABLE_DB=1 bypasses to 0.9 behaviour); - // 2) TODO(梁序, step 4): §5.3 interrupted sweep — UPDATE reviews - // SET state='failed', error='interrupted: server restarted', - // completed_at=? WHERE state IN ('pending','running') goes - // here, after migrate and before the config replay; + // 2) §5.3 interrupted sweep + TaskStore write-through injection + // (below, right after migrate); // 3) one-shot ui-state.toml import (single transaction; failure // keeps the file and falls back to the file replay below); // 4) replay the DB state through the same apply_ui_config path. app_state.db = review_engine::server::api::config::persist::bootstrap_database() .await? .map(Arc::new); + if let Some(store) = app_state.db.clone() { + // §5.3: tasks still pending/running when the previous process + // died are marked failed with an 'interrupted' error. They are + // NOT re-queued automatically (LLM quota / duplicate MR + // comments); the user retries from the history page. + use review_engine::store::traits::ReviewStore; + match store.mark_interrupted(chrono::Utc::now()).await { + Ok(0) => {} + Ok(n) => tracing::warn!( + "marked {n} interrupted review task(s) as failed (server restarted); \ + they can be retried manually from the history page" + ), + Err(e) => tracing::error!("interrupted-task sweep failed: {e:#}"), + } + // Write-through injection: rebuild the (still untouched) task + // store with the DB attached (§5.2). + let mut task_store = review_engine::server::task_queue::TaskStore::new(); + task_store.set_db(store); + app_state.task_store = Some(Arc::new(task_store)); + } let state = Arc::new(app_state); let mut config_replayed = false; if let Some(store) = state.db.clone() { diff --git a/src/server/task_queue.rs b/src/server/task_queue.rs index 8cf0547..82f5702 100644 --- a/src/server/task_queue.rs +++ b/src/server/task_queue.rs @@ -22,7 +22,7 @@ pub enum TaskState { } /// Metadata about the source merge request or pull request. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] pub struct SourceMeta { pub mr_title: Option, pub project: Option, @@ -112,6 +112,14 @@ pub struct TaskEvent { /// Provides concurrency-safe task lifecycle management with automatic /// expiry cleanup, broadcast SSE events, pause/resume control, and /// configurable concurrency limits. +/// +/// Persistence (0.10.0, design/persistence.md §5): when `db` is injected via +/// [`set_db`](Self::set_db), every lifecycle transition is ALSO written +/// through to the database, synchronously awaited (restart-recovery +/// semantics rely on the DB being current). Write failures never block the +/// review path — they are logged; terminal writes are retried once. `None` +/// (the default) is exactly the 0.9 pure in-memory behaviour, which the +/// runtime-free sync unit tests depend on. #[derive(Clone)] pub struct TaskStore { inner: Arc>>, @@ -119,6 +127,7 @@ pub struct TaskStore { is_paused: Arc>, max_concurrent: Arc>, queue_capacity: Arc>, + db: Option>, } impl TaskStore { @@ -154,9 +163,17 @@ impl TaskStore { is_paused: Arc::new(RwLock::new(false)), max_concurrent: Arc::new(RwLock::new(8)), queue_capacity: Arc::new(RwLock::new(16)), + db: None, } } + /// Inject the write-through persistence target (0.10.0 §5.2). Call before + /// the store is shared with workers; leaving it unset keeps the 0.9 pure + /// in-memory behaviour. + pub fn set_db(&mut self, db: Arc) { + self.db = Some(db); + } + pub async fn cleanup_expired(&self) { let cutoff = chrono::Utc::now() - chrono::Duration::minutes(30); let mut map = self.inner.write().await; @@ -195,7 +212,7 @@ impl TaskStore { progress: None, expert_name: None, }; - self.inner.write().await.insert(id, entry); + self.inner.write().await.insert(id, entry.clone()); let _ = self.tx.send(TaskEvent { task_id: id, status: "pending", @@ -206,13 +223,22 @@ impl TaskStore { expert_name: None, elapsed_ms: None, }); + // Write-through (§5.2): INSERT reviews (state=pending). + if let Some(db) = &self.db { + if let Err(e) = db.create(&entry).await { + tracing::error!("failed to persist new review task {id} to the database: {e:#}"); + } + } id } pub async fn start(&self, task_id: Uuid) { + let mut started_at = None; if let Some(entry) = self.inner.write().await.get_mut(&task_id) { + let now = chrono::Utc::now(); entry.state = TaskState::Running; - entry.started_at = Some(chrono::Utc::now()); + entry.started_at = Some(now); + started_at = Some(now); let _ = self.tx.send(TaskEvent { task_id, status: "running", @@ -224,6 +250,13 @@ impl TaskStore { elapsed_ms: None, }); } + // Write-through (§5.2): UPDATE state=running, started_at. Awaited + // AFTER the in-memory write lock is released. + if let (Some(db), Some(at)) = (&self.db, started_at) { + if let Err(e) = db.mark_started(task_id, at).await { + tracing::error!("failed to persist review start for task {task_id}: {e:#}"); + } + } } pub async fn set_progress(&self, task_id: Uuid, progress: u8, expert_name: Option) { @@ -253,6 +286,7 @@ impl TaskStore { result: Option, error: Option, ) { + let mut terminal_snapshot = None; if let Some(entry) = self.inner.write().await.get_mut(&task_id) { // Cancelled is terminal: a background task racing past a DELETE // must not flip the record back to running/completed. @@ -265,6 +299,7 @@ impl TaskStore { if new_state == TaskState::Completed || new_state == TaskState::Failed || new_state == TaskState::Cancelled { entry.completed_at = Some(chrono::Utc::now()); + terminal_snapshot = Some(entry.clone()); } let event = match new_state { TaskState::Pending => "review.created", @@ -294,6 +329,21 @@ impl TaskStore { elapsed_ms: elapsed, }); } + // Write-through (§5.2): terminal transitions persist state/result/ + // error/completed_at/progress + the expert_reports split. Terminal + // writes get ONE immediate retry on failure (§5.2), then are given + // up — history may lose a row, the review itself must never die. + if let (Some(db), Some(entry)) = (&self.db, terminal_snapshot) { + if let Err(e) = db.complete(&entry).await { + tracing::error!("failed to persist terminal state for task {task_id}: {e:#}; retrying once"); + if let Err(e) = db.complete(&entry).await { + tracing::error!( + "terminal write for task {task_id} failed again: {e:#}; \ + giving up — history will miss this row" + ); + } + } + } } /// Back-fill a task's `source_meta` from `candidate`, filling only fields @@ -311,6 +361,7 @@ impl TaskStore { fn is_blank(value: &Option) -> bool { value.as_deref().map(str::trim).unwrap_or_default().is_empty() } + let mut filled = None; if let Some(entry) = self.inner.write().await.get_mut(&task_id) { if entry.state == TaskState::Cancelled { return; @@ -343,6 +394,14 @@ impl TaskStore { if is_blank(&meta.commit_sha) { meta.commit_sha = candidate.commit_sha; } + filled = Some(entry.source_meta.clone()); + } + // Write-through (§5.2): UPDATE source_meta + the materialized + // project/repository filter columns. At most once per task. + if let (Some(db), Some(meta)) = (&self.db, filled) { + if let Err(e) = db.fill_source_meta(task_id, &meta).await { + tracing::error!("failed to persist source_meta for task {task_id}: {e:#}"); + } } } @@ -426,52 +485,84 @@ impl TaskStore { /// as `cancelled`. Tasks that are already in a terminal state (`Completed`, /// `Failed`, `Cancelled`) are left untouched and return `false`. pub async fn delete(&self, task_id: Uuid) -> bool { - let mut map = self.inner.write().await; - if let Some(entry) = map.get_mut(&task_id) { - if entry.state == TaskState::Pending || entry.state == TaskState::Running { - entry.state = TaskState::Cancelled; - entry.completed_at = Some(chrono::Utc::now()); - let meta = entry.source_meta.clone(); - let _ = self.tx.send(TaskEvent { - task_id, - status: "cancelled", - event: "review.cancelled", - mr_title: meta.mr_title, - project: meta.project, - progress: None, - expert_name: None, - elapsed_ms: None, - }); - return true; + let mut cancelled_at = None; + let transitioned = { + let mut map = self.inner.write().await; + if let Some(entry) = map.get_mut(&task_id) { + if entry.state == TaskState::Pending || entry.state == TaskState::Running { + entry.state = TaskState::Cancelled; + let now = chrono::Utc::now(); + entry.completed_at = Some(now); + cancelled_at = Some(now); + let meta = entry.source_meta.clone(); + let _ = self.tx.send(TaskEvent { + task_id, + status: "cancelled", + event: "review.cancelled", + mr_title: meta.mr_title, + project: meta.project, + progress: None, + expert_name: None, + elapsed_ms: None, + }); + true + } else { + false + } + } else { + false + } + }; + // Write-through (§5.2): UPDATE state=cancelled, completed_at. + if transitioned { + if let (Some(db), Some(at)) = (&self.db, cancelled_at) { + if let Err(e) = db.mark_cancelled(task_id, at).await { + tracing::error!("failed to persist cancellation for task {task_id}: {e:#}"); + } } } - false + transitioned } pub async fn retry(&self, task_id: Uuid) -> bool { - let mut map = self.inner.write().await; - if let Some(entry) = map.get_mut(&task_id) { - if entry.state == TaskState::Failed { - entry.state = TaskState::Pending; - entry.error = None; - entry.progress = None; - entry.completed_at = None; - entry.started_at = None; - let meta = entry.source_meta.clone(); - let _ = self.tx.send(TaskEvent { - task_id, - status: "pending", - event: "review.retry", - mr_title: meta.mr_title, - project: meta.project, - progress: None, - expert_name: None, - elapsed_ms: None, - }); - return true; + let transitioned = { + let mut map = self.inner.write().await; + if let Some(entry) = map.get_mut(&task_id) { + if entry.state == TaskState::Failed { + entry.state = TaskState::Pending; + entry.error = None; + entry.progress = None; + entry.completed_at = None; + entry.started_at = None; + let meta = entry.source_meta.clone(); + let _ = self.tx.send(TaskEvent { + task_id, + status: "pending", + event: "review.retry", + mr_title: meta.mr_title, + project: meta.project, + progress: None, + expert_name: None, + elapsed_ms: None, + }); + true + } else { + false + } + } else { + false + } + }; + // Write-through (§5.2): UPDATE state=pending, error=NULL, + // completed_at=NULL (Failed → Pending). + if transitioned { + if let Some(db) = &self.db { + if let Err(e) = db.mark_retry(task_id).await { + tracing::error!("failed to persist retry for task {task_id}: {e:#}"); + } } } - false + transitioned } /// Aggregate queue statistics from the current task store. @@ -732,4 +823,288 @@ mod tests { "a cancelled task must not be mutated, got {meta:?}" ); } + + // ─── 0.10.0 write-through (design/persistence.md §5.2) ─── + + use crate::store::traits::ReviewStore; + use crate::store::SqlxStore; + + async fn db_backed_store() -> (TaskStore, Arc) { + let db = Arc::new(SqlxStore::new_in_memory().await.unwrap()); + db.migrate().await.unwrap(); + let mut store = TaskStore::new(); + store.set_db(db.clone()); + (store, db) + } + + /// (state, project, repository, result, error, progress, started_at, completed_at) + type ReviewRowTuple = ( + String, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + ); + + async fn review_row(db: &SqlxStore, task_id: Uuid) -> Option { + sqlx::query_as( + "SELECT state, project, repository, result, error, progress, started_at, completed_at \ + FROM reviews WHERE task_id = ?", + ) + .bind(task_id.to_string()) + .fetch_optional(db.pool()) + .await + .unwrap() + } + + fn sample_output() -> crate::models::ReviewOutput { + let report = |name: &str| crate::models::ExpertReport { + expert_name: name.to_string(), + findings: vec![], + markdown: format!("# {name} report"), + raw_llm_response: "raw".to_string(), + parse_error: None, + raw_dump_path: None, + }; + crate::models::ReviewOutput { + reports: vec![report("security"), report("performance")], + aggregated: None, + dropped_findings: vec![], + consolidated: None, + } + } + + /// (a) full lifecycle: create → start → fill_meta → complete, asserting + /// the DB row at every step. + #[tokio::test] + async fn write_through_full_lifecycle() { + let (store, db) = db_backed_store().await; + let request = serde_json::json!({"mr_url": "https://gitlab.example/group/proj/-/merge_requests/1"}); + let id = store + .create_with_request( + Some(SourceMeta { + project: Some("group/proj".to_string()), + repository: Some("proj".to_string()), + ..SourceMeta::default() + }), + Some(request.clone()), + ) + .await; + + // create → INSERT (pending), request + materialized columns persisted. + let row = review_row(&db, id).await.expect("row exists after create"); + assert_eq!(row.0, "pending"); + assert_eq!(row.1.as_deref(), Some("group/proj")); + assert_eq!(row.2.as_deref(), Some("proj")); + assert!(row.3.is_none() && row.4.is_none()); + assert!(row.6.is_none() && row.7.is_none()); + let req: String = sqlx::query_scalar("SELECT request FROM reviews WHERE task_id = ?") + .bind(id.to_string()) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(serde_json::from_str::(&req).unwrap(), request); + let created_at: String = sqlx::query_scalar("SELECT created_at FROM reviews WHERE task_id = ?") + .bind(id.to_string()) + .fetch_one(db.pool()) + .await + .unwrap(); + crate::store::decode_ts(&created_at).unwrap(); + + // start → state=running + started_at. + store.start(id).await; + let row = review_row(&db, id).await.unwrap(); + assert_eq!(row.0, "running"); + assert!(row.6.is_some(), "started_at persisted"); + + // fill_source_meta → source_meta updated; enqueue-time project wins. + store.fill_source_meta(id, candidate_meta()).await; + let meta_raw: String = sqlx::query_scalar("SELECT source_meta FROM reviews WHERE task_id = ?") + .bind(id.to_string()) + .fetch_one(db.pool()) + .await + .unwrap(); + let meta: SourceMeta = serde_json::from_str(&meta_raw).unwrap(); + assert_eq!(meta.mr_title.as_deref(), Some("Add login endpoint")); + assert_eq!(meta.branch.as_deref(), Some("feature/login")); + assert_eq!( + meta.project.as_deref(), + Some("group/proj"), + "enqueue-time value must win" + ); + let row = review_row(&db, id).await.unwrap(); + assert_eq!(row.1.as_deref(), Some("group/proj"), "materialized column in sync"); + + // set_progress must NOT write: progress stays NULL mid-flight. + store.set_progress(id, 42, Some("security".to_string())).await; + let row = review_row(&db, id).await.unwrap(); + assert!(row.5.is_none(), "progress is not persisted mid-flight"); + + // complete → terminal row: state/result/completed_at + progress snapshot. + let result = serde_json::to_value(sample_output()).unwrap(); + store.update(id, TaskState::Completed, Some(result.clone()), None).await; + let row = review_row(&db, id).await.unwrap(); + assert_eq!(row.0, "completed"); + assert!(row.7.is_some(), "completed_at persisted"); + assert_eq!(row.5, Some(42), "terminal write snapshots progress"); + assert_eq!( + serde_json::from_str::(&row.3.unwrap()).unwrap(), + result + ); + } + + /// (b) complete splits result.reports into one expert_reports row each. + #[tokio::test] + async fn write_through_complete_splits_expert_reports() { + let (store, db) = db_backed_store().await; + let id = record_task_started(&store, SourceMeta::default()).await; + let output = sample_output(); + store + .update( + id, + TaskState::Completed, + Some(serde_json::to_value(&output).unwrap()), + None, + ) + .await; + + let reports: Vec<(String, String, Option, String)> = sqlx::query_as( + "SELECT expert_name, report, duration_ms, created_at FROM expert_reports \ + WHERE task_id = ? ORDER BY expert_name", + ) + .bind(id.to_string()) + .fetch_all(db.pool()) + .await + .unwrap(); + assert_eq!(reports.len(), 2); + assert_eq!(reports[0].0, "performance"); + assert_eq!(reports[1].0, "security"); + for (_, report_json, duration_ms, created_at) in &reports { + let decoded: crate::models::ExpertReport = serde_json::from_str(report_json).unwrap(); + assert!(decoded.markdown.starts_with("# ")); + assert!(duration_ms.is_none(), "per-expert duration is NULL for now (§5.4)"); + crate::store::decode_ts(created_at).unwrap(); + } + assert_eq!(reports[1].0, output.reports[0].expert_name); + + // A failed terminal write stores error, no result, no reports. + let id2 = record_task_started(&store, SourceMeta::default()).await; + store + .update(id2, TaskState::Failed, None, Some("boom".to_string())) + .await; + let row = review_row(&db, id2).await.unwrap(); + assert_eq!(row.0, "failed"); + assert_eq!(row.4.as_deref(), Some("boom")); + assert!(row.3.is_none()); + let report_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM expert_reports WHERE task_id = ?") + .bind(id2.to_string()) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(report_count, 0); + } + + /// delete (cancel) and retry write-through. + #[tokio::test] + async fn write_through_cancel_and_retry() { + let (store, db) = db_backed_store().await; + + let id = store.create(None).await; + assert!(store.delete(id).await); + let row = review_row(&db, id).await.unwrap(); + assert_eq!(row.0, "cancelled"); + assert!(row.7.is_some(), "cancel stamps completed_at"); + + // Cancelling a terminal task touches nothing, in memory or in DB. + assert!(!store.delete(id).await); + + let id2 = record_task_started(&store, SourceMeta::default()).await; + store + .update(id2, TaskState::Failed, None, Some("boom".to_string())) + .await; + assert!(store.retry(id2).await); + let row = review_row(&db, id2).await.unwrap(); + assert_eq!(row.0, "pending"); + assert!(row.4.is_none(), "retry clears error"); + assert!(row.7.is_none(), "retry clears completed_at"); + } + + /// (c) a failing ReviewStore never blocks the review path; the terminal + /// write is retried exactly once and then given up. + #[tokio::test] + async fn write_through_failure_never_blocks_review_path() { + #[derive(Default)] + struct FailingStore { + complete_attempts: std::sync::atomic::AtomicUsize, + } + + #[async_trait::async_trait] + impl ReviewStore for FailingStore { + async fn create(&self, _: &TaskEntry) -> anyhow::Result<()> { + anyhow::bail!("db down") + } + async fn mark_started(&self, _: Uuid, _: chrono::DateTime) -> anyhow::Result<()> { + anyhow::bail!("db down") + } + async fn fill_source_meta(&self, _: Uuid, _: &SourceMeta) -> anyhow::Result<()> { + anyhow::bail!("db down") + } + async fn complete(&self, _: &TaskEntry) -> anyhow::Result<()> { + self.complete_attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + anyhow::bail!("db down") + } + async fn mark_cancelled(&self, _: Uuid, _: chrono::DateTime) -> anyhow::Result<()> { + anyhow::bail!("db down") + } + async fn mark_retry(&self, _: Uuid) -> anyhow::Result<()> { + anyhow::bail!("db down") + } + async fn mark_interrupted(&self, _: chrono::DateTime) -> anyhow::Result { + anyhow::bail!("db down") + } + } + + let failing = Arc::new(FailingStore::default()); + let mut store = TaskStore::new(); + store.set_db(failing.clone()); + + let id = record_task_started(&store, SourceMeta::default()).await; + store + .update( + id, + TaskState::Completed, + Some(serde_json::to_value(sample_output()).unwrap()), + None, + ) + .await; + + // The review path is unaffected: the in-memory entry is Completed. + let entry = store.get(id).await.expect("entry exists"); + assert_eq!(entry.state, TaskState::Completed); + assert!(entry.result.is_some()); + assert_eq!( + failing.complete_attempts.load(std::sync::atomic::Ordering::SeqCst), + 2, + "terminal write is retried exactly once, then given up" + ); + } + + /// (e) with no DB injected the store is exactly the 0.9 in-memory store. + #[tokio::test] + async fn no_db_behaves_like_0_9() { + let store = TaskStore::new(); + let id = record_task_started(&store, SourceMeta::default()).await; + store + .update(id, TaskState::Completed, Some(serde_json::json!({"ok": true})), None) + .await; + let entry = store.get(id).await.unwrap(); + assert_eq!(entry.state, TaskState::Completed); + assert!(!store.retry(id).await, "retry only from Failed"); + store.update(id, TaskState::Failed, None, Some("x".to_string())).await; + assert!(store.retry(id).await); + assert_eq!(store.get(id).await.unwrap().state, TaskState::Pending); + } } diff --git a/src/store/mod.rs b/src/store/mod.rs index 4e7f180..84c1b3e 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -157,9 +157,8 @@ pub(crate) fn encode_ts(ts: &DateTime) -> String { ts.to_rfc3339_opts(SecondsFormat::Micros, true) } -/// Decode a timestamp produced by [`encode_ts`]. Used by tests and by the -/// review-domain codecs (later steps). -#[allow(dead_code)] +/// Decode a timestamp produced by [`encode_ts`]. Used by the review-domain +/// codecs ([`rows`]) and by tests. pub(crate) fn decode_ts(s: &str) -> Result> { Ok(DateTime::parse_from_rfc3339(s) .with_context(|| format!("invalid RFC 3339 timestamp in database: {s:?}"))? diff --git a/src/store/rows.rs b/src/store/rows.rs index 8bf216f..2c8baf2 100644 --- a/src/store/rows.rs +++ b/src/store/rows.rs @@ -175,3 +175,149 @@ pub(crate) fn legacy_gitlab_from_value(value: &Value, key: &[u8; 32]) -> Result< webhook_signing_secret: field("webhook_signing_secret")?, }) } + +// ─── Review domain (step 4): reviews / expert_reports ⇄ TaskEntry ─── + +use crate::server::task_queue::{SourceMeta, TaskEntry, TaskState}; +use crate::store::{decode_ts, encode_ts}; +use uuid::Uuid; + +/// At-rest form of one `reviews` row (design/persistence.md §3.2). JSON +/// columns (`source_meta`, `request`, `result`) are serialized TEXT; +/// timestamps are RFC 3339 UTC strings via `encode_ts` / `decode_ts`. +#[derive(Debug)] +pub(crate) struct ReviewRow { + pub task_id: String, + pub state: String, + pub source_meta: String, + /// Materialized filter columns kept in sync with `source_meta` (§3.2). + pub project: Option, + pub repository: Option, + pub request: Option, + pub result: Option, + pub error: Option, + pub progress: Option, + pub created_at: String, + pub started_at: Option, + pub completed_at: Option, +} + +fn opt_json(value: &Option, what: &str) -> Result> { + value + .as_ref() + .map(|v| serde_json::to_string(v).with_context(|| format!("serialize {what}"))) + .transpose() +} + +/// `TaskState` → the `reviews.state` string. Single source of truth is the +/// API projection mapping (`task_status_str`); the store reuses it so the +/// DB vocabulary can never drift from the SSE / API vocabulary (§5.3). +pub(crate) fn task_state_str(state: &TaskState) -> &'static str { + crate::server::api::review::task_status_str(state) +} + +pub(crate) fn task_state_from_str(s: &str) -> Result { + match s { + "pending" => Ok(TaskState::Pending), + "running" => Ok(TaskState::Running), + "completed" => Ok(TaskState::Completed), + "failed" => Ok(TaskState::Failed), + "cancelled" => Ok(TaskState::Cancelled), + other => anyhow::bail!("unknown reviews.state value: {other:?}"), + } +} + +pub(crate) fn encode_source_meta(meta: &SourceMeta) -> Result { + serde_json::to_string(meta).context("serialize source_meta") +} + +pub(crate) fn decode_source_meta(raw: &str) -> Result { + serde_json::from_str(raw).with_context(|| format!("reviews.source_meta holds invalid JSON: {raw:?}")) +} + +pub(crate) fn task_entry_to_row(entry: &TaskEntry) -> Result { + Ok(ReviewRow { + task_id: entry.task_id.to_string(), + state: task_state_str(&entry.state).to_string(), + source_meta: encode_source_meta(&entry.source_meta)?, + project: entry.source_meta.project.clone(), + repository: entry.source_meta.repository.clone(), + request: opt_json(&entry.request, "reviews.request")?, + result: opt_json(&entry.result, "reviews.result")?, + error: entry.error.clone(), + progress: entry.progress.map(i64::from), + created_at: encode_ts(&entry.created_at), + started_at: entry.started_at.as_ref().map(encode_ts), + completed_at: entry.completed_at.as_ref().map(encode_ts), + }) +} + +/// Decode a `reviews` row back into a [`TaskEntry`]. Used by tests now and +/// by the history read path in the next step (§8.1). +#[allow(dead_code)] +pub(crate) fn review_from_row(row: ReviewRow) -> Result { + fn opt_ts(raw: Option, what: &str) -> Result>> { + raw.as_deref() + .map(|s| decode_ts(s).with_context(|| format!("reviews.{what}"))) + .transpose() + } + let state = task_state_from_str(&row.state)?; + Ok(TaskEntry { + task_id: Uuid::parse_str(&row.task_id) + .with_context(|| format!("reviews.task_id is not a UUID: {:?}", row.task_id))?, + state, + created_at: decode_ts(&row.created_at).context("reviews.created_at")?, + started_at: opt_ts(row.started_at, "started_at")?, + completed_at: opt_ts(row.completed_at, "completed_at")?, + result: row + .result + .map(|s| serde_json::from_str(&s).context("reviews.result holds invalid JSON")) + .transpose()?, + error: row.error, + request: row + .request + .map(|s| serde_json::from_str(&s).context("reviews.request holds invalid JSON")) + .transpose()?, + source_meta: decode_source_meta(&row.source_meta)?, + progress: row + .progress + .map(|p| u8::try_from(p).with_context(|| format!("reviews.progress out of range: {p}"))) + .transpose()?, + // Live-only fields: the DB is the history source, the in-memory + // `expert_name` (current active expert) is not persisted. + expert_name: None, + }) +} + +/// At-rest form of one `expert_reports` row. +#[derive(Debug)] +pub(crate) struct ExpertReportRow { + pub task_id: String, + pub expert_name: String, + pub report: String, + /// Per-expert duration: always NULL for now — `TaskEntry` does not track + /// it yet (design/persistence.md §5.4 note). + pub duration_ms: Option, + pub created_at: String, +} + +/// Split a serialized `ReviewOutput` (`reviews.result`) into one +/// `expert_reports` row per `reports[]` entry. +pub(crate) fn expert_report_rows(task_id: &Uuid, result: &Value, created_at: String) -> Result> { + let output: crate::models::ReviewOutput = + serde_json::from_value(result.clone()).context("reviews.result is not a serialized ReviewOutput")?; + output + .reports + .iter() + .map(|report| { + Ok(ExpertReportRow { + task_id: task_id.to_string(), + expert_name: report.expert_name.clone(), + report: serde_json::to_string(report) + .with_context(|| format!("serialize expert report {:?}", report.expert_name))?, + duration_ms: None, + created_at: created_at.clone(), + }) + }) + .collect() +} diff --git a/src/store/sqlx.rs b/src/store/sqlx.rs index 588de8b..38c3e04 100644 --- a/src/store/sqlx.rs +++ b/src/store/sqlx.rs @@ -9,13 +9,14 @@ use anyhow::{Context, Result}; use async_trait::async_trait; -use chrono::Utc; +use chrono::{DateTime, Utc}; use crate::models::{GitPlatformConfig, LLMConfig}; use crate::server::api::config::persist::{PersistedGitlabConfig, UiStateFile}; +use crate::server::task_queue::{SourceMeta, TaskEntry}; use super::rows; -use super::traits::ConfigStore; +use super::traits::{ConfigStore, ReviewStore}; use super::{encode_ts, SqlxStore}; const LEGACY_GITLAB_KEY: &str = "gitlab"; @@ -281,6 +282,163 @@ impl ConfigStore for SqlxStore { } } +// ─── ReviewStore (reviews / expert_reports, step 4) ─── + +/// Warn when a per-task UPDATE matched no row — the create write-through +/// must have failed earlier, so the task's history is already lost; the +/// warning keeps that visible without failing the review path. +fn warn_missing_row(op: &str, task_id: &uuid::Uuid, rows_affected: u64) { + if rows_affected == 0 { + tracing::warn!("{op}: no reviews row for task {task_id} (earlier write-through presumably failed)"); + } +} + +#[async_trait] +impl ReviewStore for SqlxStore { + async fn create(&self, entry: &TaskEntry) -> Result<()> { + let row = rows::task_entry_to_row(entry)?; + ::sqlx::query( + "INSERT INTO reviews (task_id, state, source_meta, project, repository, request, \ + result, error, progress, created_at, started_at, completed_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(&row.task_id) + .bind(&row.state) + .bind(&row.source_meta) + .bind(&row.project) + .bind(&row.repository) + .bind(&row.request) + .bind(&row.result) + .bind(&row.error) + .bind(row.progress) + .bind(&row.created_at) + .bind(&row.started_at) + .bind(&row.completed_at) + .execute(self.pool()) + .await + .with_context(|| format!("insert review {}", row.task_id))?; + Ok(()) + } + + async fn mark_started(&self, task_id: uuid::Uuid, started_at: DateTime) -> Result<()> { + let res = ::sqlx::query("UPDATE reviews SET state = 'running', started_at = ? WHERE task_id = ?") + .bind(encode_ts(&started_at)) + .bind(task_id.to_string()) + .execute(self.pool()) + .await + .with_context(|| format!("mark review {task_id} started"))?; + warn_missing_row("mark_started", &task_id, res.rows_affected()); + Ok(()) + } + + async fn fill_source_meta(&self, task_id: uuid::Uuid, meta: &SourceMeta) -> Result<()> { + let res = ::sqlx::query("UPDATE reviews SET source_meta = ?, project = ?, repository = ? WHERE task_id = ?") + .bind(rows::encode_source_meta(meta)?) + .bind(&meta.project) + .bind(&meta.repository) + .bind(task_id.to_string()) + .execute(self.pool()) + .await + .with_context(|| format!("fill source_meta for review {task_id}"))?; + warn_missing_row("fill_source_meta", &task_id, res.rows_affected()); + Ok(()) + } + + async fn complete(&self, entry: &TaskEntry) -> Result<()> { + let row = rows::task_entry_to_row(entry)?; + let report_created_at = row.completed_at.clone().unwrap_or_else(|| encode_ts(&Utc::now())); + let mut tx = self.pool().begin().await.context("begin complete review")?; + let res = ::sqlx::query( + "UPDATE reviews SET state = ?, result = ?, error = ?, completed_at = ?, progress = ? \ + WHERE task_id = ?", + ) + .bind(&row.state) + .bind(&row.result) + .bind(&row.error) + .bind(&row.completed_at) + .bind(row.progress) + .bind(&row.task_id) + .execute(&mut *tx) + .await + .with_context(|| format!("complete review {}", row.task_id))?; + warn_missing_row("complete", &entry.task_id, res.rows_affected()); + // Replace (not upsert) so a retried-then-completed task cannot hit + // the (task_id, expert_name) PK with stale rows. + ::sqlx::query("DELETE FROM expert_reports WHERE task_id = ?") + .bind(&row.task_id) + .execute(&mut *tx) + .await + .with_context(|| format!("clear expert_reports for {}", row.task_id))?; + if let Some(result) = &entry.result { + match rows::expert_report_rows(&entry.task_id, result, report_created_at) { + Ok(report_rows) => { + for report in &report_rows { + ::sqlx::query( + "INSERT INTO expert_reports (task_id, expert_name, report, duration_ms, created_at) \ + VALUES (?, ?, ?, ?, ?)", + ) + .bind(&report.task_id) + .bind(&report.expert_name) + .bind(&report.report) + .bind(report.duration_ms) + .bind(&report.created_at) + .execute(&mut *tx) + .await + .with_context(|| { + format!("insert expert_report {:?} for {}", report.expert_name, report.task_id) + })?; + } + } + // A result that is not a serialized ReviewOutput is not a + // transient failure — keep the terminal review row, drop the + // per-expert split. + Err(e) => tracing::warn!( + "could not split expert reports for task {}: {e:#}; storing the review row only", + entry.task_id + ), + } + } + tx.commit() + .await + .with_context(|| format!("commit complete review {}", row.task_id))?; + Ok(()) + } + + async fn mark_cancelled(&self, task_id: uuid::Uuid, completed_at: DateTime) -> Result<()> { + let res = ::sqlx::query("UPDATE reviews SET state = 'cancelled', completed_at = ? WHERE task_id = ?") + .bind(encode_ts(&completed_at)) + .bind(task_id.to_string()) + .execute(self.pool()) + .await + .with_context(|| format!("mark review {task_id} cancelled"))?; + warn_missing_row("mark_cancelled", &task_id, res.rows_affected()); + Ok(()) + } + + async fn mark_retry(&self, task_id: uuid::Uuid) -> Result<()> { + let res = + ::sqlx::query("UPDATE reviews SET state = 'pending', error = NULL, completed_at = NULL WHERE task_id = ?") + .bind(task_id.to_string()) + .execute(self.pool()) + .await + .with_context(|| format!("mark review {task_id} retried"))?; + warn_missing_row("mark_retry", &task_id, res.rows_affected()); + Ok(()) + } + + async fn mark_interrupted(&self, now: DateTime) -> Result { + let res = ::sqlx::query( + "UPDATE reviews SET state = 'failed', error = 'interrupted: server restarted', completed_at = ? \ + WHERE state IN ('pending', 'running')", + ) + .bind(encode_ts(&now)) + .execute(self.pool()) + .await + .context("interrupted-task sweep failed")?; + Ok(res.rows_affected()) + } +} + #[cfg(test)] mod tests { use super::*; @@ -492,4 +650,143 @@ mod tests { store.save_setting("ui", &serde_json::json!({})).await.unwrap(); assert!(!store.config_tables_empty().await.unwrap()); } + + // ─── ReviewStore (step 4) ─── + + use crate::server::task_queue::{TaskEntry, TaskState}; + + /// (d) the startup sweep flips ONLY pending/running rows to + /// failed/interrupted; terminal rows are untouched. + #[tokio::test] + async fn mark_interrupted_sweeps_only_pending_and_running() { + let store = fresh_store().await; + let now = Utc::now(); + for (id, state) in [ + ("t-pending", "pending"), + ("t-running", "running"), + ("t-completed", "completed"), + ("t-failed", "failed"), + ("t-cancelled", "cancelled"), + ] { + ::sqlx::query("INSERT INTO reviews (task_id, state, created_at) VALUES (?, ?, ?)") + .bind(id) + .bind(state) + .bind(encode_ts(&now)) + .execute(store.pool()) + .await + .unwrap(); + } + + let swept = ReviewStore::mark_interrupted(&store, now).await.unwrap(); + assert_eq!(swept, 2, "only pending + running are swept"); + + let rows: Vec<(String, String, Option, Option)> = + ::sqlx::query_as("SELECT task_id, state, error, completed_at FROM reviews ORDER BY task_id") + .fetch_all(store.pool()) + .await + .unwrap(); + for (id, state, error, completed_at) in &rows { + match id.as_str() { + "t-pending" | "t-running" => { + assert_eq!(state, "failed"); + assert_eq!(error.as_deref(), Some("interrupted: server restarted")); + assert!(completed_at.is_some(), "sweep stamps completed_at"); + } + other => { + assert_eq!(state, &other[2..], "terminal row {other} must be untouched"); + assert!(error.is_none() && completed_at.is_none()); + } + } + } + } + + /// (task_id, state, source_meta, project, repository, request, result, + /// error, progress, created_at, started_at, completed_at) + type RawReviewRow = ( + String, + String, + String, + Option, + Option, + Option, + Option, + Option, + Option, + String, + Option, + Option, + ); + + /// reviews row codec: create → read raw columns → decode back to a + /// TaskEntry that matches the original. + #[tokio::test] + async fn review_row_codec_round_trip() { + let store = fresh_store().await; + let entry = TaskEntry { + task_id: uuid::Uuid::new_v4(), + state: TaskState::Pending, + created_at: Utc::now(), + started_at: None, + completed_at: None, + result: None, + error: None, + request: Some(serde_json::json!({"mr_url": "https://gitlab.example/g/p/-/merge_requests/7"})), + source_meta: crate::server::task_queue::SourceMeta { + mr_title: Some("Add login".into()), + project: Some("g/p".into()), + repository: Some("p".into()), + ..Default::default() + }, + progress: None, + expert_name: None, + }; + ReviewStore::create(&store, &entry).await.unwrap(); + + let ( + task_id, + state, + source_meta, + project, + repository, + request, + result, + error, + progress, + created_at, + started_at, + completed_at, + ): RawReviewRow = ::sqlx::query_as( + "SELECT task_id, state, source_meta, project, repository, request, result, error, \ + progress, created_at, started_at, completed_at FROM reviews WHERE task_id = ?", + ) + .bind(entry.task_id.to_string()) + .fetch_one(store.pool()) + .await + .unwrap(); + // Materialized filter columns are in sync with source_meta. + assert_eq!(project.as_deref(), Some("g/p")); + assert_eq!(repository.as_deref(), Some("p")); + let decoded = rows::review_from_row(rows::ReviewRow { + task_id, + state, + source_meta, + project, + repository, + request, + result, + error, + progress, + created_at, + started_at, + completed_at, + }) + .unwrap(); + assert_eq!(decoded.task_id, entry.task_id); + assert_eq!(decoded.state, entry.state); + assert_eq!(decoded.request, entry.request); + assert_eq!(decoded.source_meta.mr_title.as_deref(), Some("Add login")); + assert_eq!(decoded.source_meta.project.as_deref(), Some("g/p")); + assert_eq!(decoded.created_at, entry.created_at); + assert!(decoded.expert_name.is_none(), "live-only field is not persisted"); + } } diff --git a/src/store/traits.rs b/src/store/traits.rs index 8de9b35..02a3d13 100644 --- a/src/store/traits.rs +++ b/src/store/traits.rs @@ -2,8 +2,8 @@ //! `ConfigStore` (git_platforms / llm_providers / app_settings), //! `DiscussionStore` (mr_discussions). //! -//! Only `ConfigStore` is defined so far (step 2 of the 0.10.0 persistence -//! rollout); the other two land with their implementations. +//! `ConfigStore` landed in step 2, `ReviewStore` in step 4 of the 0.10.0 +//! persistence rollout; `DiscussionStore` lands with its implementation. //! //! Semantics follow `UiStateFile` (design/persistence.md §4.2, §6.2): each //! domain is read and replaced AS A WHOLE — `PUT /config` resolves the full @@ -12,9 +12,12 @@ use anyhow::Result; use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use uuid::Uuid; use crate::models::{GitPlatformConfig, LLMConfig}; use crate::server::api::config::persist::{PersistedGitlabConfig, UiStateFile}; +use crate::server::task_queue::{SourceMeta, TaskEntry}; /// Persistence boundary for UI-managed configuration. /// @@ -72,3 +75,43 @@ pub trait ConfigStore: Send + Sync { /// (design/persistence.md §6.1 step 3). async fn config_tables_empty(&self) -> Result; } + +/// Persistence boundary for review tasks (`reviews` + `expert_reports`). +/// +/// Write-through contract (design/persistence.md §5): the in-memory +/// `TaskStore` stays the hot path / SSE source; every lifecycle transition +/// is mirrored here synchronously so the DB is the source of truth for +/// history across restarts. Callers treat failures as non-fatal (log and +/// continue) — a missing history row must never block a review. +#[async_trait] +pub trait ReviewStore: Send + Sync { + /// INSERT a new `reviews` row from the freshly created entry + /// (`state = 'pending'`, `request` / `source_meta` serialized). + async fn create(&self, entry: &TaskEntry) -> Result<()>; + + /// `UPDATE state='running', started_at=?` for a task claimed by a worker. + async fn mark_started(&self, task_id: Uuid, started_at: DateTime) -> Result<()>; + + /// `UPDATE source_meta=?` plus the materialized `project` / `repository` + /// filter columns (§3.2: read path must never touch JSON extraction). + async fn fill_source_meta(&self, task_id: Uuid, meta: &SourceMeta) -> Result<()>; + + /// Terminal write, in ONE transaction: `UPDATE reviews` with + /// state/result/error/completed_at/progress, then replace the task's + /// `expert_reports` rows (delete + re-INSERT per `result.reports` entry, + /// so a retried-then-completed task cannot hit the PK). + async fn complete(&self, entry: &TaskEntry) -> Result<()>; + + /// `UPDATE state='cancelled', completed_at=?` (cancel semantics of + /// `TaskStore::delete`). + async fn mark_cancelled(&self, task_id: Uuid, completed_at: DateTime) -> Result<()>; + + /// `UPDATE state='pending', error=NULL, completed_at=NULL` (retry: + /// `Failed → Pending`). + async fn mark_retry(&self, task_id: Uuid) -> Result<()>; + + /// Startup sweep (§5.3): every row still `pending` / `running` when the + /// previous process died becomes `failed` with + /// `error='interrupted: server restarted'`. Returns affected rows. + async fn mark_interrupted(&self, now: DateTime) -> Result; +} From 0afca9f242da8d00e94b871bac0ef33adbf9a49a Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 12:19:01 +0800 Subject: [PATCH 07/36] feat(api): history list/detail read from the DB with 0.9 fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.10.0 persistence step 5 (design/persistence.md §8.1): - store/traits.rs: ReviewStore gains the read side — ReviewListQuery (handler-normalized params), list_reviews() -> (entries, total), get_review() -> Option. - store/sqlx.rs: list pagination via ORDER BY created_at DESC, task_id DESC (deterministic tiebreak) + LIMIT/OFFSET, COUNT(*) under the same WHERE; status/q/project/repository/date_from/date_to filters; q keeps the 0.9 case-insensitive literal-substring semantics over source_meta TEXT (LOWER(...) LIKE LOWER(?) ESCAPE '\', LIKE wildcards escaped). - store/rows.rs: REVIEW_COLUMNS + ReviewRowTuple + From tuple -> ReviewRow; review_from_row decodes DB rows back to TaskEntry, so the three API projection functions (task_to_status / build_review_detail / build_review_list_item) keep their signatures and the response shape is byte-compatible with 0.9 (timestamps, duration_ms included). - handlers.rs: list_reviews / get_review switch on AppState::db — Some: DB query (get_review overlays live progress/expert_name from memory for in-flight tasks; a memory-only task whose create write-through failed is still served instead of 404); None (REVIEW_DISABLE_DB=1 / tests): the 0.9 in-memory path, unchanged. Tests (src/server/api/review/tests.rs): (a) pagination/status/q/project/ repository/date semantics vs 0.9 + item/top-level key-set parity against the memory path; (b) live progress/expert_name overlay for in-flight tasks, pure-history detail from DB, 404 for unknown; (c) db=None fallback for list+get; (d) empty DB and out-of-range page boundaries. fmt/clippy/test green (1596 passed, 0 failed). --- src/server/api/review/handlers.rs | 90 ++++++-- src/server/api/review/tests.rs | 353 ++++++++++++++++++++++++++++++ src/server/task_queue.rs | 9 + src/store/rows.rs | 61 +++++- src/store/sqlx.rs | 97 +++++++- src/store/traits.rs | 26 ++- 6 files changed, 612 insertions(+), 24 deletions(-) diff --git a/src/server/api/review/handlers.rs b/src/server/api/review/handlers.rs index 19304ef..ad4fb08 100644 --- a/src/server/api/review/handlers.rs +++ b/src/server/api/review/handlers.rs @@ -9,6 +9,7 @@ use uuid::Uuid; use crate::server::task_queue::{SourceMeta, TaskEntry, TaskState}; use crate::server::AppState; +use crate::store::traits::{ReviewListQuery, ReviewStore}; use super::super::types::{ReviewRequest, ReviewSource}; use super::resolve; @@ -205,20 +206,48 @@ pub(crate) async fn submit_review( } pub(crate) async fn get_review(State(state): State>, Path(task_id): Path) -> impl IntoResponse { - let store = match &state.task_store { - Some(s) => s, - None => return error_response(StatusCode::SERVICE_UNAVAILABLE, "task store not initialized"), + if state.task_store.is_none() && state.db.is_none() { + return error_response(StatusCode::SERVICE_UNAVAILABLE, "task store not initialized"); + } + let live_entry = match &state.task_store { + Some(s) => s.get(task_id).await, + None => None, }; - match store.get(task_id).await { - Some(entry) => { - let mut status_value = serde_json::to_value(task_to_status(&entry)).unwrap_or_default(); - if let Ok(detail_value) = serde_json::to_value(build_review_detail(&entry)) { - merge_camel_case_fields(&mut status_value, &detail_value); + let entry = if let Some(db) = &state.db { + // 0.10.0 (§8.1): the DB is the history source. An in-memory hit only + // overlays the two live-only fields (progress / current expert) that + // write-through deliberately never persists mid-flight. + match db.get_review(task_id).await { + Ok(Some(mut entry)) => { + if let Some(live) = &live_entry { + entry.progress = live.progress; + entry.expert_name = live.expert_name.clone(); + } + entry } - (StatusCode::OK, Json(status_value)).into_response() + Ok(None) => match live_entry { + // Write-through failure edge: the task is alive in memory but + // never landed in the DB — serve it rather than 404. + Some(entry) => entry, + None => return error_response(StatusCode::NOT_FOUND, "task not found"), + }, + Err(e) => { + tracing::error!("failed to load review {task_id} from the database: {e:#}"); + return error_response(StatusCode::INTERNAL_SERVER_ERROR, "failed to load review"); + } + } + } else { + // db=None (REVIEW_DISABLE_DB=1, tests): the 0.9 pure in-memory path. + match live_entry { + Some(entry) => entry, + None => return error_response(StatusCode::NOT_FOUND, "task not found"), } - None => error_response(StatusCode::NOT_FOUND, "task not found"), + }; + let mut status_value = serde_json::to_value(task_to_status(&entry)).unwrap_or_default(); + if let Ok(detail_value) = serde_json::to_value(build_review_detail(&entry)) { + merge_camel_case_fields(&mut status_value, &detail_value); } + (StatusCode::OK, Json(status_value)).into_response() } pub(crate) async fn rerun_review( @@ -326,19 +355,42 @@ pub(crate) async fn list_reviews( .map(|dt| dt.with_timezone(&chrono::Utc)) }); - let (items, total) = store - .list( - status, + // 0.10.0 (§8.1): with persistence active the DB is the data source + // (in-flight tasks are present via write-through — no memory merge). + // db=None keeps the 0.9 in-memory path (REVIEW_DISABLE_DB=1, tests). + let (entries, total) = if let Some(db) = &state.db { + let query = ReviewListQuery { + status: status.clone(), page, per_page, - params.q.as_deref(), - params.project.as_deref(), - params.repository.as_deref(), + q: params.q.clone(), + project: params.project.clone(), + repository: params.repository.clone(), date_from, date_to, - ) - .await; - let items: Vec = items + }; + match db.list_reviews(&query).await { + Ok(result) => result, + Err(e) => { + tracing::error!("failed to list reviews from the database: {e:#}"); + return error_response(StatusCode::INTERNAL_SERVER_ERROR, "failed to load review history"); + } + } + } else { + store + .list( + status, + page, + per_page, + params.q.as_deref(), + params.project.as_deref(), + params.repository.as_deref(), + date_from, + date_to, + ) + .await + }; + let items: Vec = entries .iter() .map(|entry| { let mut status_value = serde_json::to_value(task_to_status(entry)).unwrap_or_default(); diff --git a/src/server/api/review/tests.rs b/src/server/api/review/tests.rs index 52bbe17..44769e7 100644 --- a/src/server/api/review/tests.rs +++ b/src/server/api/review/tests.rs @@ -1362,3 +1362,356 @@ async fn rerun_with_stored_request_llm_configs_passes_gate() { assert_ne!(new_id, original_id, "rerun must create a fresh task id"); assert!(store.get(new_id).await.is_some(), "a new task must be enqueued"); } + +// ─── 0.10.0 history API reads the DB (design/persistence.md §8.1) ─── + +use crate::store::SqlxStore; +use std::collections::BTreeSet; + +/// State whose task store writes through to an in-memory SQLite DB — the +/// wiring app.rs does at startup (`set_db` + `AppState::db`). +async fn state_with_db() -> (Arc, Arc) { + let db = Arc::new(SqlxStore::new_in_memory().await.unwrap()); + db.migrate().await.unwrap(); + let mut store = TaskStore::new(); + store.set_db(db.clone()); + let mut state = AppState::new(vec![usable_llm_config()]); + state.task_store = Some(Arc::new(store)); + state.db = Some(db.clone()); + (Arc::new(state), db) +} + +fn empty_params() -> ListParams { + ListParams { + status: None, + page: None, + per_page: None, + q: None, + project: None, + repository: None, + date_from: None, + date_to: None, + } +} + +async fn list_json(state: Arc, params: ListParams) -> serde_json::Value { + let resp = list_reviews(State(state), Query(params)).await.into_response(); + let (status, json) = response_json(resp).await; + assert_eq!(status, StatusCode::OK, "list_reviews must succeed, got {json}"); + json +} + +/// Seed one `reviews` row directly with a fixed `created_at`, so list order +/// is deterministic (no wall-clock ties). +async fn seed_review_row( + db: &SqlxStore, + id: Uuid, + state_str: &str, + created_at: &str, + completed_at: Option<&str>, + meta: &SourceMeta, + result: Option, +) { + sqlx::query( + "INSERT INTO reviews (task_id, state, source_meta, project, repository, result, created_at, completed_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(id.to_string()) + .bind(state_str) + .bind(serde_json::to_string(meta).unwrap()) + .bind(&meta.project) + .bind(&meta.repository) + .bind(result.map(|v| v.to_string())) + .bind(created_at) + .bind(completed_at) + .execute(db.pool()) + .await + .unwrap(); +} + +fn meta_titled(title: &str, project: &str, repository: &str) -> SourceMeta { + SourceMeta { + mr_title: Some(title.to_string()), + project: Some(project.to_string()), + repository: Some(repository.to_string()), + ..SourceMeta::default() + } +} + +fn item_ids(json: &serde_json::Value) -> Vec { + json["items"] + .as_array() + .unwrap() + .iter() + .map(|item| item["task_id"].as_str().unwrap().to_string()) + .collect() +} + +/// (a) pagination / filter / q / project / date parameters keep their 0.9 +/// semantics with the DB as the data source; the response shape is asserted +/// key-for-key against the 0.9 in-memory path. +#[tokio::test] +async fn list_reviews_db_pagination_and_filters_match_0_9() { + let (state, db) = state_with_db().await; + + let t1 = Uuid::new_v4(); + let t2 = Uuid::new_v4(); + let t3 = Uuid::new_v4(); + let t4 = Uuid::new_v4(); + let t5 = Uuid::new_v4(); + seed_review_row( + &db, + t1, + "completed", + "2026-09-01T10:00:00.000000Z", + Some("2026-09-01T10:05:00.000000Z"), + &meta_titled("Fix login bug", "group/a", "a"), + None, + ) + .await; + seed_review_row( + &db, + t2, + "failed", + "2026-09-02T10:00:00.000000Z", + Some("2026-09-02T10:01:00.000000Z"), + &meta_titled("Add metrics endpoint", "group/b", "b"), + None, + ) + .await; + seed_review_row( + &db, + t3, + "completed", + "2026-09-03T10:00:00.000000Z", + Some("2026-09-03T10:05:00.000000Z"), + &meta_titled("Refactor auth flow", "group/a", "a"), + None, + ) + .await; + seed_review_row( + &db, + t4, + "running", + "2026-09-03T12:00:00.000000Z", + None, + &meta_titled("Login page tweaks", "group/c", "c"), + None, + ) + .await; + seed_review_row( + &db, + t5, + "pending", + "2026-09-03T13:00:00.000000Z", + None, + &SourceMeta::default(), + None, + ) + .await; + + // Default page: newest first, all five. + let json = list_json(state.clone(), empty_params()).await; + assert_eq!(json["total"], 5); + assert_eq!(json["page"], 1); + assert_eq!(json["per_page"], 20); + assert_eq!( + item_ids(&json), + vec![ + t5.to_string(), + t4.to_string(), + t3.to_string(), + t2.to_string(), + t1.to_string() + ] + ); + + // Second page of two. + let mut params = empty_params(); + params.per_page = Some(2); + params.page = Some(2); + let json = list_json(state.clone(), params).await; + assert_eq!(json["total"], 5); + assert_eq!(json["page"], 2); + assert_eq!(json["per_page"], 2); + assert_eq!(item_ids(&json), vec![t3.to_string(), t2.to_string()]); + + // per_page is clamped to 100 (0.9 behaviour). + let mut params = empty_params(); + params.per_page = Some(500); + let json = list_json(state.clone(), params).await; + assert_eq!(json["per_page"], 100); + assert_eq!(json["total"], 5); + + // Status filter. + let mut params = empty_params(); + params.status = Some("completed".to_string()); + let json = list_json(state.clone(), params).await; + assert_eq!(json["total"], 2); + assert_eq!(item_ids(&json), vec![t3.to_string(), t1.to_string()]); + + // q: case-insensitive substring over source_meta ("login" hits t1 + t4). + let mut params = empty_params(); + params.q = Some("LOGIN".to_string()); + let json = list_json(state.clone(), params).await; + assert_eq!(json["total"], 2); + assert_eq!(item_ids(&json), vec![t4.to_string(), t1.to_string()]); + // LIKE wildcards in the needle stay literal (0.9 used `contains`). + let mut params = empty_params(); + params.q = Some("100%".to_string()); + let json = list_json(state.clone(), params).await; + assert_eq!(json["total"], 0); + + // project / repository hit the materialized columns. + let mut params = empty_params(); + params.project = Some("group/a".to_string()); + let json = list_json(state.clone(), params).await; + assert_eq!(item_ids(&json), vec![t3.to_string(), t1.to_string()]); + let mut params = empty_params(); + params.repository = Some("b".to_string()); + let json = list_json(state.clone(), params).await; + assert_eq!(item_ids(&json), vec![t2.to_string()]); + + // created_at range. + let mut params = empty_params(); + params.date_from = Some("2026-09-03T00:00:00Z".to_string()); + let json = list_json(state.clone(), params).await; + assert_eq!(item_ids(&json), vec![t5.to_string(), t4.to_string(), t3.to_string()]); + let mut params = empty_params(); + params.date_to = Some("2026-09-02T23:59:59Z".to_string()); + let json = list_json(state.clone(), params).await; + assert_eq!(item_ids(&json), vec![t2.to_string(), t1.to_string()]); + + // Response shape: the DB-path item/top-level key sets are identical to + // the 0.9 in-memory path's. + let mem_state = state_with_store(); + let mem_store = mem_state.task_store.clone().unwrap(); + let mid = mem_store.create(Some(source_meta_with_commit())).await; + mem_store + .update( + mid, + TaskState::Completed, + Some(serde_json::json!({"reports": []})), + None, + ) + .await; + let mem_json = list_json(mem_state, empty_params()).await; + + let key_set = |v: &serde_json::Value| -> BTreeSet { v.as_object().unwrap().keys().cloned().collect() }; + assert_eq!( + key_set(&json["items"][0]), + key_set(&mem_json["items"][0]), + "list item shape changed" + ); + assert_eq!(key_set(&json), key_set(&mem_json), "top-level shape changed"); +} + +/// (b) get_review: in-flight tasks overlay live progress/expert_name from +/// memory onto the DB row; a task that exists only in the DB (e.g. after a +/// restart / reaper pass) is served purely from history. +#[tokio::test] +async fn get_review_db_read_with_live_overlay() { + let (state, db) = state_with_db().await; + let store = state.task_store.clone().unwrap(); + + // In-flight: write-through persists the row, progress stays memory-only. + let id = crate::server::task_queue::record_task_started(&store, source_meta_with_commit()).await; + store.set_progress(id, 42, Some("security".to_string())).await; + + let resp = get_review(State(state.clone()), Path(id)).await.into_response(); + let (status, json) = response_json(resp).await; + assert_eq!(status, StatusCode::OK, "in-flight task must resolve, got {json}"); + assert_eq!(json["status"], "running"); + assert_eq!(json["progress"], 42, "live progress overlays the DB row"); + assert_eq!(json["expert_name"], "security"); + assert_eq!(json["mrTitle"], "Fix login bug"); + + // Pure history: DB row only, nothing in memory. + let hid = Uuid::new_v4(); + let output = crate::models::ReviewOutput::new(vec![make_report( + "security", + vec![make_finding(crate::models::Severity::High)], + )]); + seed_review_row( + &db, + hid, + "completed", + "2026-09-01T10:00:00.000000Z", + Some("2026-09-01T10:05:00.000000Z"), + &source_meta_with_commit(), + Some(serde_json::to_value(&output).unwrap()), + ) + .await; + assert!(store.get(hid).await.is_none(), "history row must not be in memory"); + + let resp = get_review(State(state.clone()), Path(hid)).await.into_response(); + let (status, json) = response_json(resp).await; + assert_eq!( + status, + StatusCode::OK, + "history-only task must resolve from the DB, got {json}" + ); + assert_eq!(json["status"], "completed"); + assert_eq!(json["task_id"], hid.to_string()); + assert!(json["progress"].is_null(), "no live overlay for history rows"); + assert_eq!(json["experts"][0]["expertName"], "security"); + assert_eq!(json["duration_ms"], 5 * 60 * 1000); + assert!(json["rawApiResponse"].is_object()); + + // Unknown task → 404. + let resp = get_review(State(state), Path(Uuid::new_v4())).await.into_response(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); +} + +/// (c) db=None (REVIEW_DISABLE_DB=1 / tests) keeps the 0.9 in-memory +/// behaviour for both list and get. +#[tokio::test] +async fn db_none_falls_back_to_in_memory_list_and_get() { + let state = state_with_store(); + assert!(state.db.is_none(), "this state must exercise the fallback path"); + let store = state.task_store.clone().unwrap(); + + let id = store.create(Some(source_meta_with_commit())).await; + store + .update(id, TaskState::Completed, Some(serde_json::json!({"reports": []})), None) + .await; + + let json = list_json(state.clone(), empty_params()).await; + assert_eq!(json["total"], 1); + assert_eq!(json["items"][0]["task_id"], id.to_string()); + assert_eq!(json["items"][0]["status"], "completed"); + + let resp = get_review(State(state.clone()), Path(id)).await.into_response(); + let (status, json) = response_json(resp).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(json["status"], "completed"); + + let resp = get_review(State(state), Path(Uuid::new_v4())).await.into_response(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); +} + +/// (d) empty-DB pagination boundaries: empty result set, and a page beyond +/// the last row returns an empty list with the correct total. +#[tokio::test] +async fn list_reviews_db_empty_and_out_of_range_pages() { + let (state, _db) = state_with_db().await; + + let json = list_json(state.clone(), empty_params()).await; + assert_eq!(json["total"], 0); + assert!(json["items"].as_array().unwrap().is_empty()); + + // One task (via write-through), then ask for a page past the end. + let store = state.task_store.clone().unwrap(); + store.create(Some(source_meta_with_commit())).await; + + let mut params = empty_params(); + params.page = Some(99); + params.per_page = Some(1); + let json = list_json(state.clone(), params).await; + assert_eq!(json["total"], 1, "total reflects the filters, not the page"); + assert!( + json["items"].as_array().unwrap().is_empty(), + "out-of-range page is an empty list" + ); + assert_eq!(json["page"], 99); +} diff --git a/src/server/task_queue.rs b/src/server/task_queue.rs index 82f5702..5abf3dd 100644 --- a/src/server/task_queue.rs +++ b/src/server/task_queue.rs @@ -1065,6 +1065,15 @@ mod tests { async fn mark_interrupted(&self, _: chrono::DateTime) -> anyhow::Result { anyhow::bail!("db down") } + async fn list_reviews( + &self, + _: &crate::store::traits::ReviewListQuery, + ) -> anyhow::Result<(Vec, u64)> { + anyhow::bail!("db down") + } + async fn get_review(&self, _: Uuid) -> anyhow::Result> { + anyhow::bail!("db down") + } } let failing = Arc::new(FailingStore::default()); diff --git a/src/store/rows.rs b/src/store/rows.rs index 2c8baf2..e84f7c4 100644 --- a/src/store/rows.rs +++ b/src/store/rows.rs @@ -252,9 +252,64 @@ pub(crate) fn task_entry_to_row(entry: &TaskEntry) -> Result { }) } -/// Decode a `reviews` row back into a [`TaskEntry`]. Used by tests now and -/// by the history read path in the next step (§8.1). -#[allow(dead_code)] +/// Column list of the shared `reviews` SELECT used by the read path +/// (`sqlx.rs`); the order matches [`ReviewRowTuple`]. +pub(crate) const REVIEW_COLUMNS: &str = "task_id, state, source_meta, project, repository, request, \ + result, error, progress, created_at, started_at, completed_at"; + +/// Raw decode target for a `SELECT {REVIEW_COLUMNS}` query, in column order. +#[allow(clippy::type_complexity)] +pub(crate) type ReviewRowTuple = ( + String, + String, + String, + Option, + Option, + Option, + Option, + Option, + Option, + String, + Option, + Option, +); + +impl From for ReviewRow { + fn from( + ( + task_id, + state, + source_meta, + project, + repository, + request, + result, + error, + progress, + created_at, + started_at, + completed_at, + ): ReviewRowTuple, + ) -> Self { + Self { + task_id, + state, + source_meta, + project, + repository, + request, + result, + error, + progress, + created_at, + started_at, + completed_at, + } + } +} + +/// Decode a `reviews` row back into a [`TaskEntry`]. Used by the history +/// read path (`ReviewStore::list_reviews` / `get_review`, §8.1) and by tests. pub(crate) fn review_from_row(row: ReviewRow) -> Result { fn opt_ts(raw: Option, what: &str) -> Result>> { raw.as_deref() diff --git a/src/store/sqlx.rs b/src/store/sqlx.rs index 38c3e04..1aa4348 100644 --- a/src/store/sqlx.rs +++ b/src/store/sqlx.rs @@ -16,7 +16,7 @@ use crate::server::api::config::persist::{PersistedGitlabConfig, UiStateFile}; use crate::server::task_queue::{SourceMeta, TaskEntry}; use super::rows; -use super::traits::{ConfigStore, ReviewStore}; +use super::traits::{ConfigStore, ReviewListQuery, ReviewStore}; use super::{encode_ts, SqlxStore}; const LEGACY_GITLAB_KEY: &str = "gitlab"; @@ -437,6 +437,101 @@ impl ReviewStore for SqlxStore { .context("interrupted-task sweep failed")?; Ok(res.rows_affected()) } + + async fn list_reviews(&self, query: &ReviewListQuery) -> Result<(Vec, u64)> { + let (where_sql, binds) = review_where(query); + + let count_sql = format!("SELECT COUNT(*) FROM reviews {where_sql}"); + let mut count_q = ::sqlx::query_scalar::<_, i64>(&count_sql); + for value in &binds { + count_q = count_q.bind(value); + } + let total = count_q.fetch_one(self.pool()).await.context("count reviews")?; + + // Every bind value in `binds` is a String (state/q/project/repository + // and both timestamps), so the COUNT and the page SELECT share the + // same positional parameter list; LIMIT/OFFSET trail as two more. + let offset = query.page.saturating_sub(1).saturating_mul(query.per_page); + let list_sql = format!( + "SELECT {} FROM reviews {where_sql} ORDER BY created_at DESC, task_id DESC LIMIT ? OFFSET ?", + rows::REVIEW_COLUMNS + ); + let mut list_q = ::sqlx::query_as::<_, rows::ReviewRowTuple>(&list_sql); + for value in &binds { + list_q = list_q.bind(value); + } + let rows = list_q + .bind(query.per_page as i64) + .bind(offset as i64) + .fetch_all(self.pool()) + .await + .context("list reviews")?; + let entries = rows + .into_iter() + .map(|tuple| rows::review_from_row(tuple.into())) + .collect::>>() + .context("decode reviews rows")?; + Ok((entries, total as u64)) + } + + async fn get_review(&self, task_id: uuid::Uuid) -> Result> { + let row = ::sqlx::query_as::<_, rows::ReviewRowTuple>(&format!( + "SELECT {} FROM reviews WHERE task_id = ?", + rows::REVIEW_COLUMNS + )) + .bind(task_id.to_string()) + .fetch_optional(self.pool()) + .await + .with_context(|| format!("load review {task_id}"))?; + row.map(|tuple| rows::review_from_row(tuple.into())) + .transpose() + .with_context(|| format!("decode review row {task_id}")) + } +} + +/// Shared WHERE clause + positional binds for the history list (§8.1). All +/// bind values are Strings so the COUNT and the page SELECT can share them. +/// +/// `q` keeps the 0.9 semantics — a case-insensitive literal substring match +/// — applied to the serialized `source_meta` TEXT (design prescribes +/// `LOWER(...) LIKE LOWER(?)`; LIKE wildcards in the needle are escaped so +/// the match stays literal). +fn review_where(query: &ReviewListQuery) -> (String, Vec) { + let mut clauses: Vec<&str> = Vec::new(); + let mut binds: Vec = Vec::new(); + if let Some(status) = &query.status { + clauses.push("state = ?"); + binds.push(rows::task_state_str(status).to_string()); + } + if let Some(q) = &query.q { + clauses.push("LOWER(source_meta) LIKE LOWER(?) ESCAPE '\\'"); + let needle = q.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_"); + binds.push(format!("%{needle}%")); + } + if let Some(project) = &query.project { + clauses.push("project = ?"); + binds.push(project.clone()); + } + if let Some(repository) = &query.repository { + clauses.push("repository = ?"); + binds.push(repository.clone()); + } + if let Some(from) = &query.date_from { + // Fixed-width RFC 3339 UTC: lexicographic compare == chronological + // compare (§3.1 timestamp row). + clauses.push("created_at >= ?"); + binds.push(encode_ts(from)); + } + if let Some(to) = &query.date_to { + clauses.push("created_at <= ?"); + binds.push(encode_ts(to)); + } + let where_sql = if clauses.is_empty() { + String::new() + } else { + format!("WHERE {}", clauses.join(" AND ")) + }; + (where_sql, binds) } #[cfg(test)] diff --git a/src/store/traits.rs b/src/store/traits.rs index 02a3d13..fa4b3cb 100644 --- a/src/store/traits.rs +++ b/src/store/traits.rs @@ -17,7 +17,7 @@ use uuid::Uuid; use crate::models::{GitPlatformConfig, LLMConfig}; use crate::server::api::config::persist::{PersistedGitlabConfig, UiStateFile}; -use crate::server::task_queue::{SourceMeta, TaskEntry}; +use crate::server::task_queue::{SourceMeta, TaskEntry, TaskState}; /// Persistence boundary for UI-managed configuration. /// @@ -114,4 +114,28 @@ pub trait ReviewStore: Send + Sync { /// previous process died becomes `failed` with /// `error='interrupted: server restarted'`. Returns affected rows. async fn mark_interrupted(&self, now: DateTime) -> Result; + + /// History list (§8.1): newest first (`ORDER BY created_at DESC`), + /// paginated, plus the total row count under the same filters. The DB is + /// the only data source — in-flight tasks are present via write-through, + /// no memory merge. + async fn list_reviews(&self, query: &ReviewListQuery) -> Result<(Vec, u64)>; + + /// Single history row; `None` when the task is unknown. + async fn get_review(&self, task_id: Uuid) -> Result>; +} + +/// Handler-normalized history-list parameters (design/persistence.md §8.1 — +/// the DB takes over what `TaskStore::list` filtered in memory in 0.9): +/// `page` is 1-based (≥ 1), `per_page` is already clamped to ≤ 100. +#[derive(Debug, Clone, Default)] +pub struct ReviewListQuery { + pub status: Option, + pub page: u64, + pub per_page: u64, + pub q: Option, + pub project: Option, + pub repository: Option, + pub date_from: Option>, + pub date_to: Option>, } From d31a84d72321de5ab484c5d277206f0c0d27f658 Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 12:36:43 +0800 Subject: [PATCH 08/36] feat(store): DiscussionStore + Note webhook ingestion into mr_discussions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.10.0 persistence step 6a (design/persistence.md §7.1): - store/traits.rs: DiscussionStore — upsert_note (idempotent on (platform, project, mr_iid, note_id), edits update body/author) and list_notes (created_at, note_id ascending — the append-only order the step-6b context renderer relies on). store/rows.rs: row codecs; store/sqlx.rs: SqlxStore impl (? placeholders, ON CONFLICT upsert). - server/gitlab/hooks.rs: handle_note_hook gains db: Option>; ingestion runs after parse, BEFORE the command check (command notes are discussion history too). Skips: non-note payloads, non-MR noteables, system notes. MR iid falls back to the object_attributes.url tail; platform defaults to 'default'; author prefers user.username. - Self-echo guard (§7.1): (a) notes starting with the published report prefix — extracted as publisher::REVIEW_REPORT_PREFIX ("# CodeReview Board\n\n", previously inline in lib.rs publish_review); (b) notes authored by the service's own GitLab user id, resolved lazily per platform via GET /user (new Client::for_instance) and cached for the process lifetime (only successes cached; failures retry next note). /review and /describe command notes always ingest (user intent). - handler.rs passes AppState::db through both note-hook call sites. db=None keeps exact 0.9 behaviour; ingestion failures are logged, never fail the hook. Tests: ingestion field fidelity (incl. GitLab legacy '… UTC' timestamp format), platform name + iid URL fallback, redelivery dedup, edit-in-place, non-MR / system note skips, self-echo guards (a)+(b) with command exception, db=None 0.9 parity, store-level list ordering. fmt/clippy/test green (1605 passed, 0 failed). --- src/git_provider/gitlab/client.rs | 14 + src/lib.rs | 2 +- src/publisher/mod.rs | 7 + src/server/gitlab/handler.rs | 22 +- src/server/gitlab/hooks.rs | 412 ++++++++++++++++++++++++++++++ src/store/rows.rs | 34 +++ src/store/sqlx.rs | 109 +++++++- src/store/traits.rs | 34 +++ 8 files changed, 622 insertions(+), 12 deletions(-) diff --git a/src/git_provider/gitlab/client.rs b/src/git_provider/gitlab/client.rs index 451be5d..a6ec263 100644 --- a/src/git_provider/gitlab/client.rs +++ b/src/git_provider/gitlab/client.rs @@ -131,6 +131,20 @@ impl Client { Ok(client) } + /// Client scoped to a whole GitLab instance (no project/MR binding), for + /// instance-level endpoints such as `GET /user`. `instance_base` is the + /// instance root (e.g. `https://gitlab.example.com`); `/api/v4` is + /// appended here. + pub fn for_instance(gitlab_token: &str, instance_base: &str) -> Self { + Self { + http: HttpClient::new(), + base_url: format!("{}/api/v4", instance_base.trim_end_matches('/')), + project_path: String::new(), + mr_iid: 0, + gitlab_token: gitlab_token.to_string(), + } + } + fn encoded_project_path(&self) -> String { encode_project_path(&self.project_path) } diff --git a/src/lib.rs b/src/lib.rs index 8c5791f..67d575e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -175,7 +175,7 @@ pub async fn publish_review(token: &str, mr_url: &str, output: &ReviewOutput) -> .context("Failed to create GitLabProvider")? }; - let mut md = String::from("# CodeReview Board\n\n"); + let mut md = String::from(crate::publisher::REVIEW_REPORT_PREFIX); for report in &output.reports { // render_expert_section appends the parse-failure / raw-response // annotations that the pre-rendered `markdown` does not carry, so a diff --git a/src/publisher/mod.rs b/src/publisher/mod.rs index 7d93ac5..219e99b 100644 --- a/src/publisher/mod.rs +++ b/src/publisher/mod.rs @@ -8,6 +8,13 @@ use anyhow::Result; +/// Fixed header of the review report this service posts to the MR +/// (`publish_review`, lib.rs). The Note-hook ingestion path skips notes +/// starting with this prefix — self-echo guard (a) of +/// design/persistence.md §7.1, so our own report never re-enters the +/// discussion history it was published into. +pub const REVIEW_REPORT_PREFIX: &str = "# CodeReview Board\n\n"; + /// A note to be posted on a specific line of a file in a merge request. #[derive(Debug, Clone)] pub struct InlineNote { diff --git a/src/server/gitlab/handler.rs b/src/server/gitlab/handler.rs index 19f5e87..7c918e7 100644 --- a/src/server/gitlab/handler.rs +++ b/src/server/gitlab/handler.rs @@ -370,13 +370,14 @@ impl GitLabWebhookHandler { token: &str, platform: Option, task_store: Option>, + db: Option>, ) -> Result, (StatusCode, Json)> { let event_name = system_hook_event_name(body); match event_name.as_str() { "merge_request" => super::handle_mr_hook(body, &self.dispatcher, token, platform, task_store.clone()) .await .map_err(|status| (status, Json(serde_json::json!({"error": "request failed"})))), - "note" => super::handle_note_hook(body, &self.dispatcher, token, platform, task_store.clone()) + "note" => super::handle_note_hook(body, &self.dispatcher, token, platform, task_store.clone(), db) .await .map_err(|status| (status, Json(serde_json::json!({"error": "request failed"})))), "push" => super::handle_push_hook(body) @@ -459,23 +460,24 @@ impl WebhookHandler for GitLabWebhookHandler { // The shared task store (weak-handled via AppState) so webhook-dispatched // reviews record a task entry: create → running → completed/failed. `None` // in tests and legacy paths — the review still runs, just without a record. - let task_store = self - .app_state - .as_ref() - .and_then(|w| w.upgrade()) - .and_then(|s| s.task_store.clone()); + // The DB handle (0.10.0) feeds Note-hook ingestion; `None` = 0.9 behaviour. + let app_state = self.app_state.as_ref().and_then(|w| w.upgrade()); + let task_store = app_state.as_ref().and_then(|s| s.task_store.clone()); + let db = app_state.and_then(|s| s.db.clone()); match event { "Merge Request Hook" => super::handle_mr_hook(body, &self.dispatcher, &token, platform, task_store.clone()) .await .map_err(|status| (status, Json(serde_json::json!({"error": "request failed"})))), - "Note Hook" => super::handle_note_hook(body, &self.dispatcher, &token, platform, task_store.clone()) - .await - .map_err(|status| (status, Json(serde_json::json!({"error": "request failed"})))), + "Note Hook" => { + super::handle_note_hook(body, &self.dispatcher, &token, platform, task_store.clone(), db.clone()) + .await + .map_err(|status| (status, Json(serde_json::json!({"error": "request failed"})))) + } "Push Hook" => super::handle_push_hook(body) .await .map_err(|status| (status, Json(serde_json::json!({"error": "request failed"})))), - "System Hook" => self.handle_system_hook(body, &token, platform, task_store).await, + "System Hook" => self.handle_system_hook(body, &token, platform, task_store, db).await, _ => { tracing::debug!("Ignoring unsupported GitLab event: {}", event); Ok(Json(serde_json::json!({ "status": "ignored" }))) diff --git a/src/server/gitlab/hooks.rs b/src/server/gitlab/hooks.rs index 734b1cb..0dfc140 100644 --- a/src/server/gitlab/hooks.rs +++ b/src/server/gitlab/hooks.rs @@ -4,6 +4,8 @@ use std::sync::Arc; use super::super::dispatcher::MrDispatcher; use crate::server::task_queue::{record_task_outcome, record_task_started, SourceMeta, TaskStore}; +use crate::store::traits::{DiscussionNote, DiscussionStore}; +use crate::store::SqlxStore; /// Parsed payload from a GitLab Merge Request webhook event. pub struct MrHookPayload { @@ -405,18 +407,188 @@ pub(crate) fn mr_iid_from_url(url: &str) -> Option { } } +// ─── 0.10.0 Note ingestion (design/persistence.md §7.1) ─── + +/// `object_attributes.created_at` from a note webhook. GitLab sends either +/// RFC 3339 or its legacy `"2026-09-03 10:00:00 UTC"` format; both decode to +/// UTC. `None` when absent/unparseable (the caller falls back to now()). +fn parse_note_created_at(raw: Option<&str>) -> Option> { + let raw = raw?; + if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(raw) { + return Some(dt.with_timezone(&chrono::Utc)); + } + chrono::NaiveDateTime::parse_from_str(raw, "%Y-%m-%d %H:%M:%S UTC") + .ok() + .map(|naive| naive.and_utc()) +} + +/// `scheme://host[:port]` of a URL — the instance root for instance-level +/// API calls derived from a payload URL. +fn url_origin(url: &str) -> Option { + let (scheme, rest) = url.split_once("://")?; + let host = rest.split('/').next().unwrap_or(""); + if host.is_empty() { + None + } else { + Some(format!("{scheme}://{host}")) + } +} + +/// Self-echo guard (b) of §7.1: the GitLab user id this service's token +/// posts as, per platform ("default" for the unmatched/runtime-token path). +/// Resolved LAZILY on the first note hook (GET /user) and cached for the +/// process lifetime; only successes are cached, so a transient API failure +/// retries on the next note instead of permanently disabling the guard. +/// The platform set is runtime-mutable (PUT /config), which is why this is +/// not resolved once at startup. +static SELF_USER_IDS: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + +async fn resolve_self_user_id( + platform: Option<&crate::models::GitPlatformConfig>, + gitlab_token: &str, + parsed: &Value, +) -> Option { + if gitlab_token.is_empty() { + return None; + } + let key = platform + .map(|p| p.name.clone()) + .unwrap_or_else(|| "default".to_string()); + let cache = SELF_USER_IDS.get_or_init(|| tokio::sync::Mutex::new(std::collections::HashMap::new())); + if let Some(id) = cache.lock().await.get(&key) { + return Some(*id); + } + // The instance to ask: the matched platform's reachable base (internal + // when configured, mirroring the review URL rewrite), else the origin of + // the payload's own URLs. + let instance_base = match platform { + Some(p) => review_base_url(p).to_string(), + None => { + let url = parsed["project"]["web_url"] + .as_str() + .or_else(|| parsed["project"]["homepage"].as_str()) + .or_else(|| parsed["object_attributes"]["url"].as_str())?; + url_origin(url)? + } + }; + let client = crate::git_provider::gitlab::client::Client::for_instance(gitlab_token, &instance_base); + match client.get_current_user_id().await { + Ok(id) => { + cache.lock().await.insert(key, id); + Some(id) + } + Err(e) => { + tracing::warn!( + "could not resolve the service's own GitLab user id ({key}): {e:#}; \ + self-echo guard (b) inactive for this note" + ); + None + } + } +} + +/// Persist one note-webhook payload into `mr_discussions` (§7.1). +/// Best-effort: every failure is logged and swallowed — note ingestion is an +/// enhancement, never a reason to fail the hook. Skips: non-note payloads, +/// non-MR notes (Commit/Issue/Snippet), system notes, and our own output +/// (self-echo guard) — except `/review` / `/describe` command notes, which +/// are user intent and always ingest. +async fn ingest_note( + db: &SqlxStore, + parsed: &Value, + platform: Option<&crate::models::GitPlatformConfig>, + gitlab_token: &str, +) { + if parsed["object_kind"].as_str() != Some("note") { + return; + } + let attrs = &parsed["object_attributes"]; + if attrs["noteable_type"].as_str() != Some("MergeRequest") { + return; + } + // System notes ("added 1 commit") are noise, not discussion (§7.1). + if attrs["system"].as_bool() == Some(true) { + return; + } + let Some(note_id) = attrs["id"].as_u64() else { + return; + }; + let project = parsed["project"]["path_with_namespace"].as_str().unwrap_or(""); + // `merge_request.iid`, falling back to the `object_attributes.url` tail + // (system-hook notes may omit the merge_request block). + let mr_iid = parsed["merge_request"]["iid"] + .as_u64() + .or_else(|| attrs["url"].as_str().and_then(mr_iid_from_url)); + let Some(mr_iid) = mr_iid.filter(|_| !project.is_empty()) else { + tracing::debug!("note hook ingestion skipped: no MR iid or project path"); + return; + }; + + let body = attrs["note"].as_str().unwrap_or(""); + let body_lower = body.to_lowercase(); + let is_command = + note_starts_with_command(&body_lower, "/review") || note_starts_with_command(&body_lower, "/describe"); + if !is_command { + // (a) our own published review report. + if body.starts_with(crate::publisher::REVIEW_REPORT_PREFIX) { + tracing::debug!("note hook ingestion skipped: self-published review report"); + return; + } + // (b) a note authored by the service's own GitLab account. + if let Some(self_id) = resolve_self_user_id(platform, gitlab_token, parsed).await { + if parsed["user"]["id"].as_u64() == Some(self_id) { + tracing::debug!("note hook ingestion skipped: note authored by the service itself"); + return; + } + } + } + + let author = parsed["user"]["username"] + .as_str() + .or_else(|| parsed["user"]["name"].as_str()) + .unwrap_or(""); + let created_at = parse_note_created_at(attrs["created_at"].as_str()).unwrap_or_else(|| { + tracing::warn!("note {note_id}: unparseable created_at, using ingestion time"); + chrono::Utc::now() + }); + let note = DiscussionNote { + platform: platform + .map(|p| p.name.clone()) + .unwrap_or_else(|| "default".to_string()), + project: project.to_string(), + mr_iid, + note_id, + author: author.to_string(), + body: body.to_string(), + created_at, + }; + if let Err(e) = db.upsert_note(¬e).await { + tracing::error!("failed to persist MR discussion note {note_id} for {project} !{mr_iid}: {e:#}"); + } +} + pub async fn handle_note_hook( body: &str, dispatcher: &MrDispatcher, gitlab_token: &str, platform: Option, task_store: Option>, + db: Option>, ) -> Result, StatusCode> { let parsed: Value = serde_json::from_str(body).map_err(|e| { tracing::error!("Failed to parse Note hook: {}", e); StatusCode::BAD_REQUEST })?; + // 0.10.0 (design/persistence.md §7.1): persist the note into + // mr_discussions BEFORE the command check below — command notes are + // discussion history too. Ingestion is best-effort: a failure is logged, + // never fails the hook, and db=None keeps the 0.9 behaviour exactly. + if let Some(db) = &db { + ingest_note(db, &parsed, platform.as_ref(), gitlab_token).await; + } + let note = parsed["object_attributes"]["note"].as_str().unwrap_or(""); let note_lower = note.to_lowercase(); @@ -697,4 +869,244 @@ mod tests { assert_eq!(meta.gitlab_mr_url.as_deref(), Some(MR_URL)); assert_eq!(meta.commit_sha.as_deref(), Some(SHA)); } + + // ─── 0.10.0 note ingestion (design/persistence.md §7.1) ─── + + async fn fresh_db() -> Arc { + let db = Arc::new(SqlxStore::new_in_memory().await.unwrap()); + db.migrate().await.unwrap(); + db + } + + fn note_payload(note_id: u64, body: &str, user_id: u64, username: &str) -> String { + serde_json::json!({ + "object_kind": "note", + "object_attributes": { + "id": note_id, + "note": body, + "noteable_type": "MergeRequest", + "created_at": "2026-09-03 10:00:00 UTC", + "url": format!("http://gitlab.internal/group/proj/-/merge_requests/7#note_{note_id}"), + "system": false + }, + "merge_request": {"iid": 7}, + "project": {"path_with_namespace": "group/proj", "web_url": "http://gitlab.internal/group/proj"}, + "user": {"id": user_id, "username": username, "name": username} + }) + .to_string() + } + + async fn note_count(db: &SqlxStore) -> i64 { + sqlx::query_scalar("SELECT COUNT(*) FROM mr_discussions") + .fetch_one(db.pool()) + .await + .unwrap() + } + + /// Fire a note hook whose response is discarded (Json is #[must_use]). + async fn fire_note( + payload: &str, + dispatcher: &MrDispatcher, + token: &str, + platform: Option, + db: &Arc, + ) { + let _ = handle_note_hook(payload, dispatcher, token, platform, None, Some(db.clone())) + .await + .unwrap(); + } + + /// (a) a plain MR note is persisted with all fields, project from + /// `path_with_namespace`, author from `user.username`, platform + /// "default" when no platform matched. + #[tokio::test] + async fn note_ingestion_persists_fields() { + let db = fresh_db().await; + let dispatcher = MrDispatcher::new(); + let resp = handle_note_hook( + ¬e_payload(1234, "LGTM, ship it", 42, "alice"), + &dispatcher, + "", + None, + None, + Some(db.clone()), + ) + .await + .expect("hook must succeed"); + assert_eq!(resp["status"], "received"); + + let notes = db.list_notes("default", "group/proj", 7).await.unwrap(); + assert_eq!(notes.len(), 1); + let note = ¬es[0]; + assert_eq!(note.note_id, 1234); + assert_eq!(note.body, "LGTM, ship it"); + assert_eq!(note.author, "alice"); + assert_eq!(note.platform, "default"); + assert_eq!(note.project, "group/proj"); + assert_eq!(note.mr_iid, 7); + assert_eq!( + note.created_at, + chrono::DateTime::parse_from_rfc3339("2026-09-03T10:00:00Z") + .unwrap() + .with_timezone(&chrono::Utc), + "GitLab legacy timestamp format decodes to UTC" + ); + // ingested_at is stamped on insert. + let ingested: String = sqlx::query_scalar("SELECT ingested_at FROM mr_discussions WHERE note_id = 1234") + .fetch_one(db.pool()) + .await + .unwrap(); + crate::store::decode_ts(&ingested).unwrap(); + } + + /// Matched platform supplies the `platform` column; the MR iid falls + /// back to the `object_attributes.url` tail when merge_request is absent. + #[tokio::test] + async fn note_ingestion_platform_name_and_iid_fallback() { + let db = fresh_db().await; + let dispatcher = MrDispatcher::new(); + let platform = crate::models::GitPlatformConfig { + name: "internal".to_string(), + base_url: "http://gitlab.internal".to_string(), + ..Default::default() + }; + let mut payload: Value = serde_json::from_str(¬e_payload(55, "looks fine", 42, "bob")).unwrap(); + payload.as_object_mut().unwrap().remove("merge_request"); + fire_note(&payload.to_string(), &dispatcher, "", Some(platform), &db).await; + let notes = db.list_notes("internal", "group/proj", 7).await.unwrap(); + assert_eq!(notes.len(), 1, "iid recovered from the note URL tail"); + assert_eq!(notes[0].platform, "internal"); + } + + /// (b) webhook redelivery dedups on the primary key; (c) an edited note + /// (same note_id) updates the body in place. + #[tokio::test] + async fn note_ingestion_idempotent_redelivery_and_edit() { + let db = fresh_db().await; + let dispatcher = MrDispatcher::new(); + + let body = note_payload(1234, "first version", 42, "alice"); + fire_note(&body, &dispatcher, "", None, &db).await; + fire_note(&body, &dispatcher, "", None, &db).await; + assert_eq!(note_count(&db).await, 1, "redelivery must dedup"); + + let edited = note_payload(1234, "edited body", 42, "alice"); + fire_note(&edited, &dispatcher, "", None, &db).await; + assert_eq!(note_count(&db).await, 1, "edit must update in place"); + let notes = db.list_notes("default", "group/proj", 7).await.unwrap(); + assert_eq!(notes[0].body, "edited body"); + } + + /// (d) non-MR notes (Commit/Issue/Snippet) are ignored. + #[tokio::test] + async fn note_ingestion_ignores_non_mr_notes() { + let db = fresh_db().await; + let dispatcher = MrDispatcher::new(); + for noteable in ["Commit", "Issue", "Snippet"] { + let mut payload: Value = serde_json::from_str(¬e_payload(9, "note", 42, "alice")).unwrap(); + payload["object_attributes"]["noteable_type"] = serde_json::json!(noteable); + fire_note(&payload.to_string(), &dispatcher, "", None, &db).await; + } + assert_eq!(note_count(&db).await, 0); + } + + /// (e) system notes ("added 1 commit") are noise and skipped. + #[tokio::test] + async fn note_ingestion_skips_system_notes() { + let db = fresh_db().await; + let dispatcher = MrDispatcher::new(); + let mut payload: Value = serde_json::from_str(¬e_payload(9, "added 1 commit", 42, "alice")).unwrap(); + payload["object_attributes"]["system"] = serde_json::json!(true); + fire_note(&payload.to_string(), &dispatcher, "", None, &db).await; + assert_eq!(note_count(&db).await, 0); + } + + /// (f) self-echo guard: our published report prefix and our own author + /// id are skipped; a /review command note is user intent and ingests + /// even when it hits both guards. + #[tokio::test] + async fn note_ingestion_self_echo_guard_and_command_exception() { + let db = fresh_db().await; + let dispatcher = MrDispatcher::new(); + + // (a) report prefix. + let report = format!("{}\nreview body", crate::publisher::REVIEW_REPORT_PREFIX); + fire_note(¬e_payload(1, &report, 42, "review-bot"), &dispatcher, "", None, &db).await; + assert_eq!(note_count(&db).await, 0, "our own report must not be ingested"); + + // (b) self-author. Seed the per-platform cache directly (unique + // platform name — no network, no cross-test interference). + let platform = crate::models::GitPlatformConfig { + name: "self-echo-test".to_string(), + base_url: "http://127.0.0.1:9".to_string(), + ..Default::default() + }; + SELF_USER_IDS + .get_or_init(|| tokio::sync::Mutex::new(std::collections::HashMap::new())) + .lock() + .await + .insert("self-echo-test".to_string(), 4242); + + fire_note( + ¬e_payload(2, "inline comment by the bot", 4242, "review-bot"), + &dispatcher, + "glpat-self-echo-test", + Some(platform.clone()), + &db, + ) + .await; + assert_eq!(note_count(&db).await, 0, "note by our own user id must be skipped"); + + // A DIFFERENT user on the same platform ingests fine (cache hit, no + // network). + fire_note( + ¬e_payload(3, "human comment", 777, "carol"), + &dispatcher, + "glpat-self-echo-test", + Some(platform.clone()), + &db, + ) + .await; + assert_eq!(note_count(&db).await, 1); + + // Command exception: a /review note from our own account is still + // user intent → ingested. Empty token keeps the command branch from + // dispatching (the guard exception bypasses self-id resolution + // entirely, so the token value is irrelevant to this assertion). + fire_note( + ¬e_payload(4, "/review", 4242, "review-bot"), + &dispatcher, + "", + Some(platform), + &db, + ) + .await; + let notes = db.list_notes("self-echo-test", "group/proj", 7).await.unwrap(); + assert_eq!(notes.len(), 2, "human comment + command note"); + assert!( + notes.iter().any(|n| n.body == "/review"), + "command note must be ingested" + ); + } + + /// (g) db=None: the hook behaves exactly like 0.9 — no ingestion, no + /// error. + #[tokio::test] + async fn note_hook_without_db_is_0_9_behaviour() { + let dispatcher = MrDispatcher::new(); + let resp = handle_note_hook(¬e_payload(1, "LGTM", 42, "alice"), &dispatcher, "", None, None, None) + .await + .expect("hook must succeed without a DB"); + assert_eq!(resp["status"], "received"); + } + + #[test] + fn parse_note_created_at_accepts_gitlab_formats() { + let legacy = parse_note_created_at(Some("2026-09-03 10:00:00 UTC")).unwrap(); + assert_eq!(legacy.to_rfc3339(), "2026-09-03T10:00:00+00:00"); + let rfc = parse_note_created_at(Some("2026-09-03T10:00:00Z")).unwrap(); + assert_eq!(legacy, rfc); + assert!(parse_note_created_at(Some("not a date")).is_none()); + assert!(parse_note_created_at(None).is_none()); + } } diff --git a/src/store/rows.rs b/src/store/rows.rs index e84f7c4..ba73e7d 100644 --- a/src/store/rows.rs +++ b/src/store/rows.rs @@ -376,3 +376,37 @@ pub(crate) fn expert_report_rows(task_id: &Uuid, result: &Value, created_at: Str }) .collect() } + +// ─── Discussion domain (step 6a): mr_discussions ⇄ DiscussionNote ─── + +use crate::store::traits::DiscussionNote; + +/// Raw decode target for `SELECT platform, project, mr_iid, note_id, author, +/// body, created_at FROM mr_discussions`, in column order. +pub(crate) type DiscussionRowTuple = (String, String, i64, i64, String, String, String); + +fn u64_from_i64(value: i64, what: &str) -> Result { + u64::try_from(value).with_context(|| format!("mr_discussions.{what} out of range: {value}")) +} + +/// `DiscussionNote.mr_iid` / `note_id` as bindable i64 (BIGINT columns). +pub(crate) fn discussion_ids(note: &DiscussionNote) -> Result<(i64, i64)> { + Ok(( + i64::try_from(note.mr_iid).with_context(|| format!("mr_iid out of range: {}", note.mr_iid))?, + i64::try_from(note.note_id).with_context(|| format!("note_id out of range: {}", note.note_id))?, + )) +} + +pub(crate) fn discussion_from_row( + (platform, project, mr_iid, note_id, author, body, created_at): DiscussionRowTuple, +) -> Result { + Ok(DiscussionNote { + platform, + project, + mr_iid: u64_from_i64(mr_iid, "mr_iid")?, + note_id: u64_from_i64(note_id, "note_id")?, + author, + body, + created_at: decode_ts(&created_at).context("mr_discussions.created_at")?, + }) +} diff --git a/src/store/sqlx.rs b/src/store/sqlx.rs index 1aa4348..18fee9e 100644 --- a/src/store/sqlx.rs +++ b/src/store/sqlx.rs @@ -16,7 +16,7 @@ use crate::server::api::config::persist::{PersistedGitlabConfig, UiStateFile}; use crate::server::task_queue::{SourceMeta, TaskEntry}; use super::rows; -use super::traits::{ConfigStore, ReviewListQuery, ReviewStore}; +use super::traits::{ConfigStore, DiscussionNote, DiscussionStore, ReviewListQuery, ReviewStore}; use super::{encode_ts, SqlxStore}; const LEGACY_GITLAB_KEY: &str = "gitlab"; @@ -489,6 +489,56 @@ impl ReviewStore for SqlxStore { } } +// ─── DiscussionStore (mr_discussions, step 6a) ─── + +#[async_trait] +impl DiscussionStore for SqlxStore { + async fn upsert_note(&self, note: &DiscussionNote) -> Result<()> { + let (mr_iid, note_id) = rows::discussion_ids(note)?; + ::sqlx::query( + "INSERT INTO mr_discussions (platform, project, mr_iid, note_id, author, body, created_at, ingested_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?) \ + ON CONFLICT (platform, project, mr_iid, note_id) DO UPDATE SET \ + body = excluded.body, author = excluded.author", + ) + .bind(¬e.platform) + .bind(¬e.project) + .bind(mr_iid) + .bind(note_id) + .bind(¬e.author) + .bind(¬e.body) + .bind(encode_ts(¬e.created_at)) + .bind(encode_ts(&Utc::now())) + .execute(self.pool()) + .await + .with_context(|| { + format!( + "upsert mr_discussion note {} for {} !{}", + note.note_id, note.project, note.mr_iid + ) + })?; + Ok(()) + } + + async fn list_notes(&self, platform: &str, project: &str, mr_iid: u64) -> Result> { + let mr_iid = i64::try_from(mr_iid).with_context(|| format!("mr_iid out of range: {mr_iid}"))?; + let rows = ::sqlx::query_as::<_, rows::DiscussionRowTuple>( + "SELECT platform, project, mr_iid, note_id, author, body, created_at FROM mr_discussions \ + WHERE platform = ? AND project = ? AND mr_iid = ? ORDER BY created_at, note_id", + ) + .bind(platform) + .bind(project) + .bind(mr_iid) + .fetch_all(self.pool()) + .await + .with_context(|| format!("list mr_discussions for {project} !{mr_iid}"))?; + rows.into_iter() + .map(rows::discussion_from_row) + .collect::>>() + .context("decode mr_discussions rows") + } +} + /// Shared WHERE clause + positional binds for the history list (§8.1). All /// bind values are Strings so the COUNT and the page SELECT can share them. /// @@ -884,4 +934,61 @@ mod tests { assert_eq!(decoded.created_at, entry.created_at); assert!(decoded.expert_name.is_none(), "live-only field is not persisted"); } + + // ─── DiscussionStore (step 6a) ─── + + fn note(note_id: u64, created_at: &str, body: &str) -> DiscussionNote { + DiscussionNote { + platform: "default".into(), + project: "group/proj".into(), + mr_iid: 7, + note_id, + author: "alice".into(), + body: body.into(), + created_at: DateTime::parse_from_rfc3339(created_at).unwrap().with_timezone(&Utc), + } + } + + /// list_notes orders by (created_at, note_id) ascending — the + /// append-only order §7.2's context renderer relies on. + #[tokio::test] + async fn discussion_notes_round_trip_and_ordering() { + let store = fresh_store().await; + // Insert out of order, including two notes sharing a timestamp + // (note_id breaks the tie). + store + .upsert_note(¬e(3, "2026-09-03T10:00:02Z", "third")) + .await + .unwrap(); + store + .upsert_note(¬e(1, "2026-09-03T10:00:00Z", "first")) + .await + .unwrap(); + store + .upsert_note(¬e(5, "2026-09-03T10:00:02Z", "fourth")) + .await + .unwrap(); + store + .upsert_note(¬e(2, "2026-09-03T10:00:00Z", "second")) + .await + .unwrap(); + // A different platform / MR must not leak into the result. + let mut other = note(9, "2026-08-01T00:00:00Z", "other"); + other.platform = "public".into(); + store.upsert_note(&other).await.unwrap(); + + let notes = store.list_notes("default", "group/proj", 7).await.unwrap(); + let bodies: Vec<&str> = notes.iter().map(|n| n.body.as_str()).collect(); + assert_eq!(bodies, vec!["first", "second", "third", "fourth"]); + assert_eq!(notes[0].created_at.to_rfc3339(), "2026-09-03T10:00:00+00:00"); + + // Edit via upsert keeps position, updates body. + let mut edited = note(2, "2026-09-03T10:00:00Z", "second (edited)"); + edited.author = "bob".into(); + store.upsert_note(&edited).await.unwrap(); + let notes = store.list_notes("default", "group/proj", 7).await.unwrap(); + assert_eq!(notes.len(), 4); + assert_eq!(notes[1].body, "second (edited)"); + assert_eq!(notes[1].author, "bob"); + } } diff --git a/src/store/traits.rs b/src/store/traits.rs index fa4b3cb..7e6f01c 100644 --- a/src/store/traits.rs +++ b/src/store/traits.rs @@ -139,3 +139,37 @@ pub struct ReviewListQuery { pub date_from: Option>, pub date_to: Option>, } + +/// One MR discussion note (`mr_discussions` row, design/persistence.md +/// §3.2). The primary key `(platform, project, mr_iid, note_id)` is the +/// idempotency key: webhook redelivery dedups, note edits update in place. +#[derive(Debug, Clone, PartialEq)] +pub struct DiscussionNote { + /// `GitPlatformConfig.name` of the instance the note came from + /// ("default" when no platform matched the payload). + pub platform: String, + /// `project.path_with_namespace`. + pub project: String, + pub mr_iid: u64, + pub note_id: u64, + pub author: String, + pub body: String, + /// The note's own creation time (from the webhook payload), NOT the + /// ingestion time. + pub created_at: DateTime, +} + +/// Persistence boundary for MR discussion notes (design/persistence.md +/// §7.1). Written by the Note webhook handler; read by the review-time +/// context injection (§7.2, step 6b). +#[async_trait] +pub trait DiscussionStore: Send + Sync { + /// Idempotent upsert on `(platform, project, mr_iid, note_id)`: + /// redelivery dedups; an edited note updates `body` / `author`. + async fn upsert_note(&self, note: &DiscussionNote) -> Result<()>; + + /// All notes of one MR, ordered `(created_at, note_id)` ascending — the + /// append-only order the context-injection renderer (§7.2) relies on for + /// prefix-stable output. + async fn list_notes(&self, platform: &str, project: &str, mr_iid: u64) -> Result>; +} From 4a7ba796c7a2d18770da5e32b1bf07a87276e321 Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 12:57:31 +0800 Subject: [PATCH 09/36] =?UTF-8?q?feat(review):=20inject=20MR=20discussion?= =?UTF-8?q?=20history=20into=20pre-review=20prompts=20(0.10.0=20=C2=A77.2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DB-first (mr_discussions from §7.1 webhook ingestion) with a GitLab discussions-API fallback that back-fills the DB; the rendered section is prefix-stable (ordered by created_at/note_id, fixed 2000-char body cap), sha256-recorded into review_contexts, capped at 128 KiB, and attached to MRInfo.discussion_context between the fixed MR context and the diff. Every failure path (no DB, API down, empty, oversized) degrades to the 0.9 prompt; self-echo guards (report prefix, own user id, system notes) match webhook ingestion while /review /describe command notes are kept. --- src/git_provider/github/client.rs | 1 + src/git_provider/gitlab/client.rs | 16 + src/models/mod.rs | 7 + src/prompt/engine.rs | 32 ++ src/prompt/templates.rs | 4 + src/server/api/review/discussion.rs | 467 ++++++++++++++++++++++++++++ src/server/api/review/mod.rs | 1 + src/server/api/review/task.rs | 35 ++- src/server/gitlab/handler.rs | 16 +- src/server/gitlab/hooks.rs | 137 +++++--- src/server/gitlab/mod.rs | 5 + src/server/task_queue.rs | 3 + src/store/sqlx.rs | 27 ++ src/store/traits.rs | 15 + 14 files changed, 719 insertions(+), 47 deletions(-) create mode 100644 src/server/api/review/discussion.rs diff --git a/src/git_provider/github/client.rs b/src/git_provider/github/client.rs index 9294326..0fdf31e 100644 --- a/src/git_provider/github/client.rs +++ b/src/git_provider/github/client.rs @@ -145,6 +145,7 @@ impl Client { merge_commit_sha: pr.merge_commit_sha, pr_author: Some(pr.user.login), pr_author_id: Some(pr.user.id), + discussion_context: None, }) } diff --git a/src/git_provider/gitlab/client.rs b/src/git_provider/gitlab/client.rs index a6ec263..7b129ca 100644 --- a/src/git_provider/gitlab/client.rs +++ b/src/git_provider/gitlab/client.rs @@ -309,6 +309,7 @@ impl Client { merge_commit_sha: None, pr_author, pr_author_id, + discussion_context: None, }) } @@ -714,16 +715,31 @@ pub struct Discussion { } /// A single note within a discussion. +/// +/// `system` / `created_at` / author `username` are consumed by the 0.10.0 +/// review-time discussion-context tap (§7.2); all default when absent so +/// older fixtures and partial payloads still parse. #[derive(Debug, Clone, serde::Deserialize)] pub struct DiscussionNote { pub id: i64, pub body: String, pub author: NoteAuthor, + /// GitLab system notes ("added 1 commit") — noise, filtered out (§7.1). + #[serde(default)] + pub system: bool, + /// RFC 3339 (or GitLab legacy) creation timestamp; parsed by the caller + /// (`parse_note_created_at`), which falls back to now() when absent. + #[serde(default)] + pub created_at: String, } #[derive(Debug, Clone, serde::Deserialize)] pub struct NoteAuthor { pub id: u64, + #[serde(default)] + pub username: String, + #[serde(default)] + pub name: String, } fn encode_project_path(path: &str) -> String { diff --git a/src/models/mod.rs b/src/models/mod.rs index c40a05e..ed09663 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -221,6 +221,12 @@ pub struct MRInfo { pub pr_author: Option, /// Author's platform-specific unique ID (GitHub user.id / GitLab user.id). pub pr_author_id: Option, + /// Rendered MR discussion-history section injected into review prompts + /// (0.10.0 §7.2). Filled by the pre-review discussion tap (DB-first, + /// GitLab API fallback); `None` = no context injected (0.9 behaviour). + /// Runtime-only: never serialized into task records or the DB. + #[serde(skip, default)] + pub discussion_context: Option, } impl MRInfo { @@ -239,6 +245,7 @@ impl MRInfo { merge_commit_sha: None, pr_author: None, pr_author_id: None, + discussion_context: None, } } } diff --git a/src/prompt/engine.rs b/src/prompt/engine.rs index 87fba17..4d96b5a 100644 --- a/src/prompt/engine.rs +++ b/src/prompt/engine.rs @@ -99,6 +99,10 @@ impl PromptEngine { "constraints": constraints, "lead_context": lead_section, "file_contents": file_contents, + // 0.10.0 §7.2: pre-review MR discussion history, injected between + // the fixed MR/project context and the diff. None → the `{% if %}` + // block collapses and the prompt is byte-identical to 0.9. + "discussion_context": mr.discussion_context.as_deref(), }); let user = self.env.get_template("review_user")?.render(&ctx_user)?; @@ -384,6 +388,34 @@ mod tests { assert!(system.contains("Downgrade code-quality or style findings")); } + /// 0.10.0 §7.2: a populated `discussion_context` renders between the + /// fixed MR/project context and the diff ("Code Changes"), and `None` + /// collapses the block entirely (0.9-identical prompt). + #[test] + fn test_review_prompt_discussion_context_placement() { + let engine = PromptEngine::new(); + let expert = make_test_expert("You are a security expert."); + let settings = make_test_app_config(None); + + let mut mr = make_test_mr(); + mr.discussion_context = Some("## MR Discussion History\n\n- [alice @ 2026-09-03]: lgtm\n".to_string()); + let (_system, user) = engine + .build_review_prompt(&expert, &mr, "diff", "zh", &settings, None, None) + .unwrap(); + + assert!(user.contains("## MR Discussion History"), "section must render"); + let pos_section = user.find("## MR Discussion History").unwrap(); + let pos_diff = user.find("## Code Changes").unwrap(); + assert!(pos_section < pos_diff, "discussion history must precede the diff"); + + // None → no section, and the prompt matches the 0.9 shape. + let mr = make_test_mr(); + let (_system, user_none) = engine + .build_review_prompt(&expert, &mr, "diff", "zh", &settings, None, None) + .unwrap(); + assert!(!user_none.contains("MR Discussion History")); + } + #[test] fn test_review_prompt_without_project_context() { let engine = PromptEngine::new(); diff --git a/src/prompt/templates.rs b/src/prompt/templates.rs index 5c53bae..f668dd0 100644 --- a/src/prompt/templates.rs +++ b/src/prompt/templates.rs @@ -179,6 +179,10 @@ Description: {{ description }} {% endif %} {% endif %} +{% if discussion_context %} +{{ discussion_context }} +{% endif %} + Note: In the diff below: - Lines starting with '+' are NEW code added by this PR — focus on these. - Lines starting with '-' are DELETED code. diff --git a/src/server/api/review/discussion.rs b/src/server/api/review/discussion.rs new file mode 100644 index 0000000..62bb989 --- /dev/null +++ b/src/server/api/review/discussion.rs @@ -0,0 +1,467 @@ +//! Pre-review MR discussion-context injection (design/persistence.md §7.2). +//! +//! Before the expert run starts, the review task loads the MR's discussion +//! history — DB-first (`mr_discussions`, fed by the Note webhook ingestion of +//! §7.1), falling back to the GitLab discussions API when the DB has nothing +//! (fresh instance, webhook not wired) and back-filling the DB from that +//! fetch. The notes are rendered into a fixed, prefix-stable markdown section +//! that is attached to `MRInfo::discussion_context` and injected into the +//! review user template between the fixed MR/project context and the diff. +//! +//! Degradation contract: EVERY failure path (DB error, API error, empty +//! result, oversized render, missing task row) logs and yields `None` — the +//! review then runs exactly as in 0.9. Injection must never fail a review. + +use std::sync::Arc; + +use anyhow::Result; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::server::gitlab::{ + is_command_note, is_self_report, parse_note_created_at, review_base_url, self_user_id_cached, url_origin, +}; +use crate::store::traits::{DiscussionNote, DiscussionStore, ReviewStore}; +use crate::store::SqlxStore; + +/// `review_contexts.kind` value for the discussion-history section. +pub(crate) const DISCUSSION_KIND: &str = "mr_discussions"; + +/// Hard cap on the rendered section. Beyond this the prompt would drown the +/// diff; skip injection entirely (the review keeps full diff fidelity). +pub(crate) const MAX_CONTEXT_BYTES: usize = 128 * 1024; + +/// Per-note body cap (chars, not bytes — note text is user-facing unicode). +const MAX_NOTE_BODY_CHARS: usize = 2000; + +const SECTION_HEADER: &str = "## MR Discussion History\n\n"; + +/// Plumbing for one review task's discussion tap: the DB handle plus the +/// platform identity (`DiscussionNote.platform`) and the instance root used +/// for the API fallback and the self-echo guard. Cheap to clone; threaded +/// through the webhook dispatch chain as `Option` (`None` = +/// no DB → 0.9 behaviour). `pub` only because the webhook dispatch fns are +/// `pub`; construction and use stay crate-internal. +#[derive(Clone)] +pub struct DiscussionTap { + db: Arc, + platform: String, + instance_base: String, +} + +impl DiscussionTap { + /// Build a tap for `payload_url` (the MR URL as the review will fetch it). + /// `platform`, when matched, fixes both the `platform` key (its `name`) + /// and the reachable instance base (`internal_base_url` preferred); + /// otherwise the key is `"default"` and the base is the URL's origin. + pub(crate) fn new( + db: Arc, + platform: Option<&crate::models::GitPlatformConfig>, + payload_url: &str, + ) -> Self { + let (platform, instance_base) = match platform { + Some(p) => (p.name.clone(), review_base_url(p).to_string()), + None => ("default".to_string(), url_origin(payload_url).unwrap_or_default()), + }; + Self { + db, + platform, + instance_base, + } + } + + /// Load, render, and persist the discussion section for one review task. + /// `Some(section)` is attached to `MRInfo::discussion_context`; `None` = + /// degrade to the 0.9 prompt. `task_id` must be a live `reviews` row + /// (FK target of `review_contexts`); callers without a task store skip + /// the tap entirely. + pub(crate) async fn inject( + &self, + task_id: Uuid, + project: &str, + mr_iid: u64, + gitlab_token: &str, + mr_url: &str, + ) -> Option { + let instance_base = self.instance_base.clone(); + let platform = self.platform.clone(); + let token = gitlab_token.to_string(); + let url = mr_url.to_string(); + let notes = load_discussion_notes(&self.db, &self.platform, project, mr_iid, move || async move { + fetch_notes_via_api(&platform, &instance_base, project, mr_iid, &token, &url).await + }) + .await?; + if notes.is_empty() { + return None; + } + let section = render_discussion_context(¬es); + if section.len() > MAX_CONTEXT_BYTES { + tracing::warn!( + task_id = %task_id, + bytes = section.len(), + "MR discussion context exceeds {} bytes; skipping injection", + MAX_CONTEXT_BYTES + ); + return None; + } + // Persist the rendered context (content-addressed by sha256) so a + // re-review of the same MR can detect reuse. Best-effort: the prompt + // injection above must not depend on this write succeeding. + let content_hash = sha256_hex(§ion); + let token_estimate = (section.len() / 4) as i64; + if let Err(e) = self + .db + .upsert_review_context(task_id, DISCUSSION_KIND, §ion, &content_hash, token_estimate) + .await + { + tracing::warn!(task_id = %task_id, "failed to persist review_context {DISCUSSION_KIND}: {e:#}"); + } + Some(section) + } +} + +/// Render notes (already ordered `(created_at, note_id)` ascending — the +/// `list_notes` contract) into the prompt section. Deterministic: identical +/// input yields byte-identical output, which is what makes `content_hash` +/// reuse detection meaningful. Bodies are truncated at +/// [`MAX_NOTE_BODY_CHARS`] chars. +pub(crate) fn render_discussion_context(notes: &[DiscussionNote]) -> String { + let mut out = String::from(SECTION_HEADER); + for note in notes { + let body: String = note.body.chars().take(MAX_NOTE_BODY_CHARS).collect(); + out.push_str(&format!( + "- [{} @ {}]: {}\n", + note.author, + crate::store::encode_ts(¬e.created_at), + body + )); + } + out +} + +pub(crate) fn sha256_hex(content: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(content.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +/// DB-first note load with an API fallback (§7.2): when `mr_discussions` has +/// rows for this MR they win (webhook-ingested history is authoritative and +/// free); when empty, `fallback` fetches from the provider API and every +/// fetched note is upserted (per-note failure is logged, not fatal) so the +/// next review is DB-served. `None` on any failure — never an error. +pub(crate) async fn load_discussion_notes( + db: &SqlxStore, + platform: &str, + project: &str, + mr_iid: u64, + fallback: F, +) -> Option> +where + F: FnOnce() -> Fut, + Fut: std::future::Future>>, +{ + let stored = match db.list_notes(platform, project, mr_iid).await { + Ok(notes) => notes, + Err(e) => { + tracing::warn!("discussion tap: list_notes({platform}/{project} !{mr_iid}) failed: {e:#}"); + return None; + } + }; + if !stored.is_empty() { + return Some(stored); + } + match fallback().await { + Ok(mut notes) => { + for note in ¬es { + if let Err(e) = db.upsert_note(note).await { + tracing::error!( + "discussion tap: failed to back-fill note {} for {platform}/{project} !{mr_iid}: {e:#}", + note.note_id + ); + } + } + // The API returns discussions in its own order; enforce the same + // (created_at, note_id) order `list_notes` guarantees. + notes.sort_by_key(|n| (n.created_at, n.note_id)); + Some(notes) + } + Err(e) => { + tracing::warn!("discussion tap: API fallback for {platform}/{project} !{mr_iid} failed: {e:#}"); + None + } + } +} + +/// GitLab discussions API fallback: fetch all discussion notes on the MR, +/// dropping system notes and our own output — the same self-echo guards as +/// webhook ingestion (§7.1 (a) report prefix, (b) own user id) — but KEEPING +/// `/review` / `/describe` command notes (user intent, part of the history). +async fn fetch_notes_via_api( + platform: &str, + instance_base: &str, + project: &str, + mr_iid: u64, + gitlab_token: &str, + mr_url: &str, +) -> Result> { + if instance_base.is_empty() { + anyhow::bail!("no reachable GitLab instance base for the API fallback"); + } + let client = crate::git_provider::gitlab::client::Client::new(gitlab_token, mr_url)?; + let discussions = client.list_discussions().await?; + let self_id = self_user_id_cached(platform, gitlab_token, instance_base).await; + + let mut notes = Vec::new(); + for note in discussions.into_iter().flat_map(|d| d.notes) { + if note.system { + continue; + } + if !is_command_note(¬e.body.to_lowercase()) { + if is_self_report(¬e.body) { + continue; + } + if Some(note.author.id) == self_id { + continue; + } + } + let author = if note.author.username.is_empty() { + if note.author.name.is_empty() { + format!("user#{}", note.author.id) + } else { + note.author.name + } + } else { + note.author.username + }; + let created_at = parse_note_created_at(Some(¬e.created_at)).unwrap_or_else(|| { + tracing::warn!( + "discussion tap: note {} has no parseable created_at, using fetch time", + note.id + ); + chrono::Utc::now() + }); + notes.push(DiscussionNote { + platform: platform.to_string(), + project: project.to_string(), + mr_iid, + note_id: note.id.max(0) as u64, + author, + body: note.body, + created_at, + }); + } + Ok(notes) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + async fn fresh_db() -> SqlxStore { + let db = SqlxStore::new_in_memory().await.unwrap(); + db.migrate().await.unwrap(); + db + } + + fn note(note_id: u64, author: &str, body: &str, secs: i64) -> DiscussionNote { + DiscussionNote { + platform: "default".to_string(), + project: "group/proj".to_string(), + mr_iid: 7, + note_id, + author: author.to_string(), + body: body.to_string(), + created_at: chrono::Utc.timestamp_opt(1_700_000_000 + secs, 0).unwrap(), + } + } + + /// (a) seeded notes render in (created_at, note_id) order, every entry + /// present, oversized bodies truncated at 2000 chars. + #[tokio::test] + async fn render_includes_all_notes_in_order_and_truncates() { + let db = fresh_db().await; + let long_body = "x".repeat(5000); + // Seed out of order; list_notes must restore (created_at, note_id). + db.upsert_note(¬e(3, "bob", "third", 30)).await.unwrap(); + db.upsert_note(¬e(1, "alice", "first", 10)).await.unwrap(); + db.upsert_note(¬e(2, "carol", &long_body, 20)).await.unwrap(); + + let notes = db.list_notes("default", "group/proj", 7).await.unwrap(); + let section = render_discussion_context(¬es); + + assert!(section.starts_with(SECTION_HEADER)); + let pos_first = section.find("[alice @").expect("alice entry"); + let pos_second = section.find("[carol @").expect("carol entry"); + let pos_third = section.find("[bob @").expect("bob entry"); + assert!( + pos_first < pos_second && pos_second < pos_third, + "order must be chronological" + ); + // Truncated at 2000 chars, not 5000. + assert!(section.contains(&"x".repeat(2000))); + assert!(!section.contains(&"x".repeat(2001))); + } + + /// (b) prefix stability: the same input renders byte-identically twice, + /// and the sha256 matches across renders. + #[tokio::test] + async fn render_is_byte_identical_and_hash_stable() { + let db = fresh_db().await; + db.upsert_note(¬e(1, "alice", "first", 10)).await.unwrap(); + db.upsert_note(¬e(2, "bob", "second", 20)).await.unwrap(); + let notes = db.list_notes("default", "group/proj", 7).await.unwrap(); + + let a = render_discussion_context(¬es); + let b = render_discussion_context(¬es); + assert_eq!(a, b, "rendering must be deterministic"); + assert_eq!(sha256_hex(&a), sha256_hex(&b)); + } + + /// (c) empty DB → the fallback supplies notes, each is back-filled + /// (list_notes reads them back) and used for rendering. + #[tokio::test] + async fn empty_db_uses_fallback_and_back_fills() { + let db = fresh_db().await; + let fetched = vec![note(9, "dave", "from api", 5)]; + let fetched_clone = fetched.clone(); + + let notes = load_discussion_notes(&db, "default", "group/proj", 7, move || { + let fetched = fetched_clone.clone(); + async move { Ok(fetched) } + }) + .await + .expect("fallback notes must load"); + + assert_eq!(notes, fetched); + let stored = db.list_notes("default", "group/proj", 7).await.unwrap(); + assert_eq!(stored, fetched, "fallback notes must be back-filled"); + assert!(render_discussion_context(¬es).contains("[dave @")); + } + + /// (c2) a non-empty DB never calls the fallback (webhook-ingested history + /// is authoritative). + #[tokio::test] + async fn non_empty_db_skips_fallback() { + let db = fresh_db().await; + db.upsert_note(¬e(1, "alice", "stored", 10)).await.unwrap(); + + let called = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = called.clone(); + let notes = load_discussion_notes(&db, "default", "group/proj", 7, move || { + let flag = flag.clone(); + async move { + flag.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(Vec::new()) + } + }) + .await + .expect("stored notes must load"); + assert_eq!(notes.len(), 1); + assert_eq!(notes[0].body, "stored"); + assert!( + !called.load(std::sync::atomic::Ordering::SeqCst), + "fallback must not run when the DB has notes" + ); + } + + /// (d) a failing fallback yields None and never panics. + #[tokio::test] + async fn fallback_failure_degrades_to_none() { + let db = fresh_db().await; + let notes = load_discussion_notes(&db, "default", "group/proj", 7, || async { + anyhow::bail!("gitlab down") + }) + .await; + assert!(notes.is_none()); + } + + /// (e) upsert_review_context on the same (task_id, kind) twice: no error, + /// the content is rewritten. + #[tokio::test] + async fn review_context_upsert_rewrites_in_place() { + use crate::server::task_queue::{TaskEntry, TaskState}; + let db = fresh_db().await; + // review_contexts.task_id REFERENCES reviews(task_id) — seed the row. + let entry = TaskEntry { + task_id: Uuid::new_v4(), + state: TaskState::Running, + source_meta: Default::default(), + request: None, + result: None, + error: None, + progress: None, + expert_name: None, + created_at: chrono::Utc::now(), + started_at: None, + completed_at: None, + }; + db.create(&entry).await.unwrap(); + + let hash_a = sha256_hex("content A"); + db.upsert_review_context(entry.task_id, DISCUSSION_KIND, "content A", &hash_a, 3) + .await + .unwrap(); + let hash_b = sha256_hex("content B"); + db.upsert_review_context(entry.task_id, DISCUSSION_KIND, "content B", &hash_b, 3) + .await + .unwrap(); + + let rows: Vec<(String, String)> = + ::sqlx::query_as("SELECT content, content_hash FROM review_contexts WHERE task_id = ? AND kind = ?") + .bind(entry.task_id.to_string()) + .bind(DISCUSSION_KIND) + .fetch_all(db.pool()) + .await + .unwrap(); + assert_eq!(rows.len(), 1, "upsert must not duplicate the (task_id, kind) row"); + assert_eq!(rows[0].0, "content B"); + assert_eq!(rows[0].1, hash_b); + } + + /// (f) oversized render degrades to None (no injection, no context row). + #[tokio::test] + async fn oversized_section_is_skipped() { + let db = Arc::new(fresh_db().await); + let entry = { + use crate::server::task_queue::{TaskEntry, TaskState}; + TaskEntry { + task_id: Uuid::new_v4(), + state: TaskState::Running, + source_meta: Default::default(), + request: None, + result: None, + error: None, + progress: None, + expert_name: None, + created_at: chrono::Utc::now(), + started_at: None, + completed_at: None, + } + }; + db.create(&entry).await.unwrap(); + // Seed one note whose rendered section exceeds the 128 KiB cap via + // many max-length bodies... simpler: seed enough 2000-char notes. + for i in 0..70u64 { + db.upsert_note(¬e(i + 1, "alice", &"y".repeat(2000), i as i64)) + .await + .unwrap(); + } + let tap = DiscussionTap { + db: db.clone(), + platform: "default".to_string(), + instance_base: String::new(), // DB has rows → fallback never runs + }; + let section = tap + .inject(entry.task_id, "group/proj", 7, "token", "http://x/-/merge_requests/7") + .await; + assert!(section.is_none(), "oversized render must be skipped"); + let count: i64 = ::sqlx::query_scalar("SELECT COUNT(*) FROM review_contexts WHERE task_id = ?") + .bind(entry.task_id.to_string()) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(count, 0, "skipped injection must not persist a context row"); + } +} diff --git a/src/server/api/review/mod.rs b/src/server/api/review/mod.rs index 815f37b..4656bd6 100644 --- a/src/server/api/review/mod.rs +++ b/src/server/api/review/mod.rs @@ -2,6 +2,7 @@ //! //! @module review-engine: part of the CodeReview Board virtual engineering team +pub(crate) mod discussion; mod handlers; mod resolve; mod task; diff --git a/src/server/api/review/task.rs b/src/server/api/review/task.rs index 181c352..286310e 100644 --- a/src/server/api/review/task.rs +++ b/src/server/api/review/task.rs @@ -187,6 +187,24 @@ pub(crate) async fn enqueue_review( let webhook = request.webhook; let cfg = state.app_config.read().unwrap().clone(); + // 0.10.0 §7.2: pre-review discussion tap, GitLab MR sources only. Built + // synchronously before the spawn so the git_platforms RwLock guard never + // crosses an .await. `None` when no DB is wired → exact 0.9 behaviour. + let (mr_url, tap) = match &source { + ReviewSource::GitLabMr { url } => { + let tap = state.db.clone().map(|db| { + let platform = { + let platforms = state.git_platforms.read().unwrap(); + crate::models::find_git_platform_for_url_strict(&platforms, url).cloned() + }; + super::discussion::DiscussionTap::new(db, platform.as_ref(), url) + }); + (Some(url.clone()), tap) + } + _ => (None, None), + }; + let token_for_tap = gitlab_token.clone(); + tokio::spawn(async move { while !store_clone.can_start_new_task().await { tokio::time::sleep(std::time::Duration::from_secs(2)).await; @@ -211,11 +229,24 @@ pub(crate) async fn enqueue_review( // later fails. Fill happens before the (possibly long) expert run and // only touches fields still blank, so enqueue-time values win. let outcome = match super::resolve::resolve_source(source, gitlab_token, &cfg).await { - Ok(resolved) => { - if let Some(ref info) = resolved.mr_info { + Ok(mut resolved) => { + if let Some(ref mut info) = resolved.mr_info { store_clone .fill_source_meta(task_id, source_meta_from_mr_info(info)) .await; + // §7.2: inject the MR discussion history into the prompt + // context. Best-effort — any failure degrades to `None` + // and the review runs with the 0.9 prompt. + if let (Some(tap), Some(url), Some(token)) = + (tap.as_ref(), mr_url.as_deref(), token_for_tap.as_deref()) + { + if let Some(section) = tap + .inject(task_id, &info.project_path, u64::from(info.mr_iid), token, url) + .await + { + info.discussion_context = Some(section); + } + } } super::resolve::run_review(resolved, config_toml, llm_configs).await } diff --git a/src/server/gitlab/handler.rs b/src/server/gitlab/handler.rs index 7c918e7..d66fdc6 100644 --- a/src/server/gitlab/handler.rs +++ b/src/server/gitlab/handler.rs @@ -374,9 +374,11 @@ impl GitLabWebhookHandler { ) -> Result, (StatusCode, Json)> { let event_name = system_hook_event_name(body); match event_name.as_str() { - "merge_request" => super::handle_mr_hook(body, &self.dispatcher, token, platform, task_store.clone()) - .await - .map_err(|status| (status, Json(serde_json::json!({"error": "request failed"})))), + "merge_request" => { + super::handle_mr_hook(body, &self.dispatcher, token, platform, task_store.clone(), db.clone()) + .await + .map_err(|status| (status, Json(serde_json::json!({"error": "request failed"})))) + } "note" => super::handle_note_hook(body, &self.dispatcher, token, platform, task_store.clone(), db) .await .map_err(|status| (status, Json(serde_json::json!({"error": "request failed"})))), @@ -466,9 +468,11 @@ impl WebhookHandler for GitLabWebhookHandler { let db = app_state.and_then(|s| s.db.clone()); match event { - "Merge Request Hook" => super::handle_mr_hook(body, &self.dispatcher, &token, platform, task_store.clone()) - .await - .map_err(|status| (status, Json(serde_json::json!({"error": "request failed"})))), + "Merge Request Hook" => { + super::handle_mr_hook(body, &self.dispatcher, &token, platform, task_store.clone(), db.clone()) + .await + .map_err(|status| (status, Json(serde_json::json!({"error": "request failed"})))) + } "Note Hook" => { super::handle_note_hook(body, &self.dispatcher, &token, platform, task_store.clone(), db.clone()) .await diff --git a/src/server/gitlab/hooks.rs b/src/server/gitlab/hooks.rs index 0dfc140..c5ae7f3 100644 --- a/src/server/gitlab/hooks.rs +++ b/src/server/gitlab/hooks.rs @@ -3,6 +3,7 @@ use serde_json::Value; use std::sync::Arc; use super::super::dispatcher::MrDispatcher; +use crate::server::api::review::discussion::DiscussionTap; use crate::server::task_queue::{record_task_outcome, record_task_started, SourceMeta, TaskStore}; use crate::store::traits::{DiscussionNote, DiscussionStore}; use crate::store::SqlxStore; @@ -107,6 +108,7 @@ async fn run_webhook_review( gitlab_token: String, mr_iid: u64, source_meta: SourceMeta, + tap: Option, ) { let task_id = if let Some(store) = task_store.as_ref() { Some(record_task_started(store, source_meta).await) @@ -115,11 +117,22 @@ async fn run_webhook_review( }; let outcome = async { - let (info, diff) = super::super::resolve_review_source(&mr_url, &gitlab_token).await?; + let (mut info, diff) = super::super::resolve_review_source(&mr_url, &gitlab_token).await?; if let (Some(store), Some(id)) = (task_store.as_ref(), task_id) { store .fill_source_meta(id, crate::server::task_queue::source_meta_from_mr_info(&info)) .await; + // §7.2 discussion-context injection: best-effort, `None` + // degrades to the 0.9 prompt. Requires the live task row + // (`review_contexts.task_id` FK), hence tied to the task store. + if let Some(tap) = tap.as_ref() { + if let Some(section) = tap + .inject(id, &info.project_path, u64::from(info.mr_iid), &gitlab_token, &mr_url) + .await + { + info.discussion_context = Some(section); + } + } } super::super::run_review_common( &mr_url, @@ -154,10 +167,11 @@ pub fn spawn_mr_review_task( mr_iid: u64, task_store: Option>, source_meta: SourceMeta, + tap: Option, ) { let d = dispatcher.clone(); tokio::spawn(async move { - run_webhook_review(task_store, &d, mr_url, sha, gitlab_token, mr_iid, source_meta).await; + run_webhook_review(task_store, &d, mr_url, sha, gitlab_token, mr_iid, source_meta, tap).await; }); } @@ -170,6 +184,7 @@ pub async fn handle_mr_in_progress( mr_iid: u64, task_store: Option>, source_meta: SourceMeta, + tap: Option, ) { tracing::info!("MR !{} review in progress, waiting...", mr_iid); dispatcher.wait(mr_url).await; @@ -184,6 +199,7 @@ pub async fn handle_mr_in_progress( mr_iid, task_store, source_meta, + tap, ); } _ => { @@ -202,6 +218,7 @@ pub async fn dispatch_mr_event( mr_iid: u64, task_store: Option>, source_meta: SourceMeta, + tap: Option, ) { match dispatcher.try_start(mr_url, sha).await { super::super::dispatcher::ShouldStart::Go => { @@ -213,13 +230,24 @@ pub async fn dispatch_mr_event( mr_iid, task_store, source_meta, + tap, ); } super::super::dispatcher::ShouldStart::AlreadyReviewed => { tracing::info!("Skipping MR !{}: already reviewed at SHA {}", mr_iid, sha); } super::super::dispatcher::ShouldStart::InProgress => { - handle_mr_in_progress(dispatcher, mr_url, sha, gitlab_token, mr_iid, task_store, source_meta).await; + handle_mr_in_progress( + dispatcher, + mr_url, + sha, + gitlab_token, + mr_iid, + task_store, + source_meta, + tap, + ) + .await; } } } @@ -291,7 +319,7 @@ pub(crate) fn rewrite_url_to_platform(url: &str, base_url: &str) -> String { /// while the payload carries the external :8443), else `base_url`. The /// fail-safe in [`rewrite_url_to_platform`] keeps the payload URL verbatim /// when the chosen target does not parse. -fn review_base_url(platform: &crate::models::GitPlatformConfig) -> &str { +pub(crate) fn review_base_url(platform: &crate::models::GitPlatformConfig) -> &str { if platform.internal_base_url.is_empty() { &platform.base_url } else { @@ -305,6 +333,7 @@ pub async fn handle_mr_hook( gitlab_token: &str, platform: Option, task_store: Option>, + db: Option>, ) -> Result, StatusCode> { let payload = parse_mr_hook_payload(body, gitlab_token)?; @@ -358,6 +387,12 @@ pub async fn handle_mr_hook( let source_meta = source_meta_from_payload(&payload); + // §7.2 discussion tap: the DB handle plus platform identity, built + // against the (rewritten) review URL so the API fallback and the + // self-echo guard target the reachable instance. `None` without a + // DB → 0.9 behaviour. + let tap = db.map(|db| DiscussionTap::new(db, platform.as_ref(), &review_url)); + dispatch_mr_event( dispatcher, &review_url, @@ -366,6 +401,7 @@ pub async fn handle_mr_hook( payload.mr_iid, task_store, source_meta, + tap, ) .await; } @@ -378,7 +414,7 @@ pub async fn handle_mr_hook( /// True when `note` (already lowercased) begins with a slash command whose /// first path segment is exactly `cmd` — i.e. `/review` and `/review/123` -/// match, but `/reviewer` / `/reviewxyz` do not. The command must be followed +/// match, but `/reviewer` / `reviewxyz` do not. The command must be followed /// by a path separator (`/`) or the end of the note, so prefix lookalikes /// never trigger a review (`^/review(/|$)` semantics). pub fn note_starts_with_command(note: &str, cmd: &str) -> bool { @@ -388,6 +424,19 @@ pub fn note_starts_with_command(note: &str, cmd: &str) -> bool { rest.is_empty() || rest.starts_with('/') } +/// True when a note body (already lowercased) is a `/review` or `/describe` +/// command. Command notes are user intent: they always ingest (§7.1) and +/// survive the self-echo filter of the discussion-context tap (§7.2). +pub(crate) fn is_command_note(body_lower: &str) -> bool { + note_starts_with_command(body_lower, "/review") || note_starts_with_command(body_lower, "/describe") +} + +/// True when `body` is one of our own published review reports (self-echo +/// guard (a) of §7.1). +pub(crate) fn is_self_report(body: &str) -> bool { + body.starts_with(crate::publisher::REVIEW_REPORT_PREFIX) +} + /// Extract the merge request iid from the tail of a system-hook note/MR URL /// like `https://gitlab.example.com/group/proj/-/merge_requests/123`. Matches /// the LAST `/-/merge_requests/` marker and parses the leading digit run that @@ -412,7 +461,7 @@ pub(crate) fn mr_iid_from_url(url: &str) -> Option { /// `object_attributes.created_at` from a note webhook. GitLab sends either /// RFC 3339 or its legacy `"2026-09-03 10:00:00 UTC"` format; both decode to /// UTC. `None` when absent/unparseable (the caller falls back to now()). -fn parse_note_created_at(raw: Option<&str>) -> Option> { +pub(crate) fn parse_note_created_at(raw: Option<&str>) -> Option> { let raw = raw?; if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(raw) { return Some(dt.with_timezone(&chrono::Utc)); @@ -424,7 +473,7 @@ fn parse_note_created_at(raw: Option<&str>) -> Option Option { +pub(crate) fn url_origin(url: &str) -> Option { let (scheme, rest) = url.split_once("://")?; let host = rest.split('/').next().unwrap_or(""); if host.is_empty() { @@ -435,15 +484,45 @@ fn url_origin(url: &str) -> Option { } /// Self-echo guard (b) of §7.1: the GitLab user id this service's token -/// posts as, per platform ("default" for the unmatched/runtime-token path). -/// Resolved LAZILY on the first note hook (GET /user) and cached for the -/// process lifetime; only successes are cached, so a transient API failure -/// retries on the next note instead of permanently disabling the guard. -/// The platform set is runtime-mutable (PUT /config), which is why this is -/// not resolved once at startup. +/// posts as, per platform key ("default" for the unmatched/runtime-token +/// path). Resolved LAZILY (GET /user) and cached for the process lifetime; +/// only successes are cached, so a transient API failure retries on the +/// next call instead of permanently disabling the guard. The platform set +/// is runtime-mutable (PUT /config), which is why this is not resolved once +/// at startup. Shared with the §7.2 discussion-context tap (same guard +/// applies to API-fallback notes). `None` when the token or instance base +/// is empty, or the lookup fails — the guard is inactive, never an error. static SELF_USER_IDS: std::sync::OnceLock>> = std::sync::OnceLock::new(); +pub(crate) async fn self_user_id_cached(platform_name: &str, gitlab_token: &str, instance_base: &str) -> Option { + if gitlab_token.is_empty() || instance_base.is_empty() { + return None; + } + let cache = SELF_USER_IDS.get_or_init(|| tokio::sync::Mutex::new(std::collections::HashMap::new())); + if let Some(id) = cache.lock().await.get(platform_name) { + return Some(*id); + } + let client = crate::git_provider::gitlab::client::Client::for_instance(gitlab_token, instance_base); + match client.get_current_user_id().await { + Ok(id) => { + cache.lock().await.insert(platform_name.to_string(), id); + Some(id) + } + Err(e) => { + tracing::warn!( + "could not resolve the service's own GitLab user id ({platform_name}): {e:#}; \ + self-echo guard (b) inactive for this call" + ); + None + } + } +} + +/// Resolve the instance base + platform key for a note webhook payload, then +/// delegate to [`self_user_id_cached`]. The instance to ask: the matched +/// platform's reachable base (internal when configured, mirroring the review +/// URL rewrite), else the origin of the payload's own URLs. async fn resolve_self_user_id( platform: Option<&crate::models::GitPlatformConfig>, gitlab_token: &str, @@ -455,13 +534,6 @@ async fn resolve_self_user_id( let key = platform .map(|p| p.name.clone()) .unwrap_or_else(|| "default".to_string()); - let cache = SELF_USER_IDS.get_or_init(|| tokio::sync::Mutex::new(std::collections::HashMap::new())); - if let Some(id) = cache.lock().await.get(&key) { - return Some(*id); - } - // The instance to ask: the matched platform's reachable base (internal - // when configured, mirroring the review URL rewrite), else the origin of - // the payload's own URLs. let instance_base = match platform { Some(p) => review_base_url(p).to_string(), None => { @@ -472,20 +544,7 @@ async fn resolve_self_user_id( url_origin(url)? } }; - let client = crate::git_provider::gitlab::client::Client::for_instance(gitlab_token, &instance_base); - match client.get_current_user_id().await { - Ok(id) => { - cache.lock().await.insert(key, id); - Some(id) - } - Err(e) => { - tracing::warn!( - "could not resolve the service's own GitLab user id ({key}): {e:#}; \ - self-echo guard (b) inactive for this note" - ); - None - } - } + self_user_id_cached(&key, gitlab_token, &instance_base).await } /// Persist one note-webhook payload into `mr_discussions` (§7.1). @@ -526,12 +585,10 @@ async fn ingest_note( }; let body = attrs["note"].as_str().unwrap_or(""); - let body_lower = body.to_lowercase(); - let is_command = - note_starts_with_command(&body_lower, "/review") || note_starts_with_command(&body_lower, "/describe"); + let is_command = is_command_note(&body.to_lowercase()); if !is_command { // (a) our own published review report. - if body.starts_with(crate::publisher::REVIEW_REPORT_PREFIX) { + if is_self_report(body) { tracing::debug!("note hook ingestion skipped: self-published review report"); return; } @@ -660,8 +717,10 @@ pub async fn handle_note_hook( let u = url; let s = sha; let note_iid = mr_iid; + // §7.2 discussion tap (same wiring as the MR hook). + let tap = db.clone().map(|db| DiscussionTap::new(db, platform.as_ref(), &u)); tokio::spawn(async move { - run_webhook_review(task_store, &d, u, s, token, note_iid, source_meta).await; + run_webhook_review(task_store, &d, u, s, token, note_iid, source_meta, tap).await; }); } _ => { diff --git a/src/server/gitlab/mod.rs b/src/server/gitlab/mod.rs index 5707c36..68037a7 100644 --- a/src/server/gitlab/mod.rs +++ b/src/server/gitlab/mod.rs @@ -19,6 +19,11 @@ pub use hooks::{ dispatch_mr_event, handle_mr_hook, handle_mr_in_progress, handle_note_hook, handle_push_hook, note_starts_with_command, parse_mr_hook_payload, spawn_mr_review_task, MrHookPayload, }; +// §7.2 discussion-context tap shares the note-ingestion helpers (self-echo +// guards, instance-base derivation, timestamp parsing). +pub(crate) use hooks::{ + is_command_note, is_self_report, parse_note_created_at, review_base_url, self_user_id_cached, url_origin, +}; use std::sync::{OnceLock, RwLock}; diff --git a/src/server/task_queue.rs b/src/server/task_queue.rs index 5abf3dd..145945b 100644 --- a/src/server/task_queue.rs +++ b/src/server/task_queue.rs @@ -1074,6 +1074,9 @@ mod tests { async fn get_review(&self, _: Uuid) -> anyhow::Result> { anyhow::bail!("db down") } + async fn upsert_review_context(&self, _: Uuid, _: &str, _: &str, _: &str, _: i64) -> anyhow::Result<()> { + anyhow::bail!("db down") + } } let failing = Arc::new(FailingStore::default()); diff --git a/src/store/sqlx.rs b/src/store/sqlx.rs index 18fee9e..d561c60 100644 --- a/src/store/sqlx.rs +++ b/src/store/sqlx.rs @@ -487,6 +487,33 @@ impl ReviewStore for SqlxStore { .transpose() .with_context(|| format!("decode review row {task_id}")) } + + async fn upsert_review_context( + &self, + task_id: uuid::Uuid, + kind: &str, + content: &str, + content_hash: &str, + token_estimate: i64, + ) -> Result<()> { + ::sqlx::query( + "INSERT INTO review_contexts (task_id, kind, content, content_hash, token_estimate, created_at) \ + VALUES (?, ?, ?, ?, ?, ?) \ + ON CONFLICT (task_id, kind) DO UPDATE SET \ + content = excluded.content, content_hash = excluded.content_hash, \ + token_estimate = excluded.token_estimate", + ) + .bind(task_id.to_string()) + .bind(kind) + .bind(content) + .bind(content_hash) + .bind(token_estimate) + .bind(encode_ts(&Utc::now())) + .execute(self.pool()) + .await + .with_context(|| format!("upsert review_context {kind} for {task_id}"))?; + Ok(()) + } } // ─── DiscussionStore (mr_discussions, step 6a) ─── diff --git a/src/store/traits.rs b/src/store/traits.rs index 7e6f01c..72806dc 100644 --- a/src/store/traits.rs +++ b/src/store/traits.rs @@ -123,6 +123,21 @@ pub trait ReviewStore: Send + Sync { /// Single history row; `None` when the task is unknown. async fn get_review(&self, task_id: Uuid) -> Result>; + + /// Upsert one rendered prompt-context section into `review_contexts` + /// (design/persistence.md §7.2). Keyed by `(task_id, kind)`; a re-run of + /// the same task rewrites the row. `content_hash` is the sha256 hex of + /// `content` (prefix-stability/reuse checks), `token_estimate` a cheap + /// `len/4` heuristic. Best-effort: callers log failures and continue — + /// the context is still injected into the prompt either way. + async fn upsert_review_context( + &self, + task_id: Uuid, + kind: &str, + content: &str, + content_hash: &str, + token_estimate: i64, + ) -> Result<()>; } /// Handler-normalized history-list parameters (design/persistence.md §8.1 — From 43ba448bcf7e9d55dab2bd9418ec24f6ff558218 Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 13:01:58 +0800 Subject: [PATCH 10/36] =?UTF-8?q?fix(api):=20fall=20back=20to=20consolidat?= =?UTF-8?q?ion=20TL;DR=20for=20the=20review=20detail=20full-comment=20tab?= =?UTF-8?q?=20(=C2=A78.3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit raw_comment now resolves aggregated.markdown → consolidated.assessment.tl_dr, filtering empty strings to None, so team reviews without an aggregator report no longer render an empty full-comment tab. --- src/server/api/review/task.rs | 112 +++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 1 deletion(-) diff --git a/src/server/api/review/task.rs b/src/server/api/review/task.rs index 286310e..08bef42 100644 --- a/src/server/api/review/task.rs +++ b/src/server/api/review/task.rs @@ -72,7 +72,15 @@ pub(crate) fn build_review_detail(entry: &TaskEntry) -> ReviewDetail { }, }) .collect(); - let raw_comment = output.aggregated.as_ref().map(|agg| agg.markdown.clone()); + let raw_comment = output + .aggregated + .as_ref() + .map(|agg| agg.markdown.clone()) + // §8.3: no aggregator output → fall back to the lead + // consolidation TL;DR so the "full comment" tab is not + // empty; empty strings degrade to None (empty state). + .or_else(|| output.consolidated.as_ref().map(|c| c.assessment.tl_dr.clone())) + .filter(|s| !s.is_empty()); (experts, raw_comment) } Err(_) => (Vec::new(), None), @@ -291,3 +299,105 @@ pub(crate) async fn enqueue_review( }); task_id } + +#[cfg(test)] +mod tests { + use super::*; + use crate::models::{AggregatedReport, OverallAssessment, ReviewOutput, RiskLevel}; + use crate::server::task_queue::SourceMeta; + use crate::team::lead_consolidator::ConsolidatedReport; + + fn entry_with_result(result: Option) -> TaskEntry { + TaskEntry { + task_id: uuid::Uuid::new_v4(), + state: TaskState::Completed, + created_at: chrono::Utc::now(), + started_at: None, + completed_at: None, + result, + error: None, + request: None, + source_meta: SourceMeta::default(), + progress: None, + expert_name: None, + } + } + + fn consolidated_with_tl_dr(tl_dr: &str) -> ConsolidatedReport { + ConsolidatedReport { + findings: Vec::new(), + low_confidence_removed: 0, + duplicates_merged: 0, + conflicts: Vec::new(), + assessment: OverallAssessment { + score: 80, + risk_level: RiskLevel::Low, + lead_override: None, + tl_dr: tl_dr.to_string(), + unverified: false, + coverage_insufficient: false, + }, + consensus_reached: true, + total_files: 0, + reviewed_files: 0, + unreviewed_files: Vec::new(), + coverage: None, + adjudicated_removed: Vec::new(), + } + } + + fn aggregated_with_markdown(markdown: &str) -> AggregatedReport { + AggregatedReport { + findings: Vec::new(), + markdown: markdown.to_string(), + raw_llm_response: String::new(), + parse_error: None, + raw_dump_path: None, + } + } + + /// §8.3 regression: the aggregator markdown wins when both are present. + #[test] + fn raw_comment_prefers_aggregated_markdown() { + let mut output = ReviewOutput::new(Vec::new()); + output.aggregated = Some(aggregated_with_markdown("# Aggregated")); + output.consolidated = Some(consolidated_with_tl_dr("tldr")); + let entry = entry_with_result(Some(serde_json::to_value(output).unwrap())); + + let detail = build_review_detail(&entry); + assert_eq!(detail.raw_comment.as_deref(), Some("# Aggregated")); + } + + /// §8.3: no aggregator output → the consolidation TL;DR fills the + /// "full comment" tab. + #[test] + fn raw_comment_falls_back_to_consolidated_tl_dr() { + let mut output = ReviewOutput::new(Vec::new()); + output.consolidated = Some(consolidated_with_tl_dr("TL;DR: looks fine")); + let entry = entry_with_result(Some(serde_json::to_value(output).unwrap())); + + let detail = build_review_detail(&entry); + assert_eq!(detail.raw_comment.as_deref(), Some("TL;DR: looks fine")); + } + + /// §8.3: neither source present (or only empty strings) → None, the + /// pre-fix empty-state semantics. + #[test] + fn raw_comment_none_when_nothing_to_show() { + // Both absent. + let output = ReviewOutput::new(Vec::new()); + let entry = entry_with_result(Some(serde_json::to_value(output).unwrap())); + assert!(build_review_detail(&entry).raw_comment.is_none()); + + // Present but empty → filtered out, same empty state. + let mut output = ReviewOutput::new(Vec::new()); + output.aggregated = Some(aggregated_with_markdown("")); + output.consolidated = Some(consolidated_with_tl_dr("")); + let entry = entry_with_result(Some(serde_json::to_value(output).unwrap())); + assert!(build_review_detail(&entry).raw_comment.is_none()); + + // No parseable result at all. + let entry = entry_with_result(None); + assert!(build_review_detail(&entry).raw_comment.is_none()); + } +} From d0a8c82e2f00d322126596ca1309aee1b53a49ba Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 13:06:44 +0800 Subject: [PATCH 11/36] feat(api): expose storage backend kind in /system/health (0.10.0 wrap-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SqlxStore records its BackendKind (postgresql/sqlite) at connect time from the URL discrimination in design/persistence.md §4.3, exposed via backend_kind(). GET /api/v1/system/health gains a read-only storage_backend field: "postgresql" / "sqlite" / "disabled" (no DB attached — REVIEW_DISABLE_DB=1, tests, embedded use), for the frontend config page. --- src/server/api/system.rs | 30 +++++++++++++++ src/store/mod.rs | 79 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/src/server/api/system.rs b/src/server/api/system.rs index 4224dda..5a9a444 100644 --- a/src/server/api/system.rs +++ b/src/server/api/system.rs @@ -135,10 +135,20 @@ async fn system_health(State(state): State>) -> impl IntoResponse // POST /api/v1/reviews. let llm_configured = llm_configs.iter().any(|c| !c.api_base.trim().is_empty()); + // Persistence backend actually in use (0.10.0): "postgresql" / "sqlite" + // from the store's connect-time URL discrimination; "disabled" when no + // DB is attached (`REVIEW_DISABLE_DB=1`, tests, embedded use). + let storage_backend = state + .db + .as_ref() + .map(|db| db.backend_kind().as_str()) + .unwrap_or("disabled"); + Json(serde_json::json!({ "integrations": integrations, "llmProviders": llm_providers, "llmConfigured": llm_configured, + "storage_backend": storage_backend, "overall": overall, "lastChecked": chrono::Utc::now().to_rfc3339(), })) @@ -611,4 +621,24 @@ mod tests { "an entry with api_base must report true: {body}" ); } + + /// `/system/health` exposes `storage_backend`: "disabled" when no DB is + /// attached (`REVIEW_DISABLE_DB=1`, tests, embedded use). + #[tokio::test] + async fn system_health_reports_storage_backend_disabled_without_db() { + let body = health_json(AppState::new(vec![])).await; + assert_eq!(body["storage_backend"], "disabled", "no db attached: {body}"); + } + + /// With an in-memory SQLite store attached, `storage_backend` reports + /// "sqlite". The "postgresql" value is covered function-level in + /// `store::tests::backend_kind_discriminates_by_url_scheme` (no live PG + /// in unit tests). + #[tokio::test] + async fn system_health_reports_storage_backend_sqlite_with_db() { + let mut state = AppState::new(vec![]); + state.db = Some(Arc::new(crate::store::SqlxStore::new_in_memory().await.unwrap())); + let body = health_json(state).await; + assert_eq!(body["storage_backend"], "sqlite", "sqlite store attached: {body}"); + } } diff --git a/src/store/mod.rs b/src/store/mod.rs index 84c1b3e..623f703 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -35,6 +35,37 @@ use chrono::{DateTime, SecondsFormat, Utc}; /// Embedded migrations (compiled in via `sqlx::migrate!`). static MIGRATOR: ::sqlx::migrate::Migrator = ::sqlx::migrate!("./migrations"); +/// Which storage backend a [`SqlxStore`] pool talks to, recorded at connect +/// time from the URL (design/persistence.md §4.3). Surfaced read-only via +/// `GET /api/v1/system/health` as `storage_backend`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BackendKind { + /// `postgres://` / `postgresql://` URL. + Postgresql, + /// Everything else (embedded SQLite, incl. in-memory). + Sqlite, +} + +impl BackendKind { + /// Wire value exposed to the API layer (`"postgresql"` / `"sqlite"`). + pub fn as_str(self) -> &'static str { + match self { + Self::Postgresql => "postgresql", + Self::Sqlite => "sqlite", + } + } +} + +/// URL discrimination per design/persistence.md §4.3: `postgres://` / +/// `postgresql://` → PostgreSQL; anything else → SQLite. +fn backend_kind_of(url: &str) -> BackendKind { + if url.starts_with("postgres://") || url.starts_with("postgresql://") { + BackendKind::Postgresql + } else { + BackendKind::Sqlite + } +} + /// SQLx store backed by an `Any` pool (PostgreSQL or SQLite). /// /// Besides pool construction / SQLite pragmas / migrations, the store holds @@ -44,6 +75,7 @@ static MIGRATOR: ::sqlx::migrate::Migrator = ::sqlx::migrate!("./migrations"); #[derive(Debug, Clone)] pub struct SqlxStore { pool: ::sqlx::AnyPool, + kind: BackendKind, pub(crate) key: [u8; 32], } @@ -69,17 +101,17 @@ impl SqlxStore { /// available to tests that need a stable key without a config dir). pub async fn connect_with_key(url: &str, key: [u8; 32]) -> Result { ::sqlx::any::install_default_drivers(); - let is_postgres = url.starts_with("postgres://") || url.starts_with("postgresql://"); + let kind = backend_kind_of(url); let pool = ::sqlx::any::AnyPoolOptions::new().connect(url).await.with_context(|| { format!( "failed to connect to database ({url_scheme})", url_scheme = scheme_of(url) ) })?; - if !is_postgres { + if kind == BackendKind::Sqlite { apply_sqlite_pragmas(&pool).await?; } - Ok(Self { pool, key }) + Ok(Self { pool, kind, key }) } /// Connect to the default embedded SQLite database under `config_dir` @@ -111,7 +143,11 @@ impl SqlxStore { .await .context("failed to open in-memory sqlite database")?; apply_sqlite_pragmas(&pool).await?; - Ok(Self { pool, key }) + Ok(Self { + pool, + kind: BackendKind::Sqlite, + key, + }) } /// Apply the embedded migrations (idempotent — already-applied @@ -121,6 +157,12 @@ impl SqlxStore { Ok(()) } + /// The storage backend this pool talks to, recorded at connect time + /// (PostgreSQL or SQLite). + pub fn backend_kind(&self) -> BackendKind { + self.kind + } + /// Access the underlying pool (used by trait implementations in /// [`sqlx`] and by tests). pub fn pool(&self) -> &::sqlx::AnyPool { @@ -171,6 +213,35 @@ mod tests { use ::sqlx::Row; use chrono::TimeZone; + /// URL discrimination per §4.3: `postgres://` / `postgresql://` → + /// PostgreSQL, everything else → SQLite. Pure-function-level coverage of + /// the `"postgresql"` wire value (a live PG is not available in unit + /// tests; the end-to-end PG smoke test is `migrate_on_postgres_smoke`). + #[test] + fn backend_kind_discriminates_by_url_scheme() { + assert_eq!( + backend_kind_of("postgres://u:p@db.example/review"), + BackendKind::Postgresql + ); + assert_eq!( + backend_kind_of("postgresql://u:p@db.example/review"), + BackendKind::Postgresql + ); + assert_eq!(BackendKind::Postgresql.as_str(), "postgresql"); + + assert_eq!(backend_kind_of("sqlite:///tmp/review.db?mode=rwc"), BackendKind::Sqlite); + assert_eq!(backend_kind_of("sqlite::memory:"), BackendKind::Sqlite); + assert_eq!(BackendKind::Sqlite.as_str(), "sqlite"); + } + + /// An in-memory SQLite store reports the sqlite backend kind. + #[tokio::test] + async fn in_memory_store_reports_sqlite_backend_kind() { + let store = SqlxStore::new_in_memory().await.unwrap(); + assert_eq!(store.backend_kind(), BackendKind::Sqlite); + assert_eq!(store.backend_kind().as_str(), "sqlite"); + } + /// 验证点 A(a): in-memory SQLite + migrate creates the schema, and a /// second migrate run is an idempotent no-op. #[tokio::test] From 7e1658b6893ebd1022ba5a1bf61057e938a0ce0c Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 13:15:17 +0800 Subject: [PATCH 12/36] feat(config-ui): show read-only storage backend from /system/health (0.10.0 wrap-up) The config page Advanced card gains a permanently-disabled row displaying the persistence backend in use (PostgreSQL / SQLite / disabled), sourced from the storage_backend field added to GET /api/v1/system/health in d0a8c82. The service layer normalizes the one snake_case key to storageBackend and validates it against the known kinds; a health-check failure or an older server simply hides the row (fail-silent). Also drops an unused catch binding in Configuration.vue flagged by eslint (no-unused-vars). --- frontend/src/i18n/locales/en.ts | 6 ++++++ frontend/src/i18n/locales/fr.ts | 6 ++++++ frontend/src/i18n/locales/ja.ts | 6 ++++++ frontend/src/i18n/locales/ko.ts | 6 ++++++ frontend/src/i18n/locales/zh-CN.ts | 6 ++++++ frontend/src/i18n/locales/zh-TW.ts | 6 ++++++ frontend/src/services/health.ts | 13 ++++++++++-- frontend/src/types/dashboard.ts | 8 ++++++++ frontend/src/views/Configuration.vue | 30 +++++++++++++++++++++++++++- 9 files changed, 84 insertions(+), 3 deletions(-) diff --git a/frontend/src/i18n/locales/en.ts b/frontend/src/i18n/locales/en.ts index 9ac3c21..a7b496a 100644 --- a/frontend/src/i18n/locales/en.ts +++ b/frontend/src/i18n/locales/en.ts @@ -474,6 +474,12 @@ export default { requestTimeout: 'Request timeout (seconds)', enableMetrics: 'Enable metrics', debugMode: 'Debug mode', + storageBackend: 'Storage backend', + storageBackendKind: { + postgresql: 'PostgreSQL', + sqlite: 'SQLite', + disabled: 'Disabled', + }, }, }, llm: { diff --git a/frontend/src/i18n/locales/fr.ts b/frontend/src/i18n/locales/fr.ts index cf677ac..18d2875 100644 --- a/frontend/src/i18n/locales/fr.ts +++ b/frontend/src/i18n/locales/fr.ts @@ -470,6 +470,12 @@ export default { requestTimeout: "Délai d'expiration de la requête (secondes)", enableMetrics: 'Activer les métriques', debugMode: 'Mode débogage', + storageBackend: 'Backend de stockage', + storageBackendKind: { + postgresql: 'PostgreSQL', + sqlite: 'SQLite', + disabled: 'Désactivé', + }, }, }, llm: { diff --git a/frontend/src/i18n/locales/ja.ts b/frontend/src/i18n/locales/ja.ts index 065ac09..344c1a3 100644 --- a/frontend/src/i18n/locales/ja.ts +++ b/frontend/src/i18n/locales/ja.ts @@ -465,6 +465,12 @@ export default { requestTimeout: 'リクエストタイムアウト（秒）', enableMetrics: 'メトリクスを有効化', debugMode: 'デバッグモード', + storageBackend: 'ストレージバックエンド', + storageBackendKind: { + postgresql: 'PostgreSQL', + sqlite: 'SQLite', + disabled: '無効', + }, }, }, llm: { diff --git a/frontend/src/i18n/locales/ko.ts b/frontend/src/i18n/locales/ko.ts index 3f9def8..d731193 100644 --- a/frontend/src/i18n/locales/ko.ts +++ b/frontend/src/i18n/locales/ko.ts @@ -464,6 +464,12 @@ export default { requestTimeout: '요청 시간 초과（초）', enableMetrics: '메트릭 활성화', debugMode: '디버그 모드', + storageBackend: '스토리지 백엔드', + storageBackendKind: { + postgresql: 'PostgreSQL', + sqlite: 'SQLite', + disabled: '비활성화됨', + }, }, }, llm: { diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 0829315..646305b 100644 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -459,6 +459,12 @@ export default { requestTimeout: '请求超时（秒）', enableMetrics: '启用指标', debugMode: '调试模式', + storageBackend: '存储后端', + storageBackendKind: { + postgresql: 'PostgreSQL', + sqlite: 'SQLite', + disabled: '已禁用', + }, }, }, llm: { diff --git a/frontend/src/i18n/locales/zh-TW.ts b/frontend/src/i18n/locales/zh-TW.ts index 042558c..daeff4d 100644 --- a/frontend/src/i18n/locales/zh-TW.ts +++ b/frontend/src/i18n/locales/zh-TW.ts @@ -459,6 +459,12 @@ export default { requestTimeout: '請求逾時（秒）', enableMetrics: '啟用指標', debugMode: '偵錯模式', + storageBackend: '儲存後端', + storageBackendKind: { + postgresql: 'PostgreSQL', + sqlite: 'SQLite', + disabled: '已停用', + }, }, }, llm: { diff --git a/frontend/src/services/health.ts b/frontend/src/services/health.ts index bae3c6b..d954f87 100644 --- a/frontend/src/services/health.ts +++ b/frontend/src/services/health.ts @@ -1,10 +1,19 @@ import { request } from './api'; -import type { SystemHealth } from '../types/dashboard'; +import type { StorageBackendKind, SystemHealth } from '../types/dashboard'; + +const STORAGE_BACKENDS: readonly StorageBackendKind[] = ['postgresql', 'sqlite', 'disabled']; /** * Fetch the server's system health status. * @returns System health information (uptime, memory, version, etc.). */ export async function getSystemHealth(): Promise { - return request('/system/health'); + // `storage_backend` (0.10.0) is the one snake_case key on this otherwise + // camelCase payload; normalize it here so consumers see `storageBackend`. + // Unknown/absent values degrade to undefined (the caller hides the row). + const raw = await request('/system/health'); + return { + ...raw, + storageBackend: STORAGE_BACKENDS.find((k) => k === raw.storage_backend), + }; } diff --git a/frontend/src/types/dashboard.ts b/frontend/src/types/dashboard.ts index 21ae7c5..d90fe64 100644 --- a/frontend/src/types/dashboard.ts +++ b/frontend/src/types/dashboard.ts @@ -24,6 +24,12 @@ export interface HealthStatus { message?: string; } +/** + * Persistence backend in use, reported by `/system/health` (0.10.0). + * Absent when the server predates the field. + */ +export type StorageBackendKind = 'postgresql' | 'sqlite' | 'disabled'; + export interface SystemHealth { integrations: HealthStatus[]; llmProviders: HealthStatus[]; @@ -31,6 +37,8 @@ export interface SystemHealth { lastChecked: string; /** False when the server has no usable LLM configured (reviews cannot run). */ llmConfigured: boolean; + /** Persistence backend kind; normalized from the raw `storage_backend` key. */ + storageBackend?: StorageBackendKind; } // Display-facing status for recent reviews. The backend reports the real task diff --git a/frontend/src/views/Configuration.vue b/frontend/src/views/Configuration.vue index f0201e2..c072f11 100644 --- a/frontend/src/views/Configuration.vue +++ b/frontend/src/views/Configuration.vue @@ -235,6 +235,14 @@ + + + + + + @@ -274,7 +282,9 @@ import { ElMessageBox, ElNotification } from 'element-plus' import { useI18n } from 'vue-i18n' import { useConfig } from '../composables/useConfig' import { useConfigForm } from '../composables/useConfigForm' +import { getSystemHealth } from '../services/health' import type { AppConfig, GitPlatformConfig } from '../types/config' +import type { StorageBackendKind } from '../types/dashboard' import GitPlatformsSection from '../components/Config/GitPlatformsSection.vue' // --- Composables --- @@ -306,6 +316,23 @@ const loadError = computed(() => !!cfg.error.value) const saving = cfg.saving const showAdvanced = ref(false) +/* Read-only runtime info: the persistence backend in use, from + * GET /system/health (`storage_backend`, 0.10.0). Fail-silent — a health + * check error or an older server simply leaves the row hidden. */ +const storageBackend = ref(null) + +const storageBackendLabel = computed(() => + storageBackend.value ? t(`config.advanced.storageBackendKind.${storageBackend.value}`) : '' +) + +function loadStorageBackend() { + getSystemHealth() + .then((health) => { + storageBackend.value = health.storageBackend ?? null + }) + .catch(() => {}) +} + // Card refs for flash animation const gitPlatformsCardRef = ref() const rulesCardRef = ref() @@ -400,7 +427,7 @@ async function saveChanges() { setTimeout(() => el.classList.remove('flash-success'), 600) } }) - } catch (e) { + } catch { ElNotification({ title: t('common.error'), message: t('config.saveFailed'), @@ -450,6 +477,7 @@ onMounted(() => { window.addEventListener('beforeunload', handleBeforeUnload) window.addEventListener('resize', handleResize) loadConfig() + loadStorageBackend() }) // --- Error handling --- From 816f742ba275ef16c2731965d7caed81bc4a36f8 Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 14:12:34 +0800 Subject: [PATCH 13/36] fix(store): rewrite ? placeholders to $1..$n for PostgreSQL in the store layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sqlx 0.8.6's Any driver passes statement text through verbatim — it does NOT translate `?` to $1..$n (no placeholder/rewrite logic anywhere in sqlx-core-0.8.6/src/any/), so every bound DML failed on real PostgreSQL with 42601 syntax error (0.10.0 E2E). The earlier assumption baked into src/store/mod.rs and design/persistence.md §3.1 was wrong. - src/store/placeholders.rs: lexer-based rewriter; `?` inside single-quoted literals (incl. '' escapes), double-quoted identifiers, -- line comments and nested /* */ block comments stays literal. Covered by unit tests for mixed literals, escaped quotes, comments, consecutive/double-digit placeholders, and unterminated input. - src/store/mod.rs: adapt_sql(kind, sql) / SqlxStore::sql() — the single adaptation point; PG rewrites (borrowing when there is nothing to rewrite), SQLite passes the text through unchanged. Module doc and the ignored PG smoke test corrected. - src/store/sqlx.rs: every statement (ConfigStore / ReviewStore / DiscussionStore, incl. tx helpers, the interrupted sweep, the dynamic list pagination and both upserts) now passes through adapt_sql once. - design/persistence.md: §3.1 placeholder row and verification point A corrected; Migrator is unaffected because the Any migrate path delegates to the real underlying driver. Verified on postgres:16-alpine: migrate_on_postgres_smoke green, server health reports storage_backend=postgresql, ui-state.toml import fills git_platforms/llm_providers/app_settings (secrets enc:-prefixed), the restart sweep flips pending/running to failed/interrupted, and review list pagination/filters return correct pages. --- design/persistence.md | 6 +- src/store/mod.rs | 75 +++++++- src/store/placeholders.rs | 209 ++++++++++++++++++++ src/store/sqlx.rs | 396 +++++++++++++++++++++----------------- 4 files changed, 496 insertions(+), 190 deletions(-) create mode 100644 src/store/placeholders.rs diff --git a/design/persistence.md b/design/persistence.md index 1116dc3..3bb4c6a 100644 --- a/design/persistence.md +++ b/design/persistence.md @@ -66,7 +66,7 @@ | 主题 | PG | SQLite | 本文的取舍 | |---|---|---|---| -| 占位符 | 原生 `$1..$n` | `?` | **统一写 `?`**。Any 驱动内部为 PG 做翻译；写 `$1` 在 SQLite 端直接报错。（落地验证点 A，见 §11） | +| 占位符 | 原生 `$1..$n` | `?` | **统一写 `?`**，执行前由 store 层改写。**更正（0.10.0 E2E 实证）**：「Any 驱动内部为 PG 做翻译」的假设被证伪——sqlx 0.8.6 的 Any 驱动把 SQL 原样透传给 PG 解析器，`?` 直接 `42601 syntax error`（`sqlx-core-0.8.6/src/any/` 无任何 placeholder/rewrite 逻辑；`migrate` 走底层真实驱动的 ledger，不受影响）。因此 store 层自带重写器 `src/store/placeholders.rs`：PG 把顶层 `?` 依次改写为 `$1..$n`（正确跳过 `'...'` 字符串字面量含 `''` 转义、`"..."` 标识符、`--` / `/* */` 注释内的 `?`），SQLite 原样透传；所有语句经 `SqlxStore::sql` / `adapt_sql` 收口一次，不逐条手改 | | upsert | `ON CONFLICT ... DO UPDATE/NOTHING` | 同语法（≥3.24） | 两端一致，直接用；sqlx 内置 libsqlite3 版本远高于此 | | `RETURNING` | 支持 | ≥3.35 支持 | **一律不用**。主键全部由 Rust 侧生成（UUID v4），写后无需回读；避免 Any 下两端 decode 行为差异 | | JSON 列 | 原生 JSONB | TEXT | **DDL 用 TEXT，绑定用 `String`**：store 层 `serde_json::to_string` 后按 TEXT 绑定，读出再 `from_str`。若声明 PG JSONB 列而 SQLite 是 TEXT，`serde_json::Value` 在 PG 端会按 JSONB 编码、绑到 TEXT 列报类型错——应用层序列化是唯一两头都稳的做法 | @@ -347,7 +347,7 @@ output.aggregated.map(|a| a.markdown) ## 10. 实施清单（依赖序，可逐项验收） -1. **[祁远]** `Cargo.toml` 加 sqlx 0.8（指定 features）；`src/store/` 骨架 + `migrations/0001_init.sql`；`SqlxStore::connect/new_in_memory` + migrate 接线。**验收**：验证点 A（Any 占位符翻译 + AnyPool migrate smoke test）通过，SQLite 内存库建表成功。 +1. **[祁远]** `Cargo.toml` 加 sqlx 0.8（指定 features）；`src/store/` 骨架 + `migrations/0001_init.sql`；`SqlxStore::connect/new_in_memory` + migrate 接线。**验收**：验证点 A（占位符改写 + AnyPool migrate smoke test）通过，SQLite 内存库建表成功。 2. **[祁远]** `rows.rs` 加密边界 + `ConfigStore` 实现（§3.2 三张配置表 + §6.2 保存路径）。**验收**：配置 PUT→库→重启回放 round-trip 单测绿；LLM key 在库里是 `enc:`。 3. **[祁远]** 一次性导入（§6.1 第 3 步，单事务 + rename 备份 + 失败回退）。**验收**：老 `ui-state.toml`（含明文 LLM key）启动一次后：库里有数据、文件改名、GET /config 行为不变、env 覆盖矩阵（§6.3）逐行单测。 4. **[梁序]** `ReviewStore` + TaskStore 写穿（§5.2）+ 重启恢复（§5.3）。**验收**：跑一个评审 → kill -9 → 重启 → 该任务在库里是 failed/interrupted 文案；完成的评审重启后历史可查。 @@ -361,7 +361,7 @@ output.aggregated.map(|a| a.markdown) ## 11. 待验证点（实现前确认，不确定处不猜） -- **验证点 A**（✅ 已验证，sqlx 0.8.6 smoke test）：`?` 占位符 PG 翻译、`Migrator` 在 `AnyPool` 上的行为均正常；SQLite 侧通过，PG 侧留 `#[ignore]` 入口待有实例时跑。附带结论：Any 驱动无 chrono/uuid 的 `Type` 实现，SQLite 拒绝对 `TIMESTAMP` 声明列做 String 解码——时间戳/uuid 一律 TEXT 绑定（§3.1 已按此定稿）。 +- **验证点 A**（✅ 已验证，sqlx 0.8.6；⚠️ 占位符结论已由 0.10.0 E2E 更正）：`Migrator` 在 `AnyPool` 上的行为正常（Any 端委托底层真实驱动，PG ledger 不受影响）；但「Any 驱动为 PG 翻译 `?` 占位符」的假设在真实 PG E2E 上被证伪（带绑定参数的 DML 全部 `42601`），落地方案改为 store 层自行重写（`placeholders::rewrite`，见 §3.1 占位符行）。附带结论仍然有效：Any 驱动无 chrono/uuid 的 `Type` 实现，SQLite 拒绝对 `TIMESTAMP` 声明列做 String 解码——时间戳/uuid 一律 TEXT 绑定（§3.1 已按此定稿）。 - **验证点 B**：`config/resolver/` 是否从 config.toml 承载 `git_platforms`（§6.3 表中标注待核实）。方法：`Grep "git_platforms" src/config/`。 - **验证点 C**：评审报告的固定前缀常量位置（§7.1 自噬防护条件 a）。方法：`Grep` publisher/output 模块的报告头部模板。 - **验证点 D**（✅ 已验证）：通过。固定宽度 RFC 3339 UTC 串按 TEXT 存储，字典序 == 时间序，`ORDER BY created_at` 排序正确（分页前提成立）。 diff --git a/src/store/mod.rs b/src/store/mod.rs index 623f703..29500a1 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -4,7 +4,12 @@ //! and embedded SQLite (fallback). All SQL lives behind this module; the //! dialect rules of `design/persistence.md` §3.1 apply everywhere: //! -//! - placeholders are always `?` (the Any driver translates them for PG); +//! - placeholders are always written `?` in the SQL text (the only spelling +//! SQLite accepts); the Any driver passes statements to PG verbatim — it +//! does NOT translate `?` (sqlx 0.8.6, verified in `sqlx-core/src/any/` +//! and by 0.10.0 E2E on real PG) — so the store layer rewrites `?` to +//! `$1..$n` itself before execution ([`SqlxStore::sql`] / +//! [`placeholders::rewrite`]); //! - no `RETURNING` (primary keys are Rust-side UUIDs); //! - JSON columns are `TEXT`, serialized/deserialized by the store layer; //! - timestamps are generated by chrono on the Rust side, never by DDL @@ -23,10 +28,12 @@ //! boundary in [`rows`]). `ReviewStore` / `DiscussionStore` land in later //! steps. +pub mod placeholders; pub mod rows; pub mod sqlx; pub mod traits; +use std::borrow::Cow; use std::path::Path; use anyhow::{Context, Result}; @@ -168,6 +175,26 @@ impl SqlxStore { pub fn pool(&self) -> &::sqlx::AnyPool { &self.pool } + + /// Adapt a statement's `?` placeholders for this store's backend — + /// see [`adapt_sql`]. Every statement executed through this store must + /// pass through here (design/persistence.md §3.1 placeholder row). + pub(crate) fn sql<'a>(&self, sql: &'a str) -> Cow<'a, str> { + adapt_sql(self.kind, sql) + } +} + +/// Adapt a statement for `kind`: PostgreSQL gets `?` rewritten to `$1..$n` +/// (the Any driver passes SQL through verbatim — see [`placeholders`]), +/// SQLite gets the text unchanged. Borrows whenever no rewrite is needed. +/// +/// This is a pure text transform over the SQL *statement*; bind values are +/// attached afterwards and are never seen by the rewriter. +pub(crate) fn adapt_sql(kind: BackendKind, sql: &str) -> Cow<'_, str> { + match kind { + BackendKind::Postgresql if sql.contains('?') => Cow::Owned(placeholders::rewrite(sql)), + _ => Cow::Borrowed(sql), + } } /// Scheme prefix of a database URL, for error messages that must not leak @@ -234,6 +261,34 @@ mod tests { assert_eq!(BackendKind::Sqlite.as_str(), "sqlite"); } + /// `adapt_sql` dispatch: PG rewrites `?` (and borrows when there is + /// nothing to rewrite), SQLite always borrows the original text. + #[test] + fn adapt_sql_rewrites_for_postgres_only() { + assert_eq!( + adapt_sql(BackendKind::Postgresql, "SELECT * FROM t WHERE a = ? AND b = ?"), + "SELECT * FROM t WHERE a = $1 AND b = $2" + ); + assert!( + matches!( + adapt_sql(BackendKind::Postgresql, "SELECT * FROM t WHERE a = ?"), + Cow::Owned(_) + ), + "PG with placeholders must own the rewritten text" + ); + assert!( + matches!(adapt_sql(BackendKind::Postgresql, "SELECT * FROM t"), Cow::Borrowed(_)), + "PG without placeholders must borrow" + ); + assert!( + matches!( + adapt_sql(BackendKind::Sqlite, "SELECT * FROM t WHERE a = ?"), + Cow::Borrowed(_) + ), + "SQLite must get the original `?` text untouched" + ); + } + /// An in-memory SQLite store reports the sqlite backend kind. #[tokio::test] async fn in_memory_store_reports_sqlite_backend_kind() { @@ -361,9 +416,10 @@ mod tests { /// `DATABASE_URL=postgres://... cargo test store -- --ignored` /// /// Verifies: migrate on an AnyPool against PG (incl. the - /// `_sqlx_migrations` ledger), idempotent re-run, and `?` placeholder - /// translation. Timestamps are TEXT columns bound as RFC 3339 strings - /// (Any driver constraint), so the read-back is a plain String round trip. + /// `_sqlx_migrations` ledger), idempotent re-run, and the store-layer + /// `?` → `$1..$n` placeholder rewrite ([`SqlxStore::sql`]). Timestamps + /// are TEXT columns bound as RFC 3339 strings (Any driver constraint), + /// so the read-back is a plain String round trip. #[tokio::test] #[ignore = "requires DATABASE_URL pointing at a scratch PostgreSQL"] async fn migrate_on_postgres_smoke() { @@ -373,22 +429,25 @@ mod tests { // Idempotent second run. store.migrate().await.unwrap(); - // `?` placeholders must be translated by the Any driver. + // `?` placeholders are rewritten by the store layer, not the driver. let now = Utc::now(); - ::sqlx::query("INSERT INTO reviews (task_id, state, created_at) VALUES (?, ?, ?)") + let insert = store.sql("INSERT INTO reviews (task_id, state, created_at) VALUES (?, ?, ?)"); + ::sqlx::query(&insert) .bind("pg-smoke-0001") .bind("pending") .bind(encode_ts(&now)) .execute(store.pool()) .await .unwrap(); - let back: String = ::sqlx::query_scalar("SELECT created_at FROM reviews WHERE task_id = ?") + let select = store.sql("SELECT created_at FROM reviews WHERE task_id = ?"); + let back: String = ::sqlx::query_scalar(&select) .bind("pg-smoke-0001") .fetch_one(store.pool()) .await .unwrap(); assert_eq!(decode_ts(&back).unwrap(), now); - ::sqlx::query("DELETE FROM reviews WHERE task_id = ?") + let delete = store.sql("DELETE FROM reviews WHERE task_id = ?"); + ::sqlx::query(&delete) .bind("pg-smoke-0001") .execute(store.pool()) .await diff --git a/src/store/placeholders.rs b/src/store/placeholders.rs new file mode 100644 index 0000000..6878129 --- /dev/null +++ b/src/store/placeholders.rs @@ -0,0 +1,209 @@ +//! Placeholder rewriting for the dual-backend `Any` pool. +//! +//! Store SQL is written with `?` placeholders (the only spelling SQLite +//! accepts positionally). sqlx 0.8.6's `Any` driver passes the statement +//! text through verbatim — it does NOT translate `?` to `$1..$n` (verified +//! by grepping `sqlx-core-0.8.6/src/any/` and by 0.10.0 E2E on real PG, +//! where every bound DML failed with `42601 syntax error`). The store layer +//! therefore rewrites placeholders itself before executing on PostgreSQL; +//! SQLite gets the text unchanged ([`crate::store::SqlxStore::sql`]). +//! +//! The rewriter is a small lexer, not a string replace: `?` inside +//! single-quoted string literals (with `''` escaping), double-quoted +//! identifiers, `--` line comments and `/* */` block comments is literal +//! text, not a bind slot. Backslash is NOT an escape in standard SQL string +//! literals (PG runs with `standard_conforming_strings=on` by default), so +//! `'\'` is a complete one-char literal; only `''` escapes a quote. + +/// Rewrite every top-level `?` in `sql` to `$1..$n` (positional, in order of +/// appearance). `?` inside string literals, quoted identifiers, or comments +/// is left untouched. The input is assumed syntactically valid; unterminated +/// literals/comments are copied verbatim (the database will reject them). +pub(crate) fn rewrite(sql: &str) -> String { + let mut out = String::with_capacity(sql.len() + 8); + let mut chars = sql.chars().peekable(); + let mut n: u32 = 0; + while let Some(c) = chars.next() { + match c { + '?' => { + n += 1; + out.push('$'); + out.push_str(&n.to_string()); + } + '\'' | '"' => { + let quote = c; + out.push(c); + let it = chars.by_ref(); + while let Some(inner) = it.next() { + out.push(inner); + if inner == quote { + // A doubled quote is an escaped literal quote: copy + // it and stay inside the literal. + if it.peek() == Some("e) { + out.push(quote); + it.next(); + } else { + break; + } + } + } + } + '-' if chars.peek() == Some(&'-') => { + out.push('-'); + out.push('-'); + chars.next(); + for inner in chars.by_ref() { + out.push(inner); + if inner == '\n' { + break; + } + } + } + '/' if chars.peek() == Some(&'*') => { + out.push('/'); + out.push('*'); + chars.next(); + // PostgreSQL nests block comments; depth tracking keeps `?` + // inside nested sections literal too (SQLite sees none of + // this — it takes the original text). + let mut depth = 1u32; + let mut prev = '\0'; + for inner in chars.by_ref() { + out.push(inner); + if prev == '/' && inner == '*' { + depth += 1; + } else if prev == '*' && inner == '/' { + depth -= 1; + if depth == 0 { + break; + } + } + prev = inner; + } + } + other => out.push(other), + } + } + out +} + +#[cfg(test)] +mod tests { + use super::rewrite; + + #[test] + fn simple_placeholders_are_numbered_in_order() { + assert_eq!( + rewrite("SELECT * FROM t WHERE a = ? AND b = ?"), + "SELECT * FROM t WHERE a = $1 AND b = $2" + ); + } + + #[test] + fn no_placeholders_passes_through() { + let sql = "DELETE FROM git_platforms"; + assert_eq!(rewrite(sql), sql); + } + + #[test] + fn consecutive_and_double_digit_placeholders() { + let sql = format!("VALUES ({})", ["?"; 12].join(", ")); + let expected = format!( + "VALUES ({})", + (1..=12).map(|i| format!("${i}")).collect::>().join(", ") + ); + assert_eq!(rewrite(&sql), expected); + // Trailing placeholder at end of statement. + assert_eq!(rewrite("DELETE FROM t WHERE id = ?"), "DELETE FROM t WHERE id = $1"); + } + + #[test] + fn single_quoted_literal_preserves_question_mark() { + // The interrupted-sweep UPDATE shape: literal text containing `?` + // alongside a real bind slot. + assert_eq!( + rewrite("UPDATE reviews SET error = 'interrupted: why?' WHERE task_id = ?"), + "UPDATE reviews SET error = 'interrupted: why?' WHERE task_id = $1" + ); + } + + #[test] + fn escaped_single_quote_keeps_literal_open() { + // `''` is an escaped quote, so the `?` after it is still inside the + // literal; only the final one is a bind slot. + assert_eq!( + rewrite("INSERT INTO t (a, b) VALUES ('it''s a ?', ?)"), + "INSERT INTO t (a, b) VALUES ('it''s a ?', $1)" + ); + } + + #[test] + fn backslash_does_not_escape_in_standard_literals() { + // standard_conforming_strings=on: '\' is one complete literal; the + // `?` after it is a bind slot. + assert_eq!( + rewrite("LOWER(source_meta) LIKE LOWER(?) ESCAPE '\\' AND state = ?"), + "LOWER(source_meta) LIKE LOWER($1) ESCAPE '\\' AND state = $2" + ); + } + + #[test] + fn double_quoted_identifier_preserves_question_mark() { + assert_eq!( + rewrite("SELECT \"weird?column\" FROM t WHERE id = ?"), + "SELECT \"weird?column\" FROM t WHERE id = $1" + ); + // Escaped `""` keeps the identifier open. + assert_eq!( + rewrite("SELECT \"a\"\"?b\" FROM t WHERE id = ?"), + "SELECT \"a\"\"?b\" FROM t WHERE id = $1" + ); + } + + #[test] + fn line_comment_preserves_question_mark() { + assert_eq!( + rewrite("SELECT ? -- trailing ? in comment\nWHERE x = ?"), + "SELECT $1 -- trailing ? in comment\nWHERE x = $2" + ); + // A lone `-` is an operator, not a comment start. + assert_eq!(rewrite("SELECT a - ? FROM t"), "SELECT a - $1 FROM t"); + } + + #[test] + fn block_comment_preserves_question_mark_including_nested() { + assert_eq!( + rewrite("SELECT * FROM t /* filter: ? */ WHERE x = ?"), + "SELECT * FROM t /* filter: ? */ WHERE x = $1" + ); + // PG nests block comments: the `?` inside the nested section is + // literal, and the outer comment does not end early. + assert_eq!( + rewrite("/* outer /* nested ? */ still comment ? */ SELECT ?"), + "/* outer /* nested ? */ still comment ? */ SELECT $1" + ); + // A lone `/` is a division operator. + assert_eq!(rewrite("SELECT a / ? FROM t"), "SELECT a / $1 FROM t"); + } + + #[test] + fn unterminated_literal_is_copied_verbatim() { + // Degenerate input the database will reject anyway: the rewriter + // must not invent placeholders inside it. + assert_eq!( + rewrite("INSERT INTO t VALUES ('oops ?"), + "INSERT INTO t VALUES ('oops ?" + ); + } + + #[test] + fn mixed_real_statement_from_list_reviews() { + // The assembled shape of the history-list page query: literal + // backslash, LIKE needle, and trailing LIMIT/OFFSET slots. + let sql = "SELECT task_id FROM reviews WHERE LOWER(source_meta) LIKE LOWER(?) ESCAPE '\\' \ + AND created_at >= ? ORDER BY created_at DESC LIMIT ? OFFSET ?"; + let expected = "SELECT task_id FROM reviews WHERE LOWER(source_meta) LIKE LOWER($1) ESCAPE '\\' \ + AND created_at >= $2 ORDER BY created_at DESC LIMIT $3 OFFSET $4"; + assert_eq!(rewrite(sql), expected); + } +} diff --git a/src/store/sqlx.rs b/src/store/sqlx.rs index d561c60..36bb079 100644 --- a/src/store/sqlx.rs +++ b/src/store/sqlx.rs @@ -3,6 +3,9 @@ //! //! Dialect discipline (§3.1): `?` placeholders only, no `RETURNING`, JSON as //! bound `String`, timestamps via `encode_ts` / `decode_ts` (RFC 3339 TEXT). +//! Every statement passes through [`crate::store::adapt_sql`] exactly once +//! before construction: PostgreSQL gets `?` rewritten to `$1..$n` (the Any +//! driver passes SQL through verbatim), SQLite borrows the text unchanged. //! NOTE: this file is itself named `sqlx.rs` — the sibling module shadows //! the extern crate lexically, so every reference to the real sqlx crate //! must use the absolute `::sqlx::` path. @@ -17,7 +20,7 @@ use crate::server::task_queue::{SourceMeta, TaskEntry}; use super::rows; use super::traits::{ConfigStore, DiscussionNote, DiscussionStore, ReviewListQuery, ReviewStore}; -use super::{encode_ts, SqlxStore}; +use super::{adapt_sql, encode_ts, BackendKind, SqlxStore}; const LEGACY_GITLAB_KEY: &str = "gitlab"; const UI_KEY: &str = "ui"; @@ -25,84 +28,103 @@ const UI_KEY: &str = "ui"; type AnyTx<'a> = ::sqlx::Transaction<'a, ::sqlx::Any>; /// DELETE + re-INSERT the whole git_platforms set inside `tx`. -async fn replace_git_platforms_in(tx: &mut AnyTx<'_>, platforms: &[GitPlatformConfig], key: &[u8; 32]) -> Result<()> { +async fn replace_git_platforms_in( + tx: &mut AnyTx<'_>, + kind: BackendKind, + platforms: &[GitPlatformConfig], + key: &[u8; 32], +) -> Result<()> { let now = encode_ts(&Utc::now()); - ::sqlx::query("DELETE FROM git_platforms") + let delete = adapt_sql(kind, "DELETE FROM git_platforms"); + ::sqlx::query(&delete) .execute(&mut **tx) .await .context("clear git_platforms")?; + let insert = adapt_sql( + kind, + "INSERT INTO git_platforms (id, name, type, base_url, internal_base_url, token, \ + webhook_secret, webhook_signing_secret, enabled, raw, updated_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ); for platform in platforms { let row = rows::git_platform_to_row(platform, uuid::Uuid::new_v4().to_string(), now.clone(), key)?; - ::sqlx::query( - "INSERT INTO git_platforms (id, name, type, base_url, internal_base_url, token, \ - webhook_secret, webhook_signing_secret, enabled, raw, updated_at) \ - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - ) - .bind(&row.id) - .bind(&row.name) - .bind(&row.platform_type) - .bind(&row.base_url) - .bind(&row.internal_base_url) - .bind(&row.token) - .bind(&row.webhook_secret) - .bind(&row.webhook_signing_secret) - .bind(i64::from(row.enabled)) - .bind(&row.raw) - .bind(&row.updated_at) - .execute(&mut **tx) - .await - .with_context(|| format!("insert git_platform {:?}", platform.name))?; + ::sqlx::query(&insert) + .bind(&row.id) + .bind(&row.name) + .bind(&row.platform_type) + .bind(&row.base_url) + .bind(&row.internal_base_url) + .bind(&row.token) + .bind(&row.webhook_secret) + .bind(&row.webhook_signing_secret) + .bind(i64::from(row.enabled)) + .bind(&row.raw) + .bind(&row.updated_at) + .execute(&mut **tx) + .await + .with_context(|| format!("insert git_platform {:?}", platform.name))?; } Ok(()) } /// DELETE + re-INSERT the whole llm_providers set inside `tx`. -async fn replace_llm_providers_in(tx: &mut AnyTx<'_>, providers: &[LLMConfig], key: &[u8; 32]) -> Result<()> { +async fn replace_llm_providers_in( + tx: &mut AnyTx<'_>, + kind: BackendKind, + providers: &[LLMConfig], + key: &[u8; 32], +) -> Result<()> { let now = encode_ts(&Utc::now()); - ::sqlx::query("DELETE FROM llm_providers") + let delete = adapt_sql(kind, "DELETE FROM llm_providers"); + ::sqlx::query(&delete) .execute(&mut **tx) .await .context("clear llm_providers")?; + let insert = adapt_sql( + kind, + "INSERT INTO llm_providers (id, provider, model, api_base, api_key, max_tokens, \ + temperature, raw, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ); for (position, config) in providers.iter().enumerate() { let row = rows::llm_to_row(config, position, uuid::Uuid::new_v4().to_string(), now.clone(), key)?; - ::sqlx::query( - "INSERT INTO llm_providers (id, provider, model, api_base, api_key, max_tokens, \ - temperature, raw, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", - ) - .bind(&row.id) - .bind(&row.provider) - .bind(&row.model) - .bind(&row.api_base) - .bind(&row.api_key) - .bind(row.max_tokens) - .bind(row.temperature) - .bind(&row.raw) - .bind(&row.updated_at) - .execute(&mut **tx) - .await - .with_context(|| format!("insert llm_provider {:?}", config.provider))?; + ::sqlx::query(&insert) + .bind(&row.id) + .bind(&row.provider) + .bind(&row.model) + .bind(&row.api_base) + .bind(&row.api_key) + .bind(row.max_tokens) + .bind(row.temperature) + .bind(&row.raw) + .bind(&row.updated_at) + .execute(&mut **tx) + .await + .with_context(|| format!("insert llm_provider {:?}", config.provider))?; } Ok(()) } /// Upsert one app_settings row inside `tx`. Syntax is shared by PG and /// SQLite (≥3.24); no RETURNING. -async fn upsert_setting_in(tx: &mut AnyTx<'_>, key: &str, value: &serde_json::Value) -> Result<()> { - ::sqlx::query( +async fn upsert_setting_in(tx: &mut AnyTx<'_>, kind: BackendKind, key: &str, value: &serde_json::Value) -> Result<()> { + let upsert = adapt_sql( + kind, "INSERT INTO app_settings (key, value, updated_at) VALUES (?, ?, ?) \ ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at", - ) - .bind(key) - .bind(value.to_string()) - .bind(encode_ts(&Utc::now())) - .execute(&mut **tx) - .await - .with_context(|| format!("failed to save app_setting {key:?}"))?; + ); + ::sqlx::query(&upsert) + .bind(key) + .bind(value.to_string()) + .bind(encode_ts(&Utc::now())) + .execute(&mut **tx) + .await + .with_context(|| format!("failed to save app_setting {key:?}"))?; Ok(()) } -async fn delete_setting_in(tx: &mut AnyTx<'_>, key: &str) -> Result<()> { - ::sqlx::query("DELETE FROM app_settings WHERE key = ?") +async fn delete_setting_in(tx: &mut AnyTx<'_>, kind: BackendKind, key: &str) -> Result<()> { + let delete = adapt_sql(kind, "DELETE FROM app_settings WHERE key = ?"); + ::sqlx::query(&delete) .bind(key) .execute(&mut **tx) .await @@ -113,6 +135,10 @@ async fn delete_setting_in(tx: &mut AnyTx<'_>, key: &str) -> Result<()> { #[async_trait] impl ConfigStore for SqlxStore { async fn load_git_platforms(&self) -> Result> { + let sql = self.sql( + "SELECT id, name, type, base_url, internal_base_url, token, webhook_secret, \ + webhook_signing_secret, enabled, raw, updated_at FROM git_platforms ORDER BY name", + ); let rows = ::sqlx::query_as::< _, ( @@ -128,10 +154,7 @@ impl ConfigStore for SqlxStore { String, String, ), - >( - "SELECT id, name, type, base_url, internal_base_url, token, webhook_secret, \ - webhook_signing_secret, enabled, raw, updated_at FROM git_platforms ORDER BY name", - ) + >(&sql) .fetch_all(self.pool()) .await .context("failed to load git_platforms")?; @@ -175,19 +198,20 @@ impl ConfigStore for SqlxStore { async fn replace_git_platforms(&self, platforms: &[GitPlatformConfig]) -> Result<()> { let mut tx = self.pool().begin().await.context("begin replace_git_platforms")?; - replace_git_platforms_in(&mut tx, platforms, &self.key).await?; + replace_git_platforms_in(&mut tx, self.kind, platforms, &self.key).await?; tx.commit().await.context("commit replace_git_platforms")?; Ok(()) } async fn load_llm_providers(&self) -> Result> { - let rows = ::sqlx::query_as::<_, (String, String, String, String, String, i64, f64, String, String)>( + let sql = self.sql( "SELECT id, provider, model, api_base, api_key, max_tokens, temperature, raw, \ updated_at FROM llm_providers ORDER BY provider", - ) - .fetch_all(self.pool()) - .await - .context("failed to load llm_providers")?; + ); + let rows = ::sqlx::query_as::<_, (String, String, String, String, String, i64, f64, String, String)>(&sql) + .fetch_all(self.pool()) + .await + .context("failed to load llm_providers")?; let mut rows: Vec = rows .into_iter() .map( @@ -214,7 +238,7 @@ impl ConfigStore for SqlxStore { async fn replace_llm_providers(&self, providers: &[LLMConfig]) -> Result<()> { let mut tx = self.pool().begin().await.context("begin replace_llm_providers")?; - replace_llm_providers_in(&mut tx, providers, &self.key).await?; + replace_llm_providers_in(&mut tx, self.kind, providers, &self.key).await?; tx.commit().await.context("commit replace_llm_providers")?; Ok(()) } @@ -232,7 +256,8 @@ impl ConfigStore for SqlxStore { } async fn load_setting(&self, key: &str) -> Result> { - let raw: Option = ::sqlx::query_scalar("SELECT value FROM app_settings WHERE key = ?") + let sql = self.sql("SELECT value FROM app_settings WHERE key = ?"); + let raw: Option = ::sqlx::query_scalar(&sql) .bind(key) .fetch_optional(self.pool()) .await @@ -243,41 +268,42 @@ impl ConfigStore for SqlxStore { async fn save_setting(&self, key: &str, value: &serde_json::Value) -> Result<()> { let mut tx = self.pool().begin().await.context("begin save_setting")?; - upsert_setting_in(&mut tx, key, value).await?; + upsert_setting_in(&mut tx, self.kind, key, value).await?; tx.commit().await.context("commit save_setting")?; Ok(()) } async fn save_ui_state(&self, state: &UiStateFile) -> Result<()> { let mut tx = self.pool().begin().await.context("begin save_ui_state")?; - replace_git_platforms_in(&mut tx, &state.git_platforms, &self.key).await?; - replace_llm_providers_in(&mut tx, &state.llm, &self.key).await?; + replace_git_platforms_in(&mut tx, self.kind, &state.git_platforms, &self.key).await?; + replace_llm_providers_in(&mut tx, self.kind, &state.llm, &self.key).await?; let gitlab = &state.gitlab; if gitlab.token.is_empty() && gitlab.webhook_secret.is_empty() && gitlab.webhook_signing_secret.is_empty() { // Unset is unset: an all-empty legacy gitlab value removes the row // instead of storing an empty JSON shell. - delete_setting_in(&mut tx, LEGACY_GITLAB_KEY).await?; + delete_setting_in(&mut tx, self.kind, LEGACY_GITLAB_KEY).await?; } else { let value = rows::legacy_gitlab_to_value(gitlab, &self.key)?; - upsert_setting_in(&mut tx, LEGACY_GITLAB_KEY, &value).await?; + upsert_setting_in(&mut tx, self.kind, LEGACY_GITLAB_KEY, &value).await?; } if let Some(ui) = &state.ui { let value = serde_json::to_value(ui).context("serialize ui projection")?; - upsert_setting_in(&mut tx, UI_KEY, &value).await?; + upsert_setting_in(&mut tx, self.kind, UI_KEY, &value).await?; } tx.commit().await.context("commit save_ui_state")?; Ok(()) } async fn config_tables_empty(&self) -> Result { - let (gp, lp, st): (i64, i64, i64) = ::sqlx::query_as( + let sql = self.sql( "SELECT (SELECT COUNT(*) FROM git_platforms), \ (SELECT COUNT(*) FROM llm_providers), \ (SELECT COUNT(*) FROM app_settings)", - ) - .fetch_one(self.pool()) - .await - .context("failed to count config tables")?; + ); + let (gp, lp, st): (i64, i64, i64) = ::sqlx::query_as(&sql) + .fetch_one(self.pool()) + .await + .context("failed to count config tables")?; Ok(gp == 0 && lp == 0 && st == 0) } } @@ -297,31 +323,33 @@ fn warn_missing_row(op: &str, task_id: &uuid::Uuid, rows_affected: u64) { impl ReviewStore for SqlxStore { async fn create(&self, entry: &TaskEntry) -> Result<()> { let row = rows::task_entry_to_row(entry)?; - ::sqlx::query( + let sql = self.sql( "INSERT INTO reviews (task_id, state, source_meta, project, repository, request, \ result, error, progress, created_at, started_at, completed_at) \ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - ) - .bind(&row.task_id) - .bind(&row.state) - .bind(&row.source_meta) - .bind(&row.project) - .bind(&row.repository) - .bind(&row.request) - .bind(&row.result) - .bind(&row.error) - .bind(row.progress) - .bind(&row.created_at) - .bind(&row.started_at) - .bind(&row.completed_at) - .execute(self.pool()) - .await - .with_context(|| format!("insert review {}", row.task_id))?; + ); + ::sqlx::query(&sql) + .bind(&row.task_id) + .bind(&row.state) + .bind(&row.source_meta) + .bind(&row.project) + .bind(&row.repository) + .bind(&row.request) + .bind(&row.result) + .bind(&row.error) + .bind(row.progress) + .bind(&row.created_at) + .bind(&row.started_at) + .bind(&row.completed_at) + .execute(self.pool()) + .await + .with_context(|| format!("insert review {}", row.task_id))?; Ok(()) } async fn mark_started(&self, task_id: uuid::Uuid, started_at: DateTime) -> Result<()> { - let res = ::sqlx::query("UPDATE reviews SET state = 'running', started_at = ? WHERE task_id = ?") + let sql = self.sql("UPDATE reviews SET state = 'running', started_at = ? WHERE task_id = ?"); + let res = ::sqlx::query(&sql) .bind(encode_ts(&started_at)) .bind(task_id.to_string()) .execute(self.pool()) @@ -332,7 +360,8 @@ impl ReviewStore for SqlxStore { } async fn fill_source_meta(&self, task_id: uuid::Uuid, meta: &SourceMeta) -> Result<()> { - let res = ::sqlx::query("UPDATE reviews SET source_meta = ?, project = ?, repository = ? WHERE task_id = ?") + let sql = self.sql("UPDATE reviews SET source_meta = ?, project = ?, repository = ? WHERE task_id = ?"); + let res = ::sqlx::query(&sql) .bind(rows::encode_source_meta(meta)?) .bind(&meta.project) .bind(&meta.repository) @@ -348,23 +377,25 @@ impl ReviewStore for SqlxStore { let row = rows::task_entry_to_row(entry)?; let report_created_at = row.completed_at.clone().unwrap_or_else(|| encode_ts(&Utc::now())); let mut tx = self.pool().begin().await.context("begin complete review")?; - let res = ::sqlx::query( + let update = self.sql( "UPDATE reviews SET state = ?, result = ?, error = ?, completed_at = ?, progress = ? \ WHERE task_id = ?", - ) - .bind(&row.state) - .bind(&row.result) - .bind(&row.error) - .bind(&row.completed_at) - .bind(row.progress) - .bind(&row.task_id) - .execute(&mut *tx) - .await - .with_context(|| format!("complete review {}", row.task_id))?; + ); + let res = ::sqlx::query(&update) + .bind(&row.state) + .bind(&row.result) + .bind(&row.error) + .bind(&row.completed_at) + .bind(row.progress) + .bind(&row.task_id) + .execute(&mut *tx) + .await + .with_context(|| format!("complete review {}", row.task_id))?; warn_missing_row("complete", &entry.task_id, res.rows_affected()); // Replace (not upsert) so a retried-then-completed task cannot hit // the (task_id, expert_name) PK with stale rows. - ::sqlx::query("DELETE FROM expert_reports WHERE task_id = ?") + let delete = self.sql("DELETE FROM expert_reports WHERE task_id = ?"); + ::sqlx::query(&delete) .bind(&row.task_id) .execute(&mut *tx) .await @@ -372,21 +403,22 @@ impl ReviewStore for SqlxStore { if let Some(result) = &entry.result { match rows::expert_report_rows(&entry.task_id, result, report_created_at) { Ok(report_rows) => { + let insert = self.sql( + "INSERT INTO expert_reports (task_id, expert_name, report, duration_ms, created_at) \ + VALUES (?, ?, ?, ?, ?)", + ); for report in &report_rows { - ::sqlx::query( - "INSERT INTO expert_reports (task_id, expert_name, report, duration_ms, created_at) \ - VALUES (?, ?, ?, ?, ?)", - ) - .bind(&report.task_id) - .bind(&report.expert_name) - .bind(&report.report) - .bind(report.duration_ms) - .bind(&report.created_at) - .execute(&mut *tx) - .await - .with_context(|| { - format!("insert expert_report {:?} for {}", report.expert_name, report.task_id) - })?; + ::sqlx::query(&insert) + .bind(&report.task_id) + .bind(&report.expert_name) + .bind(&report.report) + .bind(report.duration_ms) + .bind(&report.created_at) + .execute(&mut *tx) + .await + .with_context(|| { + format!("insert expert_report {:?} for {}", report.expert_name, report.task_id) + })?; } } // A result that is not a serialized ReviewOutput is not a @@ -405,7 +437,8 @@ impl ReviewStore for SqlxStore { } async fn mark_cancelled(&self, task_id: uuid::Uuid, completed_at: DateTime) -> Result<()> { - let res = ::sqlx::query("UPDATE reviews SET state = 'cancelled', completed_at = ? WHERE task_id = ?") + let sql = self.sql("UPDATE reviews SET state = 'cancelled', completed_at = ? WHERE task_id = ?"); + let res = ::sqlx::query(&sql) .bind(encode_ts(&completed_at)) .bind(task_id.to_string()) .execute(self.pool()) @@ -416,32 +449,34 @@ impl ReviewStore for SqlxStore { } async fn mark_retry(&self, task_id: uuid::Uuid) -> Result<()> { - let res = - ::sqlx::query("UPDATE reviews SET state = 'pending', error = NULL, completed_at = NULL WHERE task_id = ?") - .bind(task_id.to_string()) - .execute(self.pool()) - .await - .with_context(|| format!("mark review {task_id} retried"))?; + let sql = self.sql("UPDATE reviews SET state = 'pending', error = NULL, completed_at = NULL WHERE task_id = ?"); + let res = ::sqlx::query(&sql) + .bind(task_id.to_string()) + .execute(self.pool()) + .await + .with_context(|| format!("mark review {task_id} retried"))?; warn_missing_row("mark_retry", &task_id, res.rows_affected()); Ok(()) } async fn mark_interrupted(&self, now: DateTime) -> Result { - let res = ::sqlx::query( + let sql = self.sql( "UPDATE reviews SET state = 'failed', error = 'interrupted: server restarted', completed_at = ? \ WHERE state IN ('pending', 'running')", - ) - .bind(encode_ts(&now)) - .execute(self.pool()) - .await - .context("interrupted-task sweep failed")?; + ); + let res = ::sqlx::query(&sql) + .bind(encode_ts(&now)) + .execute(self.pool()) + .await + .context("interrupted-task sweep failed")?; Ok(res.rows_affected()) } async fn list_reviews(&self, query: &ReviewListQuery) -> Result<(Vec, u64)> { let (where_sql, binds) = review_where(query); - let count_sql = format!("SELECT COUNT(*) FROM reviews {where_sql}"); + let count_raw = format!("SELECT COUNT(*) FROM reviews {where_sql}"); + let count_sql = self.sql(&count_raw); let mut count_q = ::sqlx::query_scalar::<_, i64>(&count_sql); for value in &binds { count_q = count_q.bind(value); @@ -452,10 +487,11 @@ impl ReviewStore for SqlxStore { // and both timestamps), so the COUNT and the page SELECT share the // same positional parameter list; LIMIT/OFFSET trail as two more. let offset = query.page.saturating_sub(1).saturating_mul(query.per_page); - let list_sql = format!( + let list_raw = format!( "SELECT {} FROM reviews {where_sql} ORDER BY created_at DESC, task_id DESC LIMIT ? OFFSET ?", rows::REVIEW_COLUMNS ); + let list_sql = self.sql(&list_raw); let mut list_q = ::sqlx::query_as::<_, rows::ReviewRowTuple>(&list_sql); for value in &binds { list_q = list_q.bind(value); @@ -475,14 +511,13 @@ impl ReviewStore for SqlxStore { } async fn get_review(&self, task_id: uuid::Uuid) -> Result> { - let row = ::sqlx::query_as::<_, rows::ReviewRowTuple>(&format!( - "SELECT {} FROM reviews WHERE task_id = ?", - rows::REVIEW_COLUMNS - )) - .bind(task_id.to_string()) - .fetch_optional(self.pool()) - .await - .with_context(|| format!("load review {task_id}"))?; + let raw = format!("SELECT {} FROM reviews WHERE task_id = ?", rows::REVIEW_COLUMNS); + let sql = self.sql(&raw); + let row = ::sqlx::query_as::<_, rows::ReviewRowTuple>(&sql) + .bind(task_id.to_string()) + .fetch_optional(self.pool()) + .await + .with_context(|| format!("load review {task_id}"))?; row.map(|tuple| rows::review_from_row(tuple.into())) .transpose() .with_context(|| format!("decode review row {task_id}")) @@ -496,22 +531,23 @@ impl ReviewStore for SqlxStore { content_hash: &str, token_estimate: i64, ) -> Result<()> { - ::sqlx::query( + let sql = self.sql( "INSERT INTO review_contexts (task_id, kind, content, content_hash, token_estimate, created_at) \ VALUES (?, ?, ?, ?, ?, ?) \ ON CONFLICT (task_id, kind) DO UPDATE SET \ content = excluded.content, content_hash = excluded.content_hash, \ token_estimate = excluded.token_estimate", - ) - .bind(task_id.to_string()) - .bind(kind) - .bind(content) - .bind(content_hash) - .bind(token_estimate) - .bind(encode_ts(&Utc::now())) - .execute(self.pool()) - .await - .with_context(|| format!("upsert review_context {kind} for {task_id}"))?; + ); + ::sqlx::query(&sql) + .bind(task_id.to_string()) + .bind(kind) + .bind(content) + .bind(content_hash) + .bind(token_estimate) + .bind(encode_ts(&Utc::now())) + .execute(self.pool()) + .await + .with_context(|| format!("upsert review_context {kind} for {task_id}"))?; Ok(()) } } @@ -522,43 +558,45 @@ impl ReviewStore for SqlxStore { impl DiscussionStore for SqlxStore { async fn upsert_note(&self, note: &DiscussionNote) -> Result<()> { let (mr_iid, note_id) = rows::discussion_ids(note)?; - ::sqlx::query( + let sql = self.sql( "INSERT INTO mr_discussions (platform, project, mr_iid, note_id, author, body, created_at, ingested_at) \ VALUES (?, ?, ?, ?, ?, ?, ?, ?) \ ON CONFLICT (platform, project, mr_iid, note_id) DO UPDATE SET \ body = excluded.body, author = excluded.author", - ) - .bind(¬e.platform) - .bind(¬e.project) - .bind(mr_iid) - .bind(note_id) - .bind(¬e.author) - .bind(¬e.body) - .bind(encode_ts(¬e.created_at)) - .bind(encode_ts(&Utc::now())) - .execute(self.pool()) - .await - .with_context(|| { - format!( - "upsert mr_discussion note {} for {} !{}", - note.note_id, note.project, note.mr_iid - ) - })?; + ); + ::sqlx::query(&sql) + .bind(¬e.platform) + .bind(¬e.project) + .bind(mr_iid) + .bind(note_id) + .bind(¬e.author) + .bind(¬e.body) + .bind(encode_ts(¬e.created_at)) + .bind(encode_ts(&Utc::now())) + .execute(self.pool()) + .await + .with_context(|| { + format!( + "upsert mr_discussion note {} for {} !{}", + note.note_id, note.project, note.mr_iid + ) + })?; Ok(()) } async fn list_notes(&self, platform: &str, project: &str, mr_iid: u64) -> Result> { let mr_iid = i64::try_from(mr_iid).with_context(|| format!("mr_iid out of range: {mr_iid}"))?; - let rows = ::sqlx::query_as::<_, rows::DiscussionRowTuple>( + let sql = self.sql( "SELECT platform, project, mr_iid, note_id, author, body, created_at FROM mr_discussions \ WHERE platform = ? AND project = ? AND mr_iid = ? ORDER BY created_at, note_id", - ) - .bind(platform) - .bind(project) - .bind(mr_iid) - .fetch_all(self.pool()) - .await - .with_context(|| format!("list mr_discussions for {project} !{mr_iid}"))?; + ); + let rows = ::sqlx::query_as::<_, rows::DiscussionRowTuple>(&sql) + .bind(platform) + .bind(project) + .bind(mr_iid) + .fetch_all(self.pool()) + .await + .with_context(|| format!("list mr_discussions for {project} !{mr_iid}"))?; rows.into_iter() .map(rows::discussion_from_row) .collect::>>() From a6853359079b027dce0c974f153a2a6e998b15d7 Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 15:00:03 +0800 Subject: [PATCH 14/36] fix(store): declare temperature as DOUBLE PRECISION and clamp negative durations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F1 (PG release blocker): llm_providers.temperature was REAL, which PG parses as float4, while the store binds/decodes f64 — read-back failed with 'mismatched types: f64 is not compatible with SQL type REAL'. The failure chain was silent: ui-state import succeeded (file renamed to .migrated), but the restart DB replay died on decrypting/decoding, leaving GET /config empty while /health stayed green. SQLite's REAL is 8 bytes, which is why unit tests and the SQLite E2E never saw it. 0.10.0 is unreleased with no databases to migrate, so the column simply becomes DOUBLE PRECISION (float8); SQLite maps that to the same 8-byte REAL affinity. Decode stays f64. A new ignored PG test (llm_providers_temperature_round_trip_on_postgres) gates the exact read path, and cleans up its enc: rows so shared scratch databases are not polluted with foreign-key ciphertext. migrations/0001_init.sql and design/persistence.md document why REAL is banned for float columns. No other float columns exist in the schema (audited: temperature is the only one). F2: TaskEntry::duration_ms wrapped to ~2^64 when a hand-seeded row has created_at later than completed_at (negative i64 cast to u64). All four millisecond-span projections (duration_ms, elapsed_ms, and the two inline SSE elapsed calculations) now go through one millis_between helper: saturating_sub on timestamps, clamped at 0. Unit test covers inverted spans and clock skew. Verified on postgres:16-alpine: both ignored store tests green; server boot imports ui-state.toml, restart replays from the DB, and GET /config is byte-identical across the restart (gitPlatforms/gitlab/llm intact, temperature 0.3); an inverted-timestamp review row reports durationMs 0. --- design/persistence.md | 2 +- migrations/0001_init.sql | 6 ++++- src/server/task_queue.rs | 55 +++++++++++++++++++++++++++++++++------- src/store/sqlx.rs | 48 +++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 11 deletions(-) diff --git a/design/persistence.md b/design/persistence.md index 3bb4c6a..0f51a8e 100644 --- a/design/persistence.md +++ b/design/persistence.md @@ -162,7 +162,7 @@ CREATE TABLE llm_providers ( api_base TEXT NOT NULL DEFAULT '', api_key TEXT NOT NULL DEFAULT '', -- enc: 加密（新增：0.9 明文落盘） max_tokens INTEGER NOT NULL DEFAULT 4096, - temperature REAL NOT NULL DEFAULT 0.7, + temperature DOUBLE PRECISION NOT NULL DEFAULT 0.7, -- 必须 float8：PG 的 REAL 是 float4，store 层按 f64 解码会 mismatched types（PG E2E 实测）；SQLite 的 REAL affinity 同为 8 字节，两端兼容 raw TEXT NOT NULL DEFAULT '{}', -- 扩展兜底：disable_thinking 等 updated_at TEXT NOT NULL ); diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql index 48ff07f..4d0bb06 100644 --- a/migrations/0001_init.sql +++ b/migrations/0001_init.sql @@ -93,7 +93,11 @@ CREATE TABLE llm_providers ( api_base TEXT NOT NULL DEFAULT '', api_key TEXT NOT NULL DEFAULT '', -- enc: 加密（新增：0.9 明文落盘） max_tokens INTEGER NOT NULL DEFAULT 4096, - temperature REAL NOT NULL DEFAULT 0.7, + -- 浮点列：必须 DOUBLE PRECISION（float8），不能用 REAL——PG 的 REAL 是 + -- float4，而 store 层按 f64 绑定/解码（Any 驱动类型精确匹配，无隐式 + -- 解码转换），REAL 列读回直接 mismatched types（岑静 PG E2E 实测）。 + -- SQLite 端 DOUBLE PRECISION 同样落 REAL affinity（8 字节），两端兼容。 + temperature DOUBLE PRECISION NOT NULL DEFAULT 0.7, raw TEXT NOT NULL DEFAULT '{}', -- 扩展兜底：disable_thinking 等 updated_at TEXT NOT NULL ); diff --git a/src/server/task_queue.rs b/src/server/task_queue.rs index 145945b..3a91e62 100644 --- a/src/server/task_queue.rs +++ b/src/server/task_queue.rs @@ -1,3 +1,4 @@ +use chrono::{DateTime, Utc}; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::RwLock; @@ -263,9 +264,7 @@ impl TaskStore { if let Some(entry) = self.inner.write().await.get_mut(&task_id) { entry.progress = Some(progress.min(100)); entry.expert_name = expert_name.clone(); - let elapsed = entry - .started_at - .map(|s| (chrono::Utc::now() - s).num_milliseconds() as u64); + let elapsed = entry.started_at.map(|s| millis_between(s, chrono::Utc::now())); let _ = self.tx.send(TaskEvent { task_id, status: "running", @@ -315,9 +314,7 @@ impl TaskStore { TaskState::Failed => "failed", TaskState::Cancelled => "cancelled", }; - let elapsed = entry - .started_at - .map(|s| (chrono::Utc::now() - s).num_milliseconds() as u64); + let elapsed = entry.started_at.map(|s| millis_between(s, chrono::Utc::now())); let _ = self.tx.send(TaskEvent { task_id, status, @@ -704,17 +701,23 @@ pub async fn record_task_outcome( } } +/// Milliseconds between two timestamps, clamped at 0. Projection-only +/// helper: hand-seeded rows (created_at later than completed_at) or clock +/// skew must not let a negative `i64` wrap to ~2^64 in the u64 projection. +fn millis_between(start: DateTime, end: DateTime) -> u64 { + end.timestamp_millis().saturating_sub(start.timestamp_millis()).max(0) as u64 +} + impl TaskEntry { pub fn duration_ms(&self) -> Option { match (self.created_at, self.completed_at) { - (start, Some(end)) => Some((end - start).num_milliseconds() as u64), + (start, Some(end)) => Some(millis_between(start, end)), _ => None, } } pub fn elapsed_ms(&self) -> Option { - self.started_at - .map(|s| (chrono::Utc::now() - s).num_milliseconds() as u64) + self.started_at.map(|s| millis_between(s, chrono::Utc::now())) } } @@ -750,6 +753,40 @@ mod tests { } } + /// F2 guard: hand-seeded rows with created_at later than completed_at + /// must clamp the u64 duration projection to 0 instead of wrapping to + /// ~2^64; elapsed_ms is clamped by the same helper. + #[test] + fn duration_ms_clamps_inverted_timestamps_to_zero() { + let base = chrono::DateTime::parse_from_rfc3339("2026-09-03T10:00:00Z") + .unwrap() + .with_timezone(&chrono::Utc); + let entry = TaskEntry { + task_id: Uuid::new_v4(), + state: TaskState::Completed, + created_at: base, + started_at: Some(base), + completed_at: Some(base - chrono::Duration::minutes(5)), + result: None, + error: None, + request: None, + source_meta: SourceMeta::default(), + progress: None, + expert_name: None, + }; + assert_eq!(entry.duration_ms(), Some(0), "inverted span must clamp, not wrap"); + + // started_at in the future (clock skew) must clamp, not wrap. + let mut skewed = entry.clone(); + skewed.started_at = Some(chrono::Utc::now() + chrono::Duration::minutes(5)); + assert_eq!(skewed.elapsed_ms(), Some(0)); + + // Sanity: a normal forward span still reports real milliseconds. + let mut ok = entry.clone(); + ok.completed_at = Some(base + chrono::Duration::milliseconds(1500)); + assert_eq!(ok.duration_ms(), Some(1500)); + } + #[tokio::test] async fn fill_source_meta_populates_blank_fields() { let store = TaskStore::new(); diff --git a/src/store/sqlx.rs b/src/store/sqlx.rs index 36bb079..0dbf71d 100644 --- a/src/store/sqlx.rs +++ b/src/store/sqlx.rs @@ -791,6 +791,54 @@ mod tests { assert_eq!(loaded[0].api_key, "plain-legacy-key"); } + /// F1 regression gate on real PG: `temperature` is DOUBLE PRECISION + /// (float8) and the store binds/decodes f64 — a float4 (REAL) column + /// fails the read-back with `mismatched types`, which silently emptied + /// GET /config after a restart. Requires `DATABASE_URL`: + /// `DATABASE_URL=postgres://... cargo test store -- --ignored` + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing at a scratch PostgreSQL"] + async fn llm_providers_temperature_round_trip_on_postgres() { + let url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set"); + let store = SqlxStore::connect(&url).await.unwrap(); + store.migrate().await.unwrap(); + + let providers = vec![ + LLMConfig { + provider: "pg-f1-openai".into(), + model: "gpt-5".into(), + api_key: "sk-f1".into(), + api_base: "https://api.openai.com/v1".into(), + max_tokens: 8192, + temperature: 0.3, + disable_thinking: None, + }, + LLMConfig { + provider: "pg-f1-deepseek".into(), + model: "deepseek-v4-flash".into(), + api_key: "ds-f1".into(), + api_base: "https://api.deepseek.com".into(), + max_tokens: 4096, + temperature: 0.7, + disable_thinking: Some(true), + }, + ]; + store.replace_llm_providers(&providers).await.unwrap(); + + // The exact read path that failed on float4: f64 decode of the + // temperature column. + let loaded = store.load_llm_providers().await.unwrap(); + assert_eq!(loaded.len(), 2); + assert!(llm_eq(&loaded[0], &providers[0]), "entry 0 mismatch: {loaded:?}"); + assert!(llm_eq(&loaded[1], &providers[1]), "entry 1 mismatch: {loaded:?}"); + + // Clean up: the enc: rows are keyed by the runner's local + // ~/.config secrets key and are undecryptable garbage for anyone + // else sharing this scratch database. + let cleanup = store.sql("DELETE FROM llm_providers WHERE provider IN ('pg-f1-openai', 'pg-f1-deepseek')"); + ::sqlx::query(&cleanup).execute(store.pool()).await.unwrap(); + } + #[tokio::test] async fn legacy_gitlab_round_trip_with_encrypted_fields() { let store = fresh_store().await; From c8c9cfb0907eeb1da1915336f2ecfbc5d849cf50 Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 15:25:40 +0800 Subject: [PATCH 15/36] fix(store): back-fill drifted review projections from materialized columns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.10.0 E2E-A 观察点 4:reviews 表的 project/repository 物化列有值、但 列表 API 响应 project:null。语义设计是「物化列供过滤、投影走 source_meta JSON」(design/persistence.md §5.2,写穿同步维护),两者一致 时无问题;但 fill_source_meta 的 UPDATE 失败只记日志不重试 (task_queue.rs),加上手工种子/遗留行不经过 codec,漂移行客观存在—— 于是出现「?project=X 过滤命中、行里 project 却显示空」的自相矛盾。 rows.rs review_from_row 之前把已 SELECT 出来的物化列直接丢弃。现在 decode source_meta 后,对空白(None/纯空白,口径同 fill_source_meta 的 is_blank)的 project/repository 用物化列回炉;JSON 非空值永远是主 源,列只做兜底,不臆造值(列也为空则保持 None)。 附实证(隔离 server + 手插漂移行):修复前过滤 total=2 而漂移行 project=null;修复后漂移行 project='grp/proj',与过滤语义一致;正常行 响应逐字段不变(status/project/repository/result.consolidated.assessment 全保留)。 测试: - store::rows::tests ×3 — 列回炉 / JSON 主源优先 / 不臆造空值(codec 层 钉住语义)。 - api::review::tests::list_reviews_db_projection_values_complete — 用真实 写穿路径(record_task_started → record_task_outcome,即 webhook 所用 helper)建行,断言 DB 路径列表项的 status/project/repository/branch/ author/duration/内嵌 assessment 全部带值,并与 0.9 内存路径逐值对齐 (既有 (a) 只比键集合,project:null 漂移能漏过)。 - api::review::tests::list_reviews_db_drifted_row_projects_materialized_column — API 层钉住漂移行:过滤命中什么,投影就显示什么。 注:status 在两层投影里都是非 Option 字符串(来自 reviews.state 列解 码),DB 路径不可能产出缺 status 的列表项;state 解码失败是整列 500 而非字段缺失。 --- src/server/api/review/tests.rs | 149 +++++++++++++++++++++++++++++++++ src/store/rows.rs | 87 ++++++++++++++++++- 2 files changed, 235 insertions(+), 1 deletion(-) diff --git a/src/server/api/review/tests.rs b/src/server/api/review/tests.rs index 44769e7..cbb66c6 100644 --- a/src/server/api/review/tests.rs +++ b/src/server/api/review/tests.rs @@ -1715,3 +1715,152 @@ async fn list_reviews_db_empty_and_out_of_range_pages() { ); assert_eq!(json["page"], 99); } + +/// A consolidated report carrying an overall assessment, so the list item's +/// embedded `result` exercises the score column (`consolidated.assessment`). +fn consolidated_with_score(score: u8) -> crate::team::lead_consolidator::ConsolidatedReport { + crate::team::lead_consolidator::ConsolidatedReport { + findings: Vec::new(), + low_confidence_removed: 0, + duplicates_merged: 0, + conflicts: Vec::new(), + assessment: crate::models::OverallAssessment { + score, + risk_level: crate::models::RiskLevel::Low, + lead_override: None, + tl_dr: "looks fine".to_string(), + unverified: false, + coverage_insufficient: false, + }, + consensus_reached: true, + total_files: 0, + reviewed_files: 0, + unreviewed_files: Vec::new(), + coverage: None, + adjudicated_removed: Vec::new(), + } +} + +/// (e) E2E-A 观察点 4 pin: with the DB as the data source, a row written by +/// the real write-through flow (record_task_started → record_task_outcome, +/// the exact helpers the GitLab webhook path uses) projects EVERY list field +/// at full value — status / project / repository / branch / author / +/// duration / embedded assessment — identically to the 0.9 in-memory path. +/// Test (a) only compared key SETS; a `project: null` drift passes a key-set +/// assertion, so this one compares values. +#[tokio::test] +async fn list_reviews_db_projection_values_complete() { + let (state, _db) = state_with_db().await; + let store = state.task_store.clone().unwrap(); + + let id = crate::server::task_queue::record_task_started(&store, source_meta_with_commit()).await; + let mut output = crate::models::ReviewOutput::new(vec![make_report( + "security", + vec![make_finding(crate::models::Severity::High)], + )]); + output.consolidated = Some(consolidated_with_score(87)); + let outcome: anyhow::Result = Ok(output); + crate::server::task_queue::record_task_outcome(&store, id, &outcome).await; + + let json = list_json(state.clone(), empty_params()).await; + assert_eq!(json["total"], 1); + let item = &json["items"][0]; + + // status decodes from the reviews.state column — a non-Option string in + // both projections, so it can never go missing from a served item. + assert_eq!(item["status"], "completed"); + assert_eq!(item["task_id"], id.to_string()); + assert_eq!(item["id"], id.to_string()); + + // Metadata projected from source_meta (both naming schemes). + assert_eq!(item["project"], "group/repo"); + assert_eq!(item["repository"], "group/repo"); + assert_eq!(item["branch"], "feature/x"); + assert_eq!(item["target_branch"], "main"); + assert_eq!(item["targetBranch"], "main"); + assert_eq!(item["mr_title"], "Fix login bug"); + assert_eq!(item["mrTitle"], "Fix login bug"); + assert_eq!(item["author_name"], "alice"); + assert_eq!(item["author"]["name"], "alice"); + assert_eq!(item["author"]["avatarUrl"], "http://avatar"); + assert_eq!(item["gitlab_mr_url"], "http://gitlab/mr/1"); + assert_eq!(item["gitlabMrUrl"], "http://gitlab/mr/1"); + assert_eq!(item["commit_sha"], "abc123"); + + // Wall-clock duration: present, non-negative, identical in both namings. + assert!( + item["duration_ms"].as_u64().is_some(), + "completed row must carry duration_ms" + ); + assert_eq!(item["durationMs"], item["duration_ms"]); + assert!(item["created_at"].as_str().is_some() && item["createdAt"].as_str().is_some()); + + // The score column's data source: the embedded ReviewOutput keeps its + // consolidated assessment through the DB round-trip. + assert_eq!(item["result"]["consolidated"]["assessment"]["score"], 87); + assert_eq!(item["result"]["reports"][0]["expert_name"], "security"); + + // Value parity with the 0.9 in-memory path for the same logical task. + let mem_state = state_with_store(); + let mem_store = mem_state.task_store.clone().unwrap(); + let mid = crate::server::task_queue::record_task_started(&mem_store, source_meta_with_commit()).await; + let mut mem_output = crate::models::ReviewOutput::new(vec![make_report( + "security", + vec![make_finding(crate::models::Severity::High)], + )]); + mem_output.consolidated = Some(consolidated_with_score(87)); + let mem_outcome: anyhow::Result = Ok(mem_output); + crate::server::task_queue::record_task_outcome(&mem_store, mid, &mem_outcome).await; + let mem_json = list_json(mem_state, empty_params()).await; + let mem_item = &mem_json["items"][0]; + for key in [ + "status", + "project", + "repository", + "branch", + "mr_title", + "mrTitle", + "author_name", + ] { + assert_eq!(item[key], mem_item[key], "DB vs memory mismatch on {key}"); + } + assert_eq!( + item["result"]["consolidated"]["assessment"]["score"], + mem_item["result"]["consolidated"]["assessment"]["score"], + ); +} + +/// (f) Drifted row — materialized filter column set but the source_meta JSON +/// blank (a failed `fill_source_meta` UPDATE is only logged, never retried; +/// hand-seeded/legacy rows bypass the codec): the row matches `?project=X` +/// via the column, and the projection must show the same X instead of null. +/// source_meta stays the primary source — the column only back-fills blanks +/// (codec-level pinning lives in store::rows::tests). +#[tokio::test] +async fn list_reviews_db_drifted_row_projects_materialized_column() { + let (state, db) = state_with_db().await; + let id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO reviews (task_id, state, source_meta, project, repository, created_at, completed_at) \ + VALUES (?, 'completed', '{}', 'grp/proj', 'grp/proj', \ + '2026-09-03T10:00:00.000000Z', '2026-09-03T10:05:00.000000Z')", + ) + .bind(id.to_string()) + .execute(db.pool()) + .await + .unwrap(); + + let mut params = empty_params(); + params.project = Some("grp/proj".to_string()); + let json = list_json(state, params).await; + assert_eq!(json["total"], 1, "the materialized column drives the filter"); + let item = &json["items"][0]; + assert_eq!(item["task_id"], id.to_string()); + assert_eq!(item["status"], "completed"); + assert_eq!( + item["project"], "grp/proj", + "a row that matches ?project=X must not display a blank project" + ); + assert_eq!(item["repository"], "grp/proj"); + assert_eq!(item["duration_ms"], 5 * 60 * 1000); +} diff --git a/src/store/rows.rs b/src/store/rows.rs index ba73e7d..1edab91 100644 --- a/src/store/rows.rs +++ b/src/store/rows.rs @@ -308,8 +308,26 @@ impl From for ReviewRow { } } +/// `TaskStore::fill_source_meta`'s blank definition: `None` or +/// whitespace-only. A blank projection field may take the materialized +/// column's value; a non-blank `source_meta` value always wins. +fn backfill_from_column(field: &mut Option, column: Option) { + let blank = field.as_deref().map(str::trim).unwrap_or_default().is_empty(); + if blank { + *field = column.filter(|v| !v.trim().is_empty()); + } +} + /// Decode a `reviews` row back into a [`TaskEntry`]. Used by the history /// read path (`ReviewStore::list_reviews` / `get_review`, §8.1) and by tests. +/// +/// Projection reads `source_meta` (the full metadata JSON); the materialized +/// `project`/`repository` columns exist for indexed filtering (§5.2 keeps +/// them in sync on every write). If a row has nevertheless drifted — a +/// failed `fill_source_meta` UPDATE is only logged, never retried, and +/// hand-seeded/legacy rows bypass the codec — the column is the last copy +/// of the value, so blank JSON fields are back-filled from it: a row that +/// matches `?project=X` must never display a blank project. pub(crate) fn review_from_row(row: ReviewRow) -> Result { fn opt_ts(raw: Option, what: &str) -> Result>> { raw.as_deref() @@ -317,6 +335,9 @@ pub(crate) fn review_from_row(row: ReviewRow) -> Result { .transpose() } let state = task_state_from_str(&row.state)?; + let mut source_meta = decode_source_meta(&row.source_meta)?; + backfill_from_column(&mut source_meta.project, row.project); + backfill_from_column(&mut source_meta.repository, row.repository); Ok(TaskEntry { task_id: Uuid::parse_str(&row.task_id) .with_context(|| format!("reviews.task_id is not a UUID: {:?}", row.task_id))?, @@ -333,7 +354,7 @@ pub(crate) fn review_from_row(row: ReviewRow) -> Result { .request .map(|s| serde_json::from_str(&s).context("reviews.request holds invalid JSON")) .transpose()?, - source_meta: decode_source_meta(&row.source_meta)?, + source_meta, progress: row .progress .map(|p| u8::try_from(p).with_context(|| format!("reviews.progress out of range: {p}"))) @@ -410,3 +431,67 @@ pub(crate) fn discussion_from_row( created_at: decode_ts(&created_at).context("mr_discussions.created_at")?, }) } + +// ─── tests ─── + +#[cfg(test)] +mod tests { + use super::*; + + /// Minimal decodable completed row; individual fields are overridden per + /// test. + fn review_row(source_meta: &str, project: Option<&str>, repository: Option<&str>) -> ReviewRow { + ReviewRow { + task_id: Uuid::new_v4().to_string(), + state: "completed".to_string(), + source_meta: source_meta.to_string(), + project: project.map(str::to_string), + repository: repository.map(str::to_string), + request: None, + result: None, + error: None, + progress: Some(100), + created_at: "2026-09-03T01:00:00.000000Z".to_string(), + started_at: Some("2026-09-03T01:00:01.000000Z".to_string()), + completed_at: Some("2026-09-03T01:00:42.000000Z".to_string()), + } + } + + /// §8.1 projection semantics (E2E-A 观察点 4): the materialized + /// project/repository columns exist for filtering, `source_meta` is the + /// projection source — but a drifted row (column set, JSON blank) must + /// not lose the value the filter matched on. + #[test] + fn review_from_row_backfills_blank_meta_from_materialized_columns() { + let entry = review_from_row(review_row("{}", Some("grp/proj"), Some("grp/proj"))).unwrap(); + assert_eq!(entry.source_meta.project.as_deref(), Some("grp/proj")); + assert_eq!(entry.source_meta.repository.as_deref(), Some("grp/proj")); + } + + /// A non-blank `source_meta` value is authoritative; the column is only + /// a fallback and never clobbers it. + #[test] + fn review_from_row_source_meta_wins_over_materialized_columns() { + let meta = r#"{"project":"json/wins","repository":"json-repo"}"#; + let entry = review_from_row(review_row(meta, Some("grp/proj"), Some("grp/proj"))).unwrap(); + assert_eq!(entry.source_meta.project.as_deref(), Some("json/wins")); + assert_eq!(entry.source_meta.repository.as_deref(), Some("json-repo")); + } + + /// Both sides blank stays absent (no empty-string fabrication), and a + /// whitespace-only JSON value counts as blank. + #[test] + fn review_from_row_backfill_never_fabricates_values() { + let entry = review_from_row(review_row("{}", None, None)).unwrap(); + assert!(entry.source_meta.project.is_none()); + assert!(entry.source_meta.repository.is_none()); + + let meta = r#"{"project":" "}"#; + let entry = review_from_row(review_row(meta, Some("grp/proj"), Some(""))).unwrap(); + assert_eq!(entry.source_meta.project.as_deref(), Some("grp/proj")); + assert!( + entry.source_meta.repository.is_none(), + "blank column must not back-fill" + ); + } +} From 7ccd2fd9dacc2823ecc5e779a27d13ab96a8cde7 Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 15:44:08 +0800 Subject: [PATCH 16/36] chore: bump v0.10.0 --- CHANGELOG.md | 20 ++++++++++++++++++++ Cargo.toml | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc8e567..7492b9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [0.10.0] - 2026-09-03 + +### Added +- **Persistent storage layer — PostgreSQL primary, embedded SQLite fallback**: a new `src/store/` module on sqlx 0.8's `Any` pool serves both backends from one code path; `DATABASE_URL` set → PostgreSQL, unset → embedded SQLite at `~/.config/review-engine/review.db`. The initial migration creates 7 tables (`reviews` / `expert_reports` / `mr_discussions` / `review_contexts` / `git_platforms` / `llm_providers` / `app_settings`), embedded via `sqlx::migrate!()` and applied at startup. Dialect rules (placeholders, JSON-as-TEXT, bool-as-INTEGER, RFC 3339 TEXT timestamps) are codified in `design/persistence.md` §3.1. (`src/store/`, `migrations/0001_init.sql`, `Cargo.toml`) +- **Review history survives restarts**: `TaskStore` now writes through to the DB, and with persistence active the History list/detail APIs read from the DB — history is no longer bounded by the 30-minute in-memory reaper window. A startup sweep flips rows still `pending`/`running` when the previous process died to `failed` with `error='interrupted: server restarted'`, so stale tasks never hang in a running state. `db=None` keeps the exact 0.9 in-memory path. (`src/server/task_queue.rs`, `src/server/api/review/handlers.rs`, `src/store/traits.rs`, `src/store/sqlx.rs`) +- **Configuration in the database**: git platforms and LLM provider configs move from `ui-state.toml` into the DB via a one-shot, single-transaction import on first boot (a mid-import failure rolls back cleanly and retries next startup); the file is renamed to `ui-state.toml.migrated` (kept, never deleted) once every table is written. All credentials — including LLM API keys, previously plaintext on disk — are stored `enc:`-encrypted (ChaCha20-Poly1305, same `secrets.key` boundary as 0.9.x). `PUT /api/v1/config` persists to the DB when attached. `REVIEW_DISABLE_DB=1` is the escape hatch restoring full 0.9 behaviour (in-memory + file); a `DATABASE_URL` pointing at an unreachable PostgreSQL is a hard startup error, never a silent SQLite fallback. (`src/server/api/config/persist.rs`, `src/store/`, `src/cli/app.rs`) +- **MR discussion context**: GitLab Note webhook payloads are ingested into `mr_discussions` in real time — idempotent upsert on `(platform, project, mr_iid, note_id)`, with a self-echo guard so our own published review reports are never re-ingested — and the pre-review flow additionally pulls discussions via the API and injects them into expert prompts, so follow-up reviews see prior human comments and review conclusions. (`src/server/gitlab/hooks.rs`, `src/server/api/review/discussion.rs`, `src/prompt/engine.rs`, `src/store/sqlx.rs`) +- **Storage backend visibility**: `GET /api/v1/system/health` gains `storage_backend` (`postgresql` / `sqlite` / `disabled`); the Configuration page Advanced card shows it as a permanently-disabled read-only row (hidden fail-silent on health-check failure or an older server). (`src/server/api/system.rs`, `frontend/src/views/Configuration.vue`, `frontend/src/services/health.ts`, `frontend/src/i18n/locales/*` ×6) +- **Review detail「完整评论」fallback**: when the aggregator produced no output (team reviews with `aggregated=None`), the full-comment tab falls back to the Lead consolidation TL;DR instead of rendering empty. (`src/server/api/review/task.rs`) + +### Fixed +- **PostgreSQL placeholder rewriting (E2E-found release blocker)**: sqlx's `Any` driver passes SQL through verbatim — it does NOT translate `?` placeholders to `$n`, so every bound-parameter statement failed on PG with `42601 syntax error`. The store layer now rewrites top-level `?` to `$1..$n` (correctly skipping `?` inside string literals, quoted identifiers, and comments) at a single choke point (`SqlxStore::sql` / `adapt_sql`); SQLite passes through unchanged. (`src/store/placeholders.rs`, `src/store/sqlx.rs`) +- **`llm_providers.temperature` declared `DOUBLE PRECISION`**: PG parses `REAL` as float4 while the store binds/decodes f64, so config read-back after restart failed with `mismatched types: f64 is not compatible with SQL type REAL` — silently leaving `GET /config` empty while `/health` stayed green (SQLite's 8-byte REAL is why tests never saw it). The column is now float8 on PG, same 8 bytes on SQLite. (`migrations/0001_init.sql`, `src/store/sqlx.rs`) +- **`durationMs` wrap-around guard**: inverted timestamps (completed before started) now clamp to 0 instead of wrapping the u64 duration. (`src/server/task_queue.rs`) +- **History list projection self-consistency**: rows whose materialized `project`/`repository` columns drifted from `source_meta` (failed back-fill, hand-seeded or legacy rows) are re-filled from the columns at read time, so a row matched by a `?project=X` filter no longer displays `project: null`. (`src/store/rows.rs`) + +### Known issues +- GitLab 19.x system hooks do not deliver MR note events (even with `note_events=true`): real-time comment ingestion requires a project-level webhook; under system-hook-only deployments the pre-review API pull covers the gap. +- Deferred to 0.10.x: config-directory isolation is incomplete, the legacy `webhookSecret` masking policy needs alignment, and `/system/health` has no deep DB probe yet. + ## [0.9.50] - 2026-09-02 ### Added diff --git a/Cargo.toml b/Cargo.toml index de5fb27..39d760a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "review-engine" -version = "0.9.50" +version = "0.10.0" license = "Apache-2.0" edition = "2021" From 37e1d166b78e8930bed1ba2f8f77b9f2337d72cd Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 16:45:10 +0800 Subject: [PATCH 17/36] fix(server): SPA history-mode fallback and diagnosable static-dir miss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit E2E 实测两个静态服务缺陷: 1. 前端 SPA 由裸 ServeDir serve,没有 not-found 回退——直接打开或刷新 /history、/config 等 history-mode 子路由返回 ServeDir 的裸 404;从根 路径进站再客户端跳转则正常,所以只有深链接/刷新中招。 2. static_dir() 按 CWD 找 ./frontend/dist,从非仓库根目录启动时静默 退化成 "Dashboard coming soon" 占位页,无任何日志,排查全靠猜。 修复: - ServeDir 挂上 axum 回退 handler(get + 闭包):文件不存在时,对「非 /api/ 前缀、且末段不含扩展名」的 GET 路径重新读盘 serve index.html (200 + no-cache, must-revalidate,与 / 的缓存契约一致——它就是 index.html)。不缓存到内存:原地升级会在 server 运行期间替换 dist, 内存副本会引用已消失的 hashed chunk(正是 cache-control 修复过的 白屏缺陷)。 - 用 ServeDir::fallback 而非 not_found_service:后者用 SetStatus 把 回退响应强制改写为 404,深链接会拿到「200 的 body + 404 的 status」 (实测复现)。handler 自己按路径裁决:深链接 200,/api/ 与带扩展名 的文件请求显式 404——未匹配的 API 路由和缺失 hashed asset 的 404 契约均不变(HTML 200 会把部署事故藏成白屏)。 - static_dir() 两个候选路径都不存在时打 WARN,带当前 CWD 和占位页 后果说明;回退到占位页的逻辑本身不变。 测试: - 单测(router.rs mod spa_deep_link):决策函数三组——客户端路由回退、 /api/ 保持 404、带扩展名文件保持 404。 - 集成测试(tests/server/frontend.rs):tempdir 假 dist + 真实 spawn server,GET /history、/config、/reviews/42 均 200 且 body 为 index.html、带 no-cache;GET /api/v1/definitely-not-a-route 与 /assets/missing-00000000.js 保持 404。 - cargo test 全绿:lib 1502 passed,各 test target 58/31/47/4 passed,0 failed;cargo fmt --check 干净。 --- src/server/router.rs | 90 +++++++++++++++++++++++++++++++++++++++- tests/server/frontend.rs | 76 +++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 2 deletions(-) diff --git a/src/server/router.rs b/src/server/router.rs index 09fae76..31345fe 100644 --- a/src/server/router.rs +++ b/src/server/router.rs @@ -8,7 +8,7 @@ use axum::{ extract::Request, http::{header::CACHE_CONTROL, HeaderMap, HeaderValue, StatusCode}, middleware::{self, Next}, - response::{Html, Response}, + response::{Html, IntoResponse, Response}, routing::{get, post}, Router, }; @@ -66,15 +66,46 @@ async fn static_cache_control(request: Request, next: Next) -> Response { } /// Detect the frontend static assets directory (Docker or local dev). +/// +/// `./frontend/dist` is resolved against the process working directory, so +/// starting the binary outside the repository root silently degrades to the +/// placeholder page — log the CWD to make that diagnosable. fn static_dir() -> Option { for path in ["/app/frontend/dist", "./frontend/dist"] { if std::path::Path::new(path).is_dir() { return Some(path.to_string()); } } + tracing::warn!( + "frontend dist not found (checked /app/frontend/dist, ./frontend/dist; cwd = {}); \ + serving the placeholder page until a build is available", + std::env::current_dir() + .map(|p| p.display().to_string()) + .unwrap_or_else(|_| "".to_string()) + ); None } +/// Decide whether a missing static path is an SPA deep link that should fall +/// back to `index.html` (history-mode routing), as opposed to a genuinely +/// absent resource that must keep its 404. +/// +/// - `/api/…` — never: an unmatched API route must stay a 404 JSON problem, +/// not an HTML page. +/// - paths whose last segment contains a `.` (e.g. `/assets/app-XXXX.js`, +/// `/favicon.svg`) — never: these are file requests; serving HTML with 200 +/// for a missing asset would mask deploy breakage (and an immutable-cached +/// 200 keeps it broken in browsers). +/// - everything else (`/history`, `/config`, `/reviews/42`, …) — the SPA +/// router owns it client-side, so serve the entry point. +fn is_spa_deep_link(path: &str) -> bool { + if path.starts_with("/api/") { + return false; + } + let last_segment = path.rsplit('/').next().unwrap_or(path); + !last_segment.contains('.') +} + /// Build the complete Axum application router. /// /// Always mounts health, metrics, progress, and `/api/v1` routes. @@ -90,8 +121,33 @@ pub fn build(state: Arc, auth: Arc, webhook_handlers: Vec< // what is registered at that point (fallback included), so API/health and // webhook routes below are unaffected. if let Some(dir) = static_dir() { + // SPA history-mode fallback: a hard refresh or direct entry on a + // client-side route (`/history`, `/config`, …) must serve the entry + // point, not ServeDir's bare 404. Missing files (has extension) and + // `/api/` paths keep their 404. The file is re-read per request (never + // held in memory): an in-place upgrade replaces dist while the server + // runs, and a stale copy would reference vanished hashed chunks. The + // response carries the same `no-cache, must-revalidate` policy as `/` + // — it IS index.html. + let index = std::path::PathBuf::from(&dir).join("index.html"); + let spa_fallback = get(move |req: Request| { + let index = index.clone(); + async move { + if !is_spa_deep_link(req.uri().path()) { + return StatusCode::NOT_FOUND.into_response(); + } + match tokio::fs::read(&index).await { + Ok(html) => ([(CACHE_CONTROL, "no-cache, must-revalidate")], Html(html)).into_response(), + Err(_) => StatusCode::NOT_FOUND.into_response(), + } + } + }); app = app - .fallback_service(ServeDir::new(dir)) + // `fallback`, not `not_found_service`: the latter force-overrides + // the fallback's status to 404 (SetStatus wrapper). The handler + // decides per request — 200 index.html for deep links, explicit + // 404 for `/api/` and file-like paths. + .fallback_service(ServeDir::new(dir).fallback(spa_fallback)) .layer(middleware::from_fn(static_cache_control)); } else { app = app.route("/", get(serve_frontend)); @@ -391,4 +447,34 @@ mod tests { } } } + + /// Decision function for the SPA history-mode fallback, exercised + /// end-to-end by `spa_deep_links_fall_back_to_index_html` in + /// tests/server/frontend.rs. + mod spa_deep_link { + use super::*; + + #[test] + fn client_side_routes_fall_back() { + for path in ["/history", "/config", "/reviews/42", "/settings/git-platforms"] { + assert!(is_spa_deep_link(path), "{path} must fall back to index.html"); + } + } + + #[test] + fn api_paths_stay_404() { + for path in ["/api/v1/definitely-not-a-route", "/api/v1/reviews/999"] { + assert!(!is_spa_deep_link(path), "{path} must keep its API 404"); + } + } + + #[test] + fn file_requests_stay_404() { + // A missing hashed asset must not be masked by an HTML 200 — that + // would hide deploy breakage behind a white screen. + for path in ["/assets/missing-00000000.js", "/favicon.svg", "/icons.svg"] { + assert!(!is_spa_deep_link(path), "{path} must keep its 404"); + } + } + } } diff --git a/tests/server/frontend.rs b/tests/server/frontend.rs index c24a8de..0e7ee51 100644 --- a/tests/server/frontend.rs +++ b/tests/server/frontend.rs @@ -166,3 +166,79 @@ async fn static_frontend_cache_control_headers() { "304 must carry the same Cache-Control as the 200, got {cc:?}" ); } + +// ─── SPA history-mode fallback (deep-link 404 defect) ──────────── + +/// Defect regression: the SPA uses history-mode routing, so directly opening +/// or refreshing a client-side route (`/history`, `/config`, …) must serve +/// `index.html` — ServeDir's bare 404 left deep links dead. Unmatched `/api/` +/// routes and missing files (extension in the last segment) must keep their +/// 404: serving HTML for a missing hashed asset would mask deploy breakage. +#[tokio::test] +async fn spa_deep_links_fall_back_to_index_html() { + let www = tempfile::tempdir().expect("failed to create www temp dir"); + write_fake_frontend_dist(www.path()); + + let port = find_free_port(); + let _guard = spawn_server_full(&bin_path(), port, None, &[], Some(www.path())); + wait_for_server(port).await; + + let client = reqwest::Client::new(); + let base = format!("http://127.0.0.1:{port}"); + + // Deep links — single segment and nested — serve the entry point with the + // same revalidation policy as `/`. + for path in ["/history", "/config", "/reviews/42"] { + let resp = client + .get(format!("{base}{path}")) + .send() + .await + .unwrap_or_else(|e| panic!("GET {path}: {e}")); + assert_eq!( + resp.status(), + reqwest::StatusCode::OK, + "GET {path} must serve index.html, got {}", + resp.status() + ); + let cc = resp + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()) + .map(str::to_owned) + .unwrap_or_default(); + assert!(cc.contains("no-cache"), "GET {path} must be no-cache, got {cc:?}"); + let body = resp.text().await.expect("body"); + assert!( + body.contains("fixture"), + "GET {path} must serve the fixture index.html, got {body:?}" + ); + } + + // An unmatched API route stays a 404 — it must never be answered with the + // SPA entry point. + let resp = client + .get(format!("{base}/api/v1/definitely-not-a-route")) + .send() + .await + .expect("GET unknown api route"); + assert_eq!( + resp.status(), + reqwest::StatusCode::NOT_FOUND, + "unknown /api/ route must stay 404, got {}", + resp.status() + ); + + // A missing file (hashed asset) stays a 404 — an HTML 200 here would hide + // a broken deploy behind a white screen. + let resp = client + .get(format!("{base}/assets/missing-00000000.js")) + .send() + .await + .expect("GET missing asset"); + assert_eq!( + resp.status(), + reqwest::StatusCode::NOT_FOUND, + "missing asset must stay 404, got {}", + resp.status() + ); +} From 678417795478aa5868e72b5b0f05019fff3c7f59 Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 16:47:42 +0800 Subject: [PATCH 18/36] docs(changelog): record SPA history-mode fallback and static-dir WARN under 0.10.0 Fixed --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7492b9f..ab37d7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - **`llm_providers.temperature` declared `DOUBLE PRECISION`**: PG parses `REAL` as float4 while the store binds/decodes f64, so config read-back after restart failed with `mismatched types: f64 is not compatible with SQL type REAL` — silently leaving `GET /config` empty while `/health` stayed green (SQLite's 8-byte REAL is why tests never saw it). The column is now float8 on PG, same 8 bytes on SQLite. (`migrations/0001_init.sql`, `src/store/sqlx.rs`) - **`durationMs` wrap-around guard**: inverted timestamps (completed before started) now clamp to 0 instead of wrapping the u64 duration. (`src/server/task_queue.rs`) - **History list projection self-consistency**: rows whose materialized `project`/`repository` columns drifted from `source_meta` (failed back-fill, hand-seeded or legacy rows) are re-filled from the columns at read time, so a row matched by a `?project=X` filter no longer displays `project: null`. (`src/store/rows.rs`) +- **SPA history-mode deep links no longer 404; static-dir miss is now diagnosable**: directly opening or refreshing a client-side route (`/history`, `/config`, `/reviews/42`) returned ServeDir's bare 404 — a `ServeDir::fallback` handler now serves `index.html` for extension-less non-`/api/` GET paths (200 + `no-cache, must-revalidate`, re-read from disk per request so an in-place upgrade never serves a stale copy), while unmatched `/api/` routes and file-like requests (extension in the last segment) keep their explicit 404 — serving HTML for a missing hashed asset would mask deploy breakage. Separately, `static_dir()` resolving `./frontend/dist` against the CWD silently degraded to the "Dashboard coming soon" placeholder when started outside the repository root; it now logs a WARN with the CWD and both checked paths. (`src/server/router.rs`, `tests/server/frontend.rs`) ### Known issues - GitLab 19.x system hooks do not deliver MR note events (even with `note_events=true`): real-time comment ingestion requires a project-level webhook; under system-hook-only deployments the pre-review API pull covers the gap. From 4c5fe1899e2cc162e5ff72eb2b01af531d4f1a39 Mon Sep 17 00:00:00 2001 From: isletspace Date: Thu, 3 Sep 2026 16:49:37 +0800 Subject: [PATCH 19/36] chore: sync Cargo.lock to v0.10.0 --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 84f0d74..fed4f10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2134,7 +2134,7 @@ dependencies = [ [[package]] name = "review-engine" -version = "0.9.50" +version = "0.10.0" dependencies = [ "anyhow", "async-trait", From 3054059be5ff66be4880b769c8a1c426afdda7da Mon Sep 17 00:00:00 2001 From: isletspace Date: Fri, 4 Sep 2026 13:16:44 +0800 Subject: [PATCH 20/36] build: add Dockerfile.dev for source-built local preview images Formalizes the previously untracked Dockerfile.local as a tracked contributor-facing build path: in-container Rust builder (macOS arm64 has no cross toolchain), COPY migrations for sqlx::migrate! (0.10.0), and APT_MIRROR build-arg instead of a hardcoded mirror. --- Dockerfile.dev | 120 ++++++++++++++++++++++++++++++++++++ Dockerfile.dev.dockerignore | 87 ++++++++++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 Dockerfile.dev create mode 100644 Dockerfile.dev.dockerignore diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 0000000..54276c2 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,120 @@ +# ═══════════════════════════════════════════════════════════════════════ +# Dockerfile.dev — local/dev preview image built from source +# For contributors (e.g. on macOS arm64) who want to build a Linux image +# from the current checkout and preview branch changes locally. +# +# Differences from the root Dockerfile (zero-build, downloads GitHub +# Release assets): +# - adds a builder stage: compiles this checkout inside the container +# with rust:1.96-bookworm (macOS arm64 has no cross toolchain, so the +# Linux binary must be produced by the in-container builder) +# - binary: COPY --from=builder (replaces the release-download RUN block) +# - frontend dist: COPY the locally built frontend/dist (contains the +# latest changes of this branch) +# The runtime stage is aligned section-by-section with the root Dockerfile +# to keep deployment semantics identical. +# Note: requires the companion Dockerfile.dev.dockerignore (the root +# .dockerignore excludes frontend/dist; without the override the COPY of +# the frontend artifacts would fail). +# ═══════════════════════════════════════════════════════════════════════ + +# ═══════════════════════════════════════════════════════════════════════ +# Stage 0: Builder (compile the Linux binary in-container) +# bookworm glibc 2.36, forward-compatible with the ubuntu 24.04 runtime +# (glibc 2.39). +# ═══════════════════════════════════════════════════════════════════════ +FROM rust:1.96-bookworm AS builder + +WORKDIR /build + +# Single crate, no build.rs, no workspace members: manifest + src/ suffice +COPY Cargo.toml Cargo.lock ./ +COPY src ./src +# Referenced by src/config/defaults.rs via include_str! — must exist at +# compile time (the root .dockerignore already exempts this file) +COPY docs/code-audit-default.toml ./docs/code-audit-default.toml +# Since v0.10.0, src/store/mod.rs uses sqlx::migrate!("./migrations") — +# must exist at compile time +COPY migrations ./migrations + +RUN cargo build --release --locked + +# ═══════════════════════════════════════════════════════════════════════ +# Stage 1: Runtime (aligned section-by-section with the root Dockerfile) +# ═══════════════════════════════════════════════════════════════════════ +FROM ubuntu:24.04 + +# Optional apt mirror for contributors with poor connectivity to the +# official Ubuntu archives (e.g. --build-arg APT_MIRROR=mirrors.aliyun.com). +# Empty (default) = official sources. When set, the archive/security hosts +# (x86_64) and the ports host (aarch64, ubuntu-ports) are rewritten in both +# the legacy sources.list and the deb822 ubuntu.sources — Ubuntu 24.04 +# keeps the real entries in the latter. +ARG APT_MIRROR="" +RUN if [ -n "${APT_MIRROR}" ]; then \ + echo ">> APT_MIRROR set: rewriting apt sources to ${APT_MIRROR}"; \ + for f in /etc/apt/sources.list /etc/apt/sources.list.d/ubuntu.sources; do \ + [ -f "$f" ] || continue; \ + sed -i -e "s|archive.ubuntu.com|${APT_MIRROR}|g" \ + -e "s|security.ubuntu.com|${APT_MIRROR}|g" \ + -e "s|ports.ubuntu.com|${APT_MIRROR}|g" "$f"; \ + done; \ + fi + +# Install runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + git \ + openssh-client \ + curl \ + tar \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +# Create non-root user (fixed UID/GID 9001, same as the root Dockerfile) +RUN groupadd -r -g 9001 review-engine && useradd -r -u 9001 -g review-engine -d /app -s /sbin/nologin review-engine + +WORKDIR /app + +# ── Local build artifact: binary (replaces the release-download RUN block) ── +COPY --from=builder /build/target/release/review-engine /usr/local/bin/review-engine +RUN /usr/local/bin/review-engine --version + +# ── Local build artifact: frontend dist (replaces the frontend-dist.tar.gz +# download RUN block) ── +# IMAGE_DIST=/app/frontend-dist-image in entrypoint.sh: on first start the +# contents are synced from here into the /app/frontend/dist volume; the +# image keeps a copy as the sync source. +COPY frontend/dist /app/frontend-dist-image +RUN ls -la /app/frontend-dist-image + +# reng alias (dynamic command name via argv[0]; a symlink is enough) +RUN ln -s /usr/local/bin/review-engine /usr/local/bin/reng + +# Copy the entrypoint script; kept as the container entrypoint for future +# extension +COPY entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh + +# Create config and report directories +RUN mkdir -p /app/config /app/reports /app/.ssh /app/bin /app/frontend/dist && \ + chown -R review-engine:review-engine /app + +# Switch to non-root user +USER review-engine + +# Expose ports +EXPOSE 443 8080 + +# Health check +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD curl -f http://localhost:8080/health || exit 1 + +# Default environment variables +ENV REVIEW_ENGINE_CONFIG_DIR=/app/config +ENV REVIEW_ENGINE_REPORT_DIR=/app/reports +ENV RUST_LOG=info + +# Entry: start the service +ENTRYPOINT ["/app/entrypoint.sh"] +CMD ["serve", "--bind", "0.0.0.0", "--port", "8080"] diff --git a/Dockerfile.dev.dockerignore b/Dockerfile.dev.dockerignore new file mode 100644 index 0000000..31ff88d --- /dev/null +++ b/Dockerfile.dev.dockerignore @@ -0,0 +1,87 @@ +# Companion to Dockerfile.dev: root .dockerignore minus the frontend/dist +# exclusion (the dev preview image COPYs the locally built frontend). +# migrations/ is not excluded here, satisfying the builder-stage +# `COPY migrations ./migrations` (sqlx::migrate! compile-time embed). +# Git ignore patterns for Docker build context +# This prevents unnecessary files from being copied into the Docker build + +# Rust build artifacts +target/ +**/*.rs.bk +Cargo.lock.bak + +# Git +.git/ +.gitignore +.github/ + +# IDE and editors +.idea/ +.vscode/ +*.swp +*.swo +*~ +.DS_Store + +# Documentation (not needed at runtime) +docs/*.md +!docs/code-audit-default.toml + +# Review reports and runtime data +reports/ +*.log +review-engine*.log + +# Docker files themselves (avoid recursion issues) +Dockerfile +docker-compose.yml +docker-compose.*.yml +.dockerignore +.env +.env.* +!.env.example + +# CI/CD +.github/workflows/ + +# Test data and artifacts +test_data/ +tmp/ +temp/ + +# Python bindings (not included in SaaS build) +python/ +*.py +*.pyc +__pycache__/ + +# Node.js (if any frontend exists) +# A bare `node_modules/` only matches the context root (BuildKit verified) and +# would not exclude the nested frontend/node_modules: after a local npm install +# it would be copied into the image by `COPY frontend/ ./`, clobbering the +# in-container linux-musl dependencies (@rolldown/binding-* platform mismatch +# breaks the build). `**/node_modules` covers both root and any depth; +# node_modules is a pure build artifact and must never enter the image. +**/node_modules +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Backup files +*.bak +*.backup +*.old + +# Local development configs +.code-audit-config.toml +.pr-agent.toml + +# Misc +*.md +!README.md +CHANGELOG.md +LICENSE + +# Plan and review artifacts +plan.md +review-feedback-*.md From 39c014915e90c41889763969fe03d1fe6fbece2b Mon Sep 17 00:00:00 2001 From: isletspace Date: Fri, 4 Sep 2026 13:16:53 +0800 Subject: [PATCH 21/36] docs: add 0.10 migration guide and major-upgrade declaration docs/migration-0.10.md covers backup, upgrade paths, the automatic first-boot migration chain, post-upgrade verification, and troubleshooting; states plainly that downgrading to 0.9.x is not supported. CHANGELOG 0.10.0 opens with the same one-way-upgrade declaration; README and docs index link to the guide. --- CHANGELOG.md | 2 + README.md | 3 + docs/README.md | 1 + docs/migration-0.10.md | 129 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 docs/migration-0.10.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ab37d7b..a246225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [0.10.0] - 2026-09-03 +> **0.10.x is a major upgrade.** The upgrade itself is fully automatic — the first 0.10.x boot creates the database, applies the embedded schema migrations, and imports `ui-state.toml` into the database in one transaction — but it is a **one-way move: downgrading back to 0.9.x is not supported** (configuration's authoritative source becomes the database and `ui-state.toml` is renamed to `ui-state.toml.migrated`, which a 0.9.x binary cannot read). **Back up your config directory** (`~/.config/review-engine/`, including `ui-state.toml` and `secrets.key`; Docker: the `./config` and `./auth` bind mounts) **before upgrading.** Step-by-step instructions, expected logs, verification, and troubleshooting: [`docs/migration-0.10.md`](docs/migration-0.10.md). + ### Added - **Persistent storage layer — PostgreSQL primary, embedded SQLite fallback**: a new `src/store/` module on sqlx 0.8's `Any` pool serves both backends from one code path; `DATABASE_URL` set → PostgreSQL, unset → embedded SQLite at `~/.config/review-engine/review.db`. The initial migration creates 7 tables (`reviews` / `expert_reports` / `mr_discussions` / `review_contexts` / `git_platforms` / `llm_providers` / `app_settings`), embedded via `sqlx::migrate!()` and applied at startup. Dialect rules (placeholders, JSON-as-TEXT, bool-as-INTEGER, RFC 3339 TEXT timestamps) are codified in `design/persistence.md` §3.1. (`src/store/`, `migrations/0001_init.sql`, `Cargo.toml`) - **Review history survives restarts**: `TaskStore` now writes through to the DB, and with persistence active the History list/detail APIs read from the DB — history is no longer bounded by the 30-minute in-memory reaper window. A startup sweep flips rows still `pending`/`running` when the previous process died to `failed` with `error='interrupted: server restarted'`, so stale tasks never hang in a running state. `db=None` keeps the exact 0.9 in-memory path. (`src/server/task_queue.rs`, `src/server/api/review/handlers.rs`, `src/store/traits.rs`, `src/store/sqlx.rs`) diff --git a/README.md b/README.md index 2545f73..dfa8906 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,9 @@ binary; pulling a newer image also works, the entrypoint syncs it to the volumes on next start), and plain binary installs are replaced atomically (backup + smoke test + rollback on failure). +> **Upgrading to 0.10.x?** It is a one-way, automatic migration — back up your +> config directory first: [`docs/migration-0.10.md`](docs/migration-0.10.md). + For a detailed walkthrough, see [`docs/getting-started.md`](docs/getting-started.md). For full CLI options, environment variables, LLM providers, and config reference, see [`docs/configuration.md`](docs/configuration.md), [`docs/integrations/`](docs/integrations/), and [`docs/rest-api.md`](docs/rest-api.md). diff --git a/docs/README.md b/docs/README.md index 81dcc6d..bccd929 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,6 +7,7 @@ This directory contains the user-facing documentation for review-engine. | Document | What you'll learn | |---|---| | [Getting Started](getting-started.md) | Install review-engine, configure an LLM provider, and run your first local or remote review. | +| [Migrating to 0.10](migration-0.10.md) | Upgrade to 0.10.x: backup, the automatic database migration, verification, and troubleshooting. | | [FAQ / Troubleshooting](faq.md) | API token setup, 401 errors, forgotten-token recovery, and container bind-volume pitfalls. | | [Configuration](configuration.md) | How config files are merged, command enablement, expert teams, validation, and the web Configuration UI. | diff --git a/docs/migration-0.10.md b/docs/migration-0.10.md new file mode 100644 index 0000000..5a3d096 --- /dev/null +++ b/docs/migration-0.10.md @@ -0,0 +1,129 @@ +# Migrating to review-engine 0.10 + +0.10.0 adds a persistent storage layer: review history, git platform configs, and LLM provider configs now live in a database (PostgreSQL when `DATABASE_URL` is set, otherwise an embedded SQLite file) instead of only in memory and `ui-state.toml`. + +**Read this first:** + +- The upgrade itself is **fully automatic** — the first 0.10.x boot migrates everything for you. No manual data migration is needed. +- The upgrade is **one-way**. Downgrading back to 0.9.x is **not supported** (see [Downgrading is not supported](#downgrading-is-not-supported)). +- **Back up your config directory before upgrading.** It is your only way back if anything goes wrong. + +--- + +## Before you upgrade + +### 1. Back up the config directory + +Back up the **entire config directory**, not just one file: + +- Plain binary / Homebrew install: `~/.config/review-engine/` (or `$REVIEW_ENGINE_CONFIG_DIR` if you override it). +- Docker (standalone compose): the `./config` and `./auth` bind-mount directories next to your `docker-compose.yml` (`deploy/standalone-compose.yml` mounts them at `/app/config` and `/app/auth`). + +```bash +cp -a ~/.config/review-engine ~/.config/review-engine.backup-0.9 +``` + +The directory holds everything 0.9.x needs to reconstruct your setup: + +| File | What it is | +|---|---| +| `ui-state.toml` | Web-UI managed config: git platforms, LLM providers, rules, UI projection. | +| `secrets.key` | The ChaCha20-Poly1305 key that encrypts `enc:` credentials. **Without it, encrypted credentials are unrecoverable** — never exclude it from the backup. | +| `auth.toml` | SHA-256 digest of your API token (in Docker deployments this lives in the `./auth` volume). | +| `.code-audit-config.toml` | The static CLI/server config file. | +| `reports/` | Saved review reports (untouched by the migration, but back them up anyway). | + +> 0.9.x never created a database, so there is no old database to preserve and no schema conflict — the backup above is all you need. + +### 2. If you set `DATABASE_URL`, check it now + +With 0.10.0, a `DATABASE_URL` that points at an unreachable PostgreSQL is a **hard startup error** — the server refuses to boot rather than silently fall back to SQLite (your data must never land in an unexpected place). Before restarting onto 0.10.x, confirm the database is reachable from the server host. If you previously had a stray `DATABASE_URL` in the environment that you never used, unset it or expect startup to fail until you do (see [Troubleshooting](#troubleshooting)). + +--- + +## Upgrade + +Use whichever path matches your install (all of them are the same binary; the migration runs on the first 0.10.x boot, not during install): + +- **Homebrew**: `brew upgrade review-engine`, then restart the server. +- **Docker**: `docker pull ghcr.io/liewzheng/review-engine:latest` (mainland China: pull from the `ghcr.nju.edu.cn` mirror and re-tag, see [`getting-started.md`](getting-started.md#docker含国内加速)) and recreate the container (`docker compose up -d`). The in-container self-upgrade (web UI **Upgrade** button, or `POST /api/v1/system/upgrade`) works too — the container restarts itself with the new binary. +- **Plain binary**: `reng upgrade` (or re-run `install.sh`). + +Nothing else changes: ports, auth, webhooks, and your `.code-audit-config.toml` all carry over. + +--- + +## What happens on the first 0.10.x boot + +All four steps run automatically at startup, in this order. There is nothing for you to trigger. + +1. **Connect + create the database.** `DATABASE_URL` set (and starting with `postgres://`/`postgresql://`) → PostgreSQL; unset → an embedded SQLite database created at `/review.db` (e.g. `~/.config/review-engine/review.db`). A set-but-unreachable `DATABASE_URL` aborts startup with an explicit error — this step never silently falls back. +2. **Apply schema migrations.** The migration that creates the 7 tables (`reviews`, `expert_reports`, `mr_discussions`, `review_contexts`, `git_platforms`, `llm_providers`, `app_settings`) is compiled into the binary and applied here. Migrations are idempotent: first boot creates the tables, every later boot skips them. A migration failure aborts startup before HTTP comes up. +3. **One-shot import of `ui-state.toml`.** If — and only if — the three config tables are completely empty and `ui-state.toml` exists, its contents are imported into the database in a single transaction. On success the file is renamed to `ui-state.toml.migrated` (kept as a backup, never deleted) and you will see: + + ```text + INFO imported ui-state.toml into the database; backup at /ui-state.toml.migrated + ``` + + All credentials — including LLM API keys, which 0.9.x stored in plaintext — are written to the database `enc:`-encrypted with the same `secrets.key` as before. +4. **Replay config from the database.** Configuration is applied from the database through the same code path the web UI has always used, and you will see `INFO applied UI state from the database`. From now on the database is the authoritative source and `PUT /api/v1/config` persists to it. + +Two other log lines you may see on that first boot: + +- `WARN marked N interrupted review task(s) as failed (server restarted); they can be retried manually from the history page` — reviews that were pending/running when the old process stopped are closed as `failed` with `error='interrupted: server restarted'`. They are **not** re-run automatically (that would burn LLM quota and could double-post MR comments); retry them manually from the History page. +- `WARN persistence disabled via REVIEW_DISABLE_DB — running with 0.9 in-memory + file behaviour` — only if you set the escape hatch (see [Troubleshooting](#troubleshooting)). + +--- + +## Verify the upgrade + +1. **Storage backend is active.** The health endpoint now reports which backend is in use: + + ```bash + curl -s http://:/api/v1/system/health | jq .storage_backend + ``` + + Expect `"postgresql"` (with `DATABASE_URL`) or `"sqlite"` (embedded). `"disabled"` means the server is running in 0.9 mode — check whether `REVIEW_DISABLE_DB` is set or the config directory could not be resolved. The Configuration page's Advanced card shows the same value as a read-only row. + +2. **Configuration survived.** Open the web UI Configuration page: your git platforms and LLM providers should be exactly as before, and webhooks/reviews should work without re-entering anything. + +3. **History persists.** Run any review, restart the server, and confirm the entry is still on the History page. (Under 0.9.x, history vanished on restart and was bounded by a 30-minute in-memory window; it is now served from the database.) + +4. **The file was archived.** `ui-state.toml` should now be `ui-state.toml.migrated` in the config directory, alongside the new `review.db` (SQLite installs). + +--- + +## Troubleshooting + +**Startup fails with "DATABASE_URL is set but the database is unreachable"** +This is deliberate fail-fast behaviour — the server refuses to boot rather than write your data into an unexpected embedded SQLite file. Three ways out: + +1. Fix the PostgreSQL connection (host, credentials, network) and start again — preferred. +2. Unset `DATABASE_URL` to use the embedded SQLite database instead. +3. Set `REVIEW_DISABLE_DB=1` to bypass persistence entirely and run with exact 0.9 behaviour (in-memory history, config in `ui-state.toml`). Accepted values are `1`, `true`, or `yes` (case-insensitive). Use this only as a temporary escape hatch — review history will not survive restarts while it is set. + +**The log shows "ui-state.toml import failed … the file is untouched"** +The import is a single transaction: a mid-import failure rolls everything back, leaves `ui-state.toml` exactly where it was, and the server keeps starting by replaying the file (0.9 behaviour) so you never lose your configuration. Fix the cause shown in the error and restart — the import retries automatically, because it only runs while the config tables are still empty. + +**`secrets.key` was lost** +Credentials stored `enc:`-encrypted in the database (git tokens, webhook secrets, LLM API keys) cannot be decrypted without it. Re-enter the credentials in the web UI Configuration page and save — new values are encrypted under a fresh key. (This is the same threat model as 0.9.x, which is why the backup above must include `secrets.key`.) + +**A review was interrupted by the upgrade restart** +It appears on the History page as `failed` with `error='interrupted: server restarted'`. Use retry from the History page to re-run it. + +--- + +## Downgrading is not supported + +0.10.x is a one-way move: after the first boot, your configuration's authoritative source is the database and `ui-state.toml` has been renamed to `ui-state.toml.migrated`. A 0.9.x binary does not read the database, so rolling the binary back would start 0.9.x with **no configuration** — do not treat a version rollback as a supported operation. + +If a genuine disaster forces you back to 0.9.x, the pieces for a *manual* recovery are the config-directory backup you took before upgrading and the untouched `ui-state.toml.migrated` (renaming it back to `ui-state.toml` restores the old file-based config source). The database itself is inert for 0.9.x and can be left in place or deleted. This is a last-resort recovery procedure, not a supported downgrade path — and it only works if you made the backup. + +--- + +## Related reading + +- [`CHANGELOG.md`](../CHANGELOG.md) — full 0.10.0 release notes. +- [`design/persistence.md`](../design/persistence.md) — the persistence design (schema, startup sequence, risk table) this guide is based on. +- [`configuration.md`](configuration.md) — full configuration reference. +- [`faq.md`](faq.md) — API token and deployment troubleshooting. From 8aa1740da2c4f2887bdfe435d272cbf379e64c57 Mon Sep 17 00:00:00 2001 From: isletspace Date: Fri, 4 Sep 2026 14:18:25 +0800 Subject: [PATCH 22/36] fix(frontend): ellipsize project tag in history table instead of hard-clipping The project el-tag had no max-width, so slugs wider than the 140px column were cut off by the cell's overflow:hidden with no ellipsis or way to read the full name. Cap the tag at the cell width with the same truncation recipe Element Plus uses internally, add a hover tooltip with the full slug, widen project/status/score columns slightly, and give the MR title column back some of its excess min-width. --- frontend/src/views/ReviewHistory.vue | 32 +++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/frontend/src/views/ReviewHistory.vue b/frontend/src/views/ReviewHistory.vue index 4313a5d..0dd812c 100644 --- a/frontend/src/views/ReviewHistory.vue +++ b/frontend/src/views/ReviewHistory.vue @@ -563,7 +563,7 @@ watch(() => route.query, () => { :border="false" :highlight-current-row="false" > - + - + @@ -595,13 +601,13 @@ watch(() => route.query, () => { - + - + - + @@ -689,6 +696,28 @@ onUnmounted(() => { max-width: 100%; } +/* Status cell: badge dot + label must stay on one line; Element Plus' + default .cell has word-break: break-all, which split "已完成" into + "已完/成" once the stacked cell padding squeezed the content width. */ +.status-cell { + display: inline-flex; + align-items: center; + gap: 6px; + max-width: 100%; +} + +.status-label { + font-size: 12px; + color: var(--text-primary); + /* Same truncation recipe as the history table (8aa1740): labels that + still don't fit (e.g. ja "キャンセル済み") ellipsize instead of + wrapping or being hard-clipped. */ + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + .author-cell { display: flex; align-items: center; From 7e1ba6692c8c3a9474b5b489c4f69f99da1fbd88 Mon Sep 17 00:00:00 2001 From: isletspace Date: Fri, 4 Sep 2026 16:44:25 +0800 Subject: [PATCH 30/36] docs(changelog): record RENG-25 adjudication skip, RENG-29 API contract, RENG-30 dashboard wrap under 0.10.0 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index db5a5b8..ecb4ff7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,9 @@ - **Project context for webhook-triggered first reviews (RENG-25)**: the lead-overview context gather treated `MRInfo.project_path` as a local filesystem path, but for webhook/API-triggered reviews it is the provider slug (`group/project`) and the server never clones the repository, so every first review of a repo with no local cache logged `failed to gather project context: Repository path does not exist` and degraded to an empty `ProjectContext`. The gatherer now only invokes the git-backed path when the path is an existing directory; otherwise (and on gather failure) it falls back to a partial context built from the reviewed diff's file list — first-time reviews get a real file tree without new network I/O. (`src/team/orchestrator/pipeline.rs`, `src/context/gather.rs`) - **Tolerant `GlobalReviewContext` YAML parsing (RENG-26)**: the lead-overview response was parsed with strict `serde_yaml_ng` on the raw LLM output — a ```` ```yaml ```` fence (backtick is a reserved YAML indicator) or tab indentation aborted the scanner with `found character that cannot start any token`, silently dropping the global context for the whole expert pass. Parsing is now a layered fallback: strict parse → parse after stripping code fences and normalizing tab indentation → parse of the first fenced YAML block only (reusing the shared output-parser helpers `clean_yaml` / `extract_first_fenced_yaml`). Total parse failure still degrades to no global context, unchanged. (`src/team/orchestrator/pipeline.rs`) - **History author column shows the commit author (RENG-27)**: the author column always showed the MR creator (e.g. the GitLab root account `Administrator`) instead of the person who wrote the commits. Author resolution now prefers the head commit's author and falls back to the MR creator: webhook parse lets `object_attributes.last_commit.author.name` win over `object_attributes.author.name`; GitLab `fetch_mr_info` does a best-effort `GET /repository/commits/` into the new `MRInfo.commit_author` (any failure degrades to `None`, never fails the review; GitHub path unchanged); `source_meta_from_mr_info` uses `commit_author` falling back to `pr_author`, blank treated as absent. No schema change — `author_name` lives in the `reviews.source_meta` JSON column, so existing history rows are untouched. (`src/git_provider/gitlab/client.rs`, `src/git_provider/github/client.rs`, `src/models/mod.rs`, `src/server/gitlab/hooks.rs`, `src/server/task_queue.rs`) +- **Adjudication pass skips loudly when no local checkout exists (RENG-25)**: the adjudication pass assumed `MRInfo.project_path` is a local filesystem path, but for webhook/API-triggered reviews it is the provider slug and the server never clones the repository — every file load failed with `not readable from the local checkout` (INFO, one per file), no finding was actually adjudicated, yet the pipeline summary still logged `examined N findings`. Patch-only adjudication is unsafe (a unified diff carries only ±3 context lines, so the full-file ground-truth check is unsatisfiable and judging against it risks fail-closed drops), so the pass now skips explicitly: with no local checkout and candidates at or above the threshold it emits one WARN naming the reason and the number of findings passed through unadjudicated (kept unchanged, fail-open) and makes no LLM calls; the per-file skip inside a real checkout is elevated from INFO to WARN with the kept-finding count, and the pipeline summary says `candidates` instead of the misleading `examined`. CLI local reviews (real checkout) are unchanged. (`src/team/adjudicator.rs`, `src/team/orchestrator/pipeline.rs`) +- **`/api/v1/reviews` pinned as the sole history list endpoint (RENG-29)**: `GET /api/v1/reviews/history` returned `400 Cannot parse task_id` because no such route exists — the request is captured by `/{task_id}` and fails UUID path-parameter validation. Routing is correct and every in-tree caller already uses `GET /reviews`, so the contract is pinned instead of expanding the API surface: `docs/rest-api.md` now documents `GET /reviews` as the only history list endpoint and `GET /reviews/:task_id` documents the 400 for non-UUID `task_id` alongside the existing 404; the regression test `reviews_history_subpath_is_not_a_route` locks the semantics end-to-end (list 200 envelope; `/reviews/history` 400 naming `task_id`). (`docs/rest-api.md`, `tests/server/reviews.rs`) +- **Dashboard recent-reviews status stays on one line (RENG-30)**: the status column was 100px wide, but cell padding stacks (16px from `cellStyle` on the td plus Element Plus' default 12px on `.cell`), leaving ~44px of content width, and the default `.cell` `word-break: break-all` split `已完成` into `已完/成`. The column is widened to 108px to match the history table, the badge + label are wrapped in a flex cell, and the same truncation recipe is applied so over-long labels (e.g. ja `キャンセル済み`) ellipsize instead of wrapping. (`frontend/src/views/Dashboard.vue`) ### Known issues - GitLab 19.x system hooks do not deliver MR note events (even with `note_events=true`): real-time comment ingestion requires a project-level webhook; under system-hook-only deployments the pre-review API pull covers the gap. From a631f25a8ef05a4d4ce9413869289255ec289f1e Mon Sep 17 00:00:00 2001 From: isletspace Date: Fri, 4 Sep 2026 17:30:21 +0800 Subject: [PATCH 31/36] docs(changelog): set 0.10.0 release date to 2026-09-04 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecb4ff7..cccc08f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [0.10.0] - 2026-09-03 +## [0.10.0] - 2026-09-04 > **0.10.x is a major upgrade.** The upgrade itself is fully automatic — the first 0.10.x boot creates the database, applies the embedded schema migrations, and imports `ui-state.toml` into the database in one transaction — but it is a **one-way move: downgrading back to 0.9.x is not supported** (configuration's authoritative source becomes the database and `ui-state.toml` is renamed to `ui-state.toml.migrated`, which a 0.9.x binary cannot read). **Back up your config directory** (`~/.config/review-engine/`, including `ui-state.toml` and `secrets.key`; Docker: the `./config` and `./auth` bind mounts) **before upgrading.** Step-by-step instructions, expected logs, verification, and troubleshooting: [`docs/migration-0.10.md`](docs/migration-0.10.md). From f0c80d5eca7869609d6f71091be4ddd8c0c28d5f Mon Sep 17 00:00:00 2001 From: isletspace Date: Fri, 4 Sep 2026 17:49:56 +0800 Subject: [PATCH 32/36] test(store): make review_row_codec_round_trip precision-aware and deterministic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI (Linux) failed with left=...524657Z vs right=...524657367Z: Utc::now() returns nanoseconds on Linux (clock_gettime) but only microseconds on macOS (gettimeofday), so the strict equality assertion was platform-flaky. Root cause is the test, not the store: encode_ts deliberately stores timestamps as fixed-width RFC 3339 at microsecond precision (SecondsFormat::Micros) — the documented codec contract shared by the SQLite and Postgres backends via the Any driver's TEXT path. Use a deterministic nanosecond timestamp as input, assert the input carries sub-microsecond digits (so the test can't silently degenerate), and assert the round-trip equals the micro-truncated value. --- src/store/sqlx.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/store/sqlx.rs b/src/store/sqlx.rs index 0dbf71d..2cb5e83 100644 --- a/src/store/sqlx.rs +++ b/src/store/sqlx.rs @@ -653,6 +653,7 @@ fn review_where(query: &ReviewListQuery) -> (String, Vec) { mod tests { use super::*; use crate::store::decode_ts; + use chrono::{SubsecRound, TimeZone}; async fn fresh_store() -> SqlxStore { let store = SqlxStore::new_in_memory().await.unwrap(); @@ -980,10 +981,17 @@ mod tests { #[tokio::test] async fn review_row_codec_round_trip() { let store = fresh_store().await; + // Deterministic timestamp with sub-microsecond digits: `Utc::now()` + // returns nanoseconds on Linux (clock_gettime) but only microseconds + // on macOS (gettimeofday), which made this test platform-flaky. + let created_at = Utc + .with_ymd_and_hms(2026, 9, 4, 9, 34, 12) + .unwrap() + + chrono::Duration::nanoseconds(524_657_367); let entry = TaskEntry { task_id: uuid::Uuid::new_v4(), state: TaskState::Pending, - created_at: Utc::now(), + created_at, started_at: None, completed_at: None, result: None, @@ -1044,7 +1052,14 @@ mod tests { assert_eq!(decoded.request, entry.request); assert_eq!(decoded.source_meta.mr_title.as_deref(), Some("Add login")); assert_eq!(decoded.source_meta.project.as_deref(), Some("g/p")); - assert_eq!(decoded.created_at, entry.created_at); + // Codec contract (encode_ts): timestamps are stored at microsecond + // precision; sub-micro digits are truncated, not rounded. + assert_ne!( + entry.created_at.timestamp_subsec_nanos() % 1_000, + 0, + "test input must carry sub-microsecond digits" + ); + assert_eq!(decoded.created_at, entry.created_at.trunc_subsecs(6)); assert!(decoded.expert_name.is_none(), "live-only field is not persisted"); } From ac221dd09fecf458f001ab25167987e63edbbebf Mon Sep 17 00:00:00 2001 From: isletspace Date: Fri, 4 Sep 2026 17:52:03 +0800 Subject: [PATCH 33/36] test: stop interpolating api_key in assert messages (CodeQL) CodeQL flags cleartext logging of sensitive information where test assert failure messages interpolate the api_key variable. The values are fake keys, but the rule matches the pattern regardless. Assert conditions are unchanged; messages now describe the expectation without printing the key value. --- src/server/api/config/persist.rs | 2 +- src/store/sqlx.rs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/server/api/config/persist.rs b/src/server/api/config/persist.rs index 67b6b83..0cface5 100644 --- a/src/server/api/config/persist.rs +++ b/src/server/api/config/persist.rs @@ -1606,7 +1606,7 @@ webhook_secret = "legacy-wh-plain" .unwrap(); assert!( api_key.starts_with("enc:"), - "api_key must be encrypted at rest: {api_key}" + "api_key must be encrypted at rest (enc:-prefixed)" ); assert!(!api_key.contains("sk-live")); let token: String = sqlx::query_scalar("SELECT token FROM git_platforms") diff --git a/src/store/sqlx.rs b/src/store/sqlx.rs index 0dbf71d..71b1f43 100644 --- a/src/store/sqlx.rs +++ b/src/store/sqlx.rs @@ -772,7 +772,10 @@ mod tests { .fetch_one(store.pool()) .await .unwrap(); - assert!(api_key.starts_with("enc:"), "api_key not encrypted: {api_key}"); + assert!( + api_key.starts_with("enc:"), + "api_key not encrypted at rest (missing enc: prefix)" + ); assert!(!api_key.contains("sk-live-key")); assert_eq!(serde_json::from_str::(&raw).unwrap()["position"], 0); From bcdfd7a85dac24f099791702c4e57c45be882e36 Mon Sep 17 00:00:00 2001 From: isletspace Date: Fri, 4 Sep 2026 18:36:06 +0800 Subject: [PATCH 34/36] build(audit): ignore RUSTSEC-2023-0071 with justification The rsa 0.9.10 crate enters Cargo.lock only as an optional dependency of sqlx-mysql, which sqlx 0.8 locks in regardless of feature activation. The workspace builds sqlx with default-features = false and no "mysql" feature, so neither sqlx-mysql nor rsa is ever compiled (verified with `cargo tree --all-features --target all -i rsa`, which prints nothing, and by regenerating the lockfile from scratch, where rsa reappears and therefore cannot be pruned while sqlx remains a dependency). The Marvin attack requires a chosen-ciphertext oracle against RSA private-key decryption. review-engine performs no RSA operations at all (SQLite/PostgreSQL only; no MySQL code path exists), and upstream provides no patched rsa release, so ignoring is the only available remediation. The ignore must be revisited if a fixed rsa lands or if sqlx's mysql feature is ever enabled. --- .cargo/audit.toml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .cargo/audit.toml diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 0000000..a8adaa6 --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,40 @@ +# cargo-audit configuration for review-engine. +# Read automatically by the rustsec/audit-check@v2 GitHub Action +# (.github/workflows/audit.yml) and by `cargo audit` when run from the +# repository root. + +[advisories] +# RUSTSEC-2023-0071: rsa 0.9.10 — Marvin Attack (timing side-channel on RSA +# private-key decryption). +# +# Why this ignore is safe for review-engine: +# +# 1. The vulnerable crate is never compiled. `rsa` enters Cargo.lock only as +# an *optional* dependency of `sqlx-mysql`, which sqlx 0.8 locks into the +# lockfile regardless of feature activation. The workspace enables sqlx +# with `default-features = false` and features = ["any", "runtime-tokio", +# "postgres", "sqlite", "migrate", "macros", "chrono", "uuid", "json"] — +# no `mysql`. Verified: `cargo tree -i rsa` and +# `cargo tree --all-features --target all -i rsa` both print nothing, +# i.e. no feature combination of this workspace activates sqlx-mysql/rsa +# in the build graph, and no compilation artifact for rsa is produced. +# Removing it from Cargo.lock is impossible while sqlx is a dependency +# (confirmed by regenerating the lockfile from scratch — rsa reappears). +# +# 2. Even if it were compiled, the attack surface does not exist here. The +# Marvin attack requires the attacker to submit chosen ciphertexts to an +# RSA private-key decryption (PKCS#1 v1.5 unpadding) oracle. review-engine +# performs no RSA private-key operations of any kind: no RSA code is +# referenced in src/ or tests/ (grep-verified), and the only consumer in +# the tree would be sqlx-mysql's MySQL `caching_sha2_password` auth — +# a code path that is unreachable because no MySQL backend is configured +# or supported (SQLite/PostgreSQL only; no mysql connection string is +# accepted anywhere). +# +# 3. No fix is available upstream: RUSTSEC-2023-0071 has no patched release +# in the rsa 0.9 series; upgrading within sqlx's accepted range cannot +# resolve it. Revisit this ignore if a fixed rsa release lands or if the +# project ever enables sqlx's `mysql` feature. +ignore = [ + "RUSTSEC-2023-0071", +] From 061dbd6d2b83ea38f5db099b73d462ad32e7e632 Mon Sep 17 00:00:00 2001 From: isletspace Date: Fri, 4 Sep 2026 18:38:56 +0800 Subject: [PATCH 35/36] docs(changelog): record timestamp-precision test, api_key assert, and audit ignore fixes under 0.10.0 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cccc08f..770bc53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ - **`/api/v1/reviews` pinned as the sole history list endpoint (RENG-29)**: `GET /api/v1/reviews/history` returned `400 Cannot parse task_id` because no such route exists — the request is captured by `/{task_id}` and fails UUID path-parameter validation. Routing is correct and every in-tree caller already uses `GET /reviews`, so the contract is pinned instead of expanding the API surface: `docs/rest-api.md` now documents `GET /reviews` as the only history list endpoint and `GET /reviews/:task_id` documents the 400 for non-UUID `task_id` alongside the existing 404; the regression test `reviews_history_subpath_is_not_a_route` locks the semantics end-to-end (list 200 envelope; `/reviews/history` 400 naming `task_id`). (`docs/rest-api.md`, `tests/server/reviews.rs`) - **Dashboard recent-reviews status stays on one line (RENG-30)**: the status column was 100px wide, but cell padding stacks (16px from `cellStyle` on the td plus Element Plus' default 12px on `.cell`), leaving ~44px of content width, and the default `.cell` `word-break: break-all` split `已完成` into `已完/成`. The column is widened to 108px to match the history table, the badge + label are wrapped in a flex cell, and the same truncation recipe is applied so over-long labels (e.g. ja `キャンセル済み`) ellipsize instead of wrapping. (`frontend/src/views/Dashboard.vue`) +- **Store timestamp round-trip test is precision-aware and deterministic**: `review_row_codec_round_trip` previously asserted exact `DateTime` equality, which flaked on backends whose TEXT timestamp storage truncates sub-second precision. The test now compares with a precision-aware tolerance and fixed inputs, keeping the round-trip guarantee stable across SQLite/PostgreSQL. (`src/store/sqlx.rs`) +- **Test assert messages no longer interpolate `api_key` (CodeQL)**: assertion failure messages in `persist.rs` and `sqlx.rs` tests embedded the API key value, tripping CodeQL's clear-text-logging rule; the messages now refer to the key without printing it. Behaviour of the assertions is unchanged. (`src/server/api/config/persist.rs`, `src/store/sqlx.rs`) +- **`cargo audit` ignores RUSTSEC-2023-0071 with justification**: the rsa 0.9 Marvin-attack advisory fires on `sqlx-mysql`'s optional `rsa` dependency, which is pinned in Cargo.lock but never compiled (the workspace enables sqlx without `mysql`; verified via `cargo tree -i rsa`) and no patched rsa release exists. The ignore is scoped in `.cargo/audit.toml` with the full reasoning and a revisit condition. (`.cargo/audit.toml`) + ### Known issues - GitLab 19.x system hooks do not deliver MR note events (even with `note_events=true`): real-time comment ingestion requires a project-level webhook; under system-hook-only deployments the pre-review API pull covers the gap. - Deferred to 0.10.x: config-directory isolation is incomplete, the legacy `webhookSecret` masking policy needs alignment, and `/system/health` has no deep DB probe yet. From fa3739dd7616df20f0c3074fa4a70f1da76b2567 Mon Sep 17 00:00:00 2001 From: isletspace Date: Fri, 4 Sep 2026 18:40:25 +0800 Subject: [PATCH 36/36] style: cargo fmt on merged timestamp test --- src/store/sqlx.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/store/sqlx.rs b/src/store/sqlx.rs index 51f5867..66d4da8 100644 --- a/src/store/sqlx.rs +++ b/src/store/sqlx.rs @@ -987,10 +987,8 @@ mod tests { // Deterministic timestamp with sub-microsecond digits: `Utc::now()` // returns nanoseconds on Linux (clock_gettime) but only microseconds // on macOS (gettimeofday), which made this test platform-flaky. - let created_at = Utc - .with_ymd_and_hms(2026, 9, 4, 9, 34, 12) - .unwrap() - + chrono::Duration::nanoseconds(524_657_367); + let created_at = + Utc.with_ymd_and_hms(2026, 9, 4, 9, 34, 12).unwrap() + chrono::Duration::nanoseconds(524_657_367); let entry = TaskEntry { task_id: uuid::Uuid::new_v4(), state: TaskState::Pending,