From e2c00df52b11569a8bea8dc77f2999cec134b42e Mon Sep 17 00:00:00 2001 From: WaylandYang <145302500+WaylandYang@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:04:41 +0800 Subject: [PATCH 01/10] A derivation explains itself down to the sentence (#227) Co-authored-by: Claude Fable 5.1 --- crates/utopia-core/src/models.rs | 31 +++ crates/utopia-server/src/api/graph_routes.rs | 12 + crates/utopia-server/src/api/mod.rs | 5 + crates/utopia-store/src/reasoning.rs | 123 +++++++++ .../tests/a_proof_reaches_the_sentence.rs | 254 ++++++++++++++++++ docs/decisions/0002-reasoning-engine.md | 2 +- ...-the-open-seams-before-cutting-new-ones.md | 2 +- docs/decisions/README.md | 2 +- web/src/api.ts | 29 ++ web/src/i18n/en.ts | 4 + web/src/i18n/zh.ts | 3 + web/src/pages/Graph.tsx | 101 +++++-- 12 files changed, 550 insertions(+), 18 deletions(-) create mode 100644 crates/utopia-store/tests/a_proof_reaches_the_sentence.rs diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index f4fb8879b..06e1a0b41 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -924,6 +924,37 @@ pub struct DerivedFactView { pub premises: Vec, } +/// 证明的一步：一条断言前提，连同它的证据（0002 R2）。 +/// +/// 前提一律是断言（`fact_derivations` 不记派生），所以证明是一条链而不是一棵树： +/// 派生 → 按 `seq` 排好的断言 → 每条断言的原句。叶子就是 chunk。 +#[derive(Debug, Clone, Serialize)] +pub struct ProofStep { + pub seq: i32, + pub fact_id: Uuid, + pub subject_id: Uuid, + pub subject: String, + pub predicate_id: Option, + /// 本体里的关系名；空谓词事实（0010）不参与推导，这里理论上恒有值， + /// 留 Option 是不在读路径上撒谎 + pub predicate: Option, + pub object_id: Option, + pub object: Option, + pub valid_from: Option>, + pub valid_to: Option>, + pub confidence: f32, + /// 这条前提后来被撤了。派生随之失效，但证明还要读得出「当时靠的是什么」 + pub retracted: bool, + pub evidence: Vec, +} + +/// 一条派生事实的完整证明：它本身，加上按顺序展开到原句的前提。 +#[derive(Debug, Clone, Serialize)] +pub struct Proof { + pub derived: DerivedFactView, + pub steps: Vec, +} + /// 审核队列各档的**真实条数**。 /// /// 与列表分开取是有意的：列表有上限（一页十条），数数没有。从前左栏读的是 diff --git a/crates/utopia-server/src/api/graph_routes.rs b/crates/utopia-server/src/api/graph_routes.rs index 48081132e..d693382d9 100644 --- a/crates/utopia-server/src/api/graph_routes.rs +++ b/crates/utopia-server/src/api/graph_routes.rs @@ -228,6 +228,18 @@ pub async fn fact_evidence( Ok(Json(json!({ "evidence": evidence }))) } +/// 一条派生事实的证明（0002 R2）：前提按推导顺序，每条带证据，一路到原句。 +/// 派生已失效或不存在时 `proof` 为 null——不是错误，界面据此退回文本前提 +pub async fn derived_proof( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, derived_id)): Path<(Uuid, Uuid)>, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Viewer).await?; + let proof = utopia_store::reasoning::proof(&state.pool, kb_id, derived_id).await?; + Ok(Json(json!({ "proof": proof }))) +} + /// 手动触发抽取（failed 重试 / 补配模型后补抽）。 pub async fn extract( State(state): State, diff --git a/crates/utopia-server/src/api/mod.rs b/crates/utopia-server/src/api/mod.rs index 8818406b2..bb646e08c 100644 --- a/crates/utopia-server/src/api/mod.rs +++ b/crates/utopia-server/src/api/mod.rs @@ -290,6 +290,11 @@ pub fn router(state: AppState, cfg: &AppConfig) -> Router { "/kbs/{id}/facts/{fact_id}/evidence", get(graph_routes::fact_evidence), ) + // 派生事实的证明（0002 R2）：前提按顺序展开到原句 + .route( + "/kbs/{id}/derived/{derived_id}/proof", + get(graph_routes::derived_proof), + ) .route("/kbs/{id}/events", get(events_routes::kb_events)) .route( "/kbs/{id}/sources", diff --git a/crates/utopia-store/src/reasoning.rs b/crates/utopia-store/src/reasoning.rs index a05620784..37c42f54a 100644 --- a/crates/utopia-store/src/reasoning.rs +++ b/crates/utopia-store/src/reasoning.rs @@ -857,6 +857,129 @@ pub async fn mark_inference_ran(pool: &PgPool, kb_id: Uuid) -> AppResult<()> { Ok(()) } +/// 一条派生事实的证明，展开到原句（0002 R2）。 +/// +/// `fact_derivations` 只记直接前提，而前提一律是断言，所以「递归展开」在这里 +/// 退化成一条链：派生 → 按 `seq` 的断言 → 每条断言的证据。叶子是 chunk， +/// 界面上一路点到文档。**撤了的前提照样列出并打上标记**：派生随前提失效， +/// 但「当时靠的是什么」要读得出来，那正是记录轴存在的理由。 +/// +/// 派生已失效或不存在时回 None——不是错误，界面据此收起。 +pub async fn proof( + pool: &PgPool, + kb_id: Uuid, + derived_id: Uuid, +) -> AppResult> { + let Some(derived) = derived_one(pool, kb_id, derived_id).await? else { + return Ok(None); + }; + #[allow(clippy::type_complexity)] + let rows: Vec<( + i32, + Uuid, + Uuid, + String, + Option, + Option, + Option, + Option, + Option>, + Option>, + f32, + bool, + )> = sqlx::query_as( + "SELECT fd.seq, f.id, f.subject_id, s.canonical_name, + f.predicate_id, r.label, f.object_id, o.canonical_name, + f.valid_from, f.valid_to, f.confidence, + f.invalidated_at IS NOT NULL + FROM fact_derivations fd + JOIN facts f ON f.id = fd.premise_fact_id + JOIN entities s ON s.id = f.subject_id + LEFT JOIN relation_types r ON r.id = f.predicate_id + LEFT JOIN entities o ON o.id = f.object_id + WHERE fd.derived_fact_id = $1 + ORDER BY fd.seq", + ) + .bind(derived_id) + .fetch_all(pool) + .await?; + let mut steps = Vec::with_capacity(rows.len()); + for ( + seq, + fact_id, + subject_id, + subject, + predicate_id, + predicate, + object_id, + object, + valid_from, + valid_to, + confidence, + retracted, + ) in rows + { + // 一条链最多 MAX_DEPTH 步，逐条取证据是可数的几次往返 + let evidence = crate::graph::fact_evidence(pool, fact_id).await?; + steps.push(utopia_core::models::ProofStep { + seq, + fact_id, + subject_id, + subject, + predicate_id, + predicate, + object_id, + object, + valid_from, + valid_to, + confidence, + retracted, + evidence, + }); + } + Ok(Some(utopia_core::models::Proof { derived, steps })) +} + +/// 按 id 取一条派生（失效的也取：证明要能回看）。 +async fn derived_one( + pool: &PgPool, + kb_id: Uuid, + derived_id: Uuid, +) -> AppResult> { + Ok(sqlx::query_as( + "SELECT d.id, + d.subject_id, s.canonical_name AS subject, + d.object_id, o.canonical_name AS object, + r.label AS predicate, + ru.kind AS rule, + d.valid_from, d.valid_to, d.confidence, d.derived_at, + COALESCE( + (SELECT array_agg( + ps.canonical_name || ' · ' + || COALESCE(pr.label, '?') || ' · ' + || COALESCE(po.canonical_name, '?') + ORDER BY fd.seq) + FROM fact_derivations fd + JOIN facts pf ON pf.id = fd.premise_fact_id + JOIN entities ps ON ps.id = pf.subject_id + LEFT JOIN relation_types pr ON pr.id = pf.predicate_id + LEFT JOIN entities po ON po.id = pf.object_id + WHERE fd.derived_fact_id = d.id), + ARRAY[]::text[] + ) AS premises + FROM derived_facts d + JOIN entities s ON s.id = d.subject_id + JOIN entities o ON o.id = d.object_id + JOIN relation_types r ON r.id = d.predicate_id + JOIN rules ru ON ru.id = d.rule_id + WHERE d.kb_id = $1 AND d.id = $2", + ) + .bind(kb_id) + .bind(derived_id) + .fetch_optional(pool) + .await?) +} + /// 一条派生事实，配好展示与证明所需的文本（实体面板的「推出来的」那一档）。 /// /// **证明一起取回来**：这一档存在的理由就是「这条边不是谁说的，是这么推出来的」， diff --git a/crates/utopia-store/tests/a_proof_reaches_the_sentence.rs b/crates/utopia-store/tests/a_proof_reaches_the_sentence.rs new file mode 100644 index 000000000..e7139a272 --- /dev/null +++ b/crates/utopia-store/tests/a_proof_reaches_the_sentence.rs @@ -0,0 +1,254 @@ +//! R2：一条派生的证明要能一路读到原句（`docs/decisions/0002`）。 +//! +//! 前提一律是断言（`fact_derivations` 不记派生），所以证明是一条链： +//! 派生 → 按 `seq` 排好的断言 → 每条断言的证据 → chunk。这里守三件事： +//! +//! 1. **顺序对**。`A part_of B`、`B part_of C` 推出 `A part_of C`，证明第一步是 A→B。 +//! 2. **叶子是原句**。每一步带着它的证据，引句就是当初抽出它的那句话。 +//! 3. **撤了的前提照样列出并打标记**。派生随之失效，`proof` 仍能回看当时靠的是什么。 +//! +//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆，绝不碰已有的库。 + +use sqlx::PgPool; +use utopia_store::reasoning; +use uuid::Uuid; + +struct Fixture { + org: Uuid, + kb: Uuid, + part_of: Uuid, + a: Uuid, + b: Uuid, + c: Uuid, + doc: Uuid, + chunk_ab: Uuid, + chunk_bc: Uuid, +} + +async fn seed(pool: &PgPool) -> anyhow::Result { + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let etype = Uuid::now_v7(); + let part_of = Uuid::now_v7(); + let (a, b, c) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let (src, doc, chunk_ab, chunk_bc) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'proof-test')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'proof-test')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases (id, workspace_id, name, materialize_inferences) + VALUES ($1, $2, 'proof-test', TRUE)", + ) + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, 'thing', 'Thing')", + ) + .bind(etype) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO relation_types (id, kb_id, key, label, is_transitive) + VALUES ($1, $2, 'part_of', 'part of', TRUE)", + ) + .bind(part_of) + .bind(kb) + .execute(pool) + .await?; + for (id, name) in [(a, "FarmBeats"), (b, "Azure"), (c, "Microsoft")] { + sqlx::query( + "INSERT INTO entities (id, kb_id, type_id, canonical_name) VALUES ($1, $2, $3, $4)", + ) + .bind(id) + .bind(kb) + .bind(etype) + .bind(name) + .execute(pool) + .await?; + } + sqlx::query("INSERT INTO sources (id, kb_id, name) VALUES ($1, $2, 'proof-test')") + .bind(src) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO documents (id, kb_id, source_id, filename, sha256, status) + VALUES ($1, $2, $3, 'press.md', 'proof', 'ready')", + ) + .bind(doc) + .bind(kb) + .bind(src) + .execute(pool) + .await?; + for (id, seq, text) in [ + (chunk_ab, 0i32, "FarmBeats is part of Azure."), + (chunk_bc, 1i32, "Azure is part of Microsoft."), + ] { + sqlx::query( + "INSERT INTO chunks (id, kb_id, document_id, seq, text) VALUES ($1, $2, $3, $4, $5)", + ) + .bind(id) + .bind(kb) + .bind(doc) + .bind(seq) + .bind(text) + .execute(pool) + .await?; + } + Ok(Fixture { + org, + kb, + part_of, + a, + b, + c, + doc, + chunk_ab, + chunk_bc, + }) +} + +/// 一条断言，带一句原文当证据 +async fn asserted( + pool: &PgPool, + f: &Fixture, + subject: Uuid, + object: Uuid, + chunk: Uuid, + quote: &str, +) -> anyhow::Result { + let id = Uuid::now_v7(); + sqlx::query( + "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, confidence) + VALUES ($1, $2, $3, $4, $5, 0.9)", + ) + .bind(id) + .bind(f.kb) + .bind(subject) + .bind(f.part_of) + .bind(object) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO fact_evidence (fact_id, chunk_id, quote, proposed_predicate, document_id, doc_version) + VALUES ($1, $2, $3, 'part of', $4, 1)", + ) + .bind(id) + .bind(chunk) + .bind(quote) + .bind(f.doc) + .execute(pool) + .await?; + Ok(id) +} + +#[tokio::test] +async fn a_proof_reaches_the_sentence() -> anyhow::Result<()> { + let Ok(url) = std::env::var("UTOPIA_DATABASE_URL") else { + eprintln!("跳过：未设 UTOPIA_DATABASE_URL"); + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let f = seed(&pool).await?; + + let run = async { + let ab = asserted( + &pool, + &f, + f.a, + f.b, + f.chunk_ab, + "FarmBeats is part of Azure", + ) + .await?; + let bc = asserted( + &pool, + &f, + f.b, + f.c, + f.chunk_bc, + "Azure is part of Microsoft", + ) + .await?; + reasoning::materialize(&pool, f.kb).await?; + + let derived: Vec = + reasoning::derived_for_entity(&pool, f.kb, f.a).await?; + let ac = derived + .iter() + .find(|d| d.subject_id == f.a && d.object_id == f.c) + .expect("A part_of C should be derived"); + + // 1. 顺序对，2. 叶子是原句 + let proof = reasoning::proof(&pool, f.kb, ac.id) + .await? + .expect("live derivation has a proof"); + assert_eq!(proof.derived.id, ac.id); + assert_eq!(proof.steps.len(), 2); + assert_eq!( + proof.steps[0].fact_id, ab, + "the chain starts where the derivation starts" + ); + assert_eq!(proof.steps[1].fact_id, bc); + assert_eq!(proof.steps[0].subject, "FarmBeats"); + assert_eq!(proof.steps[0].object.as_deref(), Some("Azure")); + assert_eq!(proof.steps[0].predicate.as_deref(), Some("part of")); + assert_eq!(proof.steps[0].evidence.len(), 1); + assert_eq!( + proof.steps[0].evidence[0].quote.as_deref(), + Some("FarmBeats is part of Azure"), + "the leaf of a proof is the sentence it was extracted from" + ); + assert_eq!(proof.steps[0].evidence[0].chunk_id, f.chunk_ab); + assert_eq!(proof.steps[1].evidence[0].chunk_id, f.chunk_bc); + assert!(proof.steps.iter().all(|s| !s.retracted)); + + // 一个不存在的 id 不是错误，是「没有证明」 + assert!(reasoning::proof(&pool, f.kb, Uuid::now_v7()) + .await? + .is_none()); + + // 3. 撤掉一条前提：派生失效，证明还在，且那一步打上标记 + sqlx::query("UPDATE facts SET invalidated_at = now() WHERE id = $1") + .bind(bc) + .execute(&pool) + .await?; + reasoning::materialize(&pool, f.kb).await?; + let (gone,): (bool,) = + sqlx::query_as("SELECT invalidated_at IS NOT NULL FROM derived_facts WHERE id = $1") + .bind(ac.id) + .fetch_one(&pool) + .await?; + assert!(gone, "a derivation falls with its premise"); + let proof = reasoning::proof(&pool, f.kb, ac.id) + .await? + .expect("an invalidated derivation still explains itself"); + assert!(!proof.steps[0].retracted); + assert!( + proof.steps[1].retracted, + "the retracted premise is marked, not hidden" + ); + anyhow::Ok(()) + } + .await; + + let _ = sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(f.org) + .execute(&pool) + .await; + run +} diff --git a/docs/decisions/0002-reasoning-engine.md b/docs/decisions/0002-reasoning-engine.md index 3dec2e005..976eb23fb 100644 --- a/docs/decisions/0002-reasoning-engine.md +++ b/docs/decisions/0002-reasoning-engine.md @@ -122,7 +122,7 @@ CREATE TABLE fact_derivations ( > 冷启动自动扩本体现在**一位公理都不替人声明**（`Axioms::default()`）：推理机的判据必须是人写下来的。 ### R2 · 解释 -证明树 API + UI。展开到叶子（chunk）为止，中间节点是派生事实与规则。〔**只做了一层**：`derived_for_entity` 把直接前提展开成文本。递归到叶子的 API 与 UI 未建，数据结构已足够支撑。〕 +证明树 API + UI。展开到叶子（chunk）为止，中间节点是派生事实与规则。〔**已做**（0016 B1，`feat/proof-tree`）：`reasoning::proof` + `GET /kbs/{id}/derived/{id}/proof`，实体面板里派生行展开即证明链。**形状是链不是树**——`fact_derivations` 只记断言前提（推导的输入排除派生），所以「递归展开」退化成按 `seq` 的一条链：派生 → 断言 → 各自的证据 → chunk，界面一路点到文档。撤了的前提照样列出并打标记，派生失效后证明仍可回看——记录轴的用法。先前写的「中间节点是派生事实」在这个数据模型里不会出现。〕 ### R3 · 增量维护 diff --git a/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md b/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md index eca4adf53..02fff13e2 100644 --- a/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md +++ b/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md @@ -66,7 +66,7 @@ Chat memory 与 MCP 两处在 A1 / A2 落地前标 in development。中英两份 ### B · 推理机补完——A 之后，与 C 并行 -**B1 · R2 证明树。** 递归展开到叶子 chunk 的 API + 实体面板里可展开的树。数据结构（`fact_derivations`）已够。 +**B1 · R2 证明树。**〔已做，`feat/proof-tree`；实际是链不是树，理由见 0002 R2 的修订〕递归展开到叶子 chunk 的 API + 实体面板里可展开的树。数据结构（`fact_derivations`）已够。 **B2 · 派生 vs 断言矛盾要有信号。** `axiom_violations` 加一种 kind（`derived_contradiction`），进 Review 同一档——0002 写了没做的那一行。 **B3 · `disjointWith` 进消解，合并路径复核签名。** `classify_type_drift` 改从 `entity_type_disjoint` 读（没声明就退回今天的行为，不硬编码）； `merge_entities` 搬事实前跑一遍与写入时相同的 domain / range 检查，违反的进 `axiom_violations` 而不是静默搬过去。这是 0009 与 0012 各自待做的同一件事。 diff --git a/docs/decisions/README.md b/docs/decisions/README.md index ce399d59d..81142c040 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -27,7 +27,7 @@ | | 文档 | 状态 | |---|---|---| | 0001 | [本体导入与治理路线](0001-ontology-import-and-governance.md) | 进行中 · P0–P2c 全建成；P3 按预算落地、P3a 只能手动跑；P3b 建成但形态不同；**P4b/P4c 待做**；P5 已由 0002 落地；判据 2 被 0012 推翻一半 | -| 0002 | [推理机](0002-reasoning-engine.md) | R0 建成（事实层五类含签名、本体自检八类）· R1 建成带开关默认关 · **R2 只一层、R3 未做** · 派生 vs 断言矛盾无信号 | +| 0002 | [推理机](0002-reasoning-engine.md) | R0 建成（事实层五类含签名、本体自检八类）· R1 建成带开关默认关 · R2 证明链到原句（B1）· **R3 未做** · 派生 vs 断言矛盾无信号 | | 0003 | [本体从语料里长出来，人站在哪一环](0003-ontology-growth-loop.md) | 已建成且仍在跑 · 起点已被 0010 与种子退场改写 · 「拒绝有记忆」被 0007 推翻重做 · 新说法提醒待做 | | 0004 | [语言：哪些字跟着看的人走，哪些跟着语料走](0004-language-and-localization.md) | 已建成 · L0–L3 全落地 · 界面刻意不猜浏览器语言 · 「中文内置本体」随播种退场作废 | | 0005 | [告警中心](0005-alert-center.md) | 已建成 · 五种告警 · 三个决定推翻两个（就地留痕）· `no_text_layer` 待接 | diff --git a/web/src/api.ts b/web/src/api.ts index 4b835cad6..9fa6adf77 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -324,6 +324,29 @@ export interface DerivedFact { premises: string[]; } +/** 证明的一步：一条断言前提，带它的证据。前提一律是断言，所以证明是链不是树 */ +export interface ProofStep { + seq: number; + fact_id: string; + subject_id: string; + subject: string; + predicate_id: string | null; + predicate: string | null; + object_id: string | null; + object: string | null; + valid_from: string | null; + valid_to: string | null; + confidence: number; + /** 这条前提后来被撤了；派生随之失效，证明仍要读得出当时靠的是什么 */ + retracted: boolean; + evidence: Evidence[]; +} + +export interface Proof { + derived: DerivedFact; + steps: ProofStep[]; +} + /** 审核页的分档。**与服务端的 queue 参数是同一组字面量**——拼错会拿到 * 一个明确的 unknown_queue 错误，而不是悄悄的空列表。 */ export type ReviewQueue = @@ -1318,6 +1341,12 @@ export const api = { request<{ evidence: Evidence[] }>( `/api/v1/kbs/${kbId}/facts/${factId}/evidence`, ), + /** 一条派生事实的证明（0002 R2）：前提按推导顺序，每条带证据，一路到原句。 + * 派生已失效时回 null——不是错误 */ + derivedProof: (kbId: string, derivedId: string) => + request<{ proof: Proof | null }>( + `/api/v1/kbs/${kbId}/derived/${derivedId}/proof`, + ), documentDetail: (id: string) => request<{ document: Doc; chunks: ChunkFull[] }>(`/api/v1/documents/${id}`), extractDocument: (id: string) => diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 613c9ad58..c2cce2958 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -732,6 +732,10 @@ export const en = { derivedHint: "Edges no one asserted — the engine worked them out from axioms your ontology declares. Each one shows the premises it came from.", derivedNoProof: "The premises are gone.", + /** 证明链（0002 R2）：每一步是一条断言前提，展开到原句 */ + proofStep: (n: number) => `Step ${n}`, + proofRetracted: "since retracted", + proofLoading: "Tracing the proof…", derivedPanel: "Inference", derivedRunAsk: "Re-run inference for the whole base?", derivedRunGo: "Run", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index 14c01d8e0..c5915ed25 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -674,6 +674,9 @@ export const zh: Strings = { derivedHint: "没有人断言过的边——引擎按本体声明的公理推出来的。每一条都附着它用到的前提。", derivedNoProof: "前提已经不在了。", + proofStep: (n: number) => `第 ${n} 步`, + proofRetracted: "后来撤了", + proofLoading: "正在展开证明…", derivedPanel: "推理", derivedRunAsk: "对整个库重跑一遍推理？", derivedRunGo: "跑", diff --git a/web/src/pages/Graph.tsx b/web/src/pages/Graph.tsx index 1ec96c230..5f32501fa 100644 --- a/web/src/pages/Graph.tsx +++ b/web/src/pages/Graph.tsx @@ -2443,6 +2443,7 @@ function DerivedPanel({ * Relations/Timeline/History 的紧凑行里显得是另一个产品的东西，而且十几条 * 推导堆起来是一面墙。证明是「问了才看」的东西，收进展开区正合适。 */ function DerivedRow({ + kbId, d, otherId, otherName, @@ -2450,6 +2451,7 @@ function DerivedRow({ onToggle, onNavigate, }: { + kbId: string; d: DerivedFact; otherId: string; otherName: string; @@ -2490,23 +2492,91 @@ function DerivedRow({ {d.premises.length} - {/* 证明：前提按推导顺序。**边框与 EvidenceList 同一档**—— - 两者是同一件事的两种形态：一个给出处，一个给推理链 */} - {open && ( -
-
    - {d.premises.map((p, i) => ( -
  1. - {p} -
    2. - ))} -
+ {/* 证明：前提按推导顺序，每条展开到原句（0002 R2）。**边框与 EvidenceList + 同一档**——两者是同一件事的两种形态：一个给出处，一个给推理链 */} + {open && } +
+ ); +} + +/** 一条派生的证明链。展开时才取——证明是「问了才看」的东西。 + * + * 每一步是一条断言前提：三元组在上，它的原句在下，原句可点进文档。 + * 前提被撤过的打标记但不藏：派生随之失效，而「当时靠的是什么」正是记录轴要答的。 + * 取不到（派生已失效）就退回列表里带来的那几行文本，不空着。 */ +function ProofChain({ kbId, d }: { kbId: string; d: DerivedFact }) { + const proof = useQuery({ + queryKey: ["proof", d.id], + queryFn: () => api.derivedProof(kbId, d.id), + }); + const steps = proof.data?.proof?.steps; + return ( +
+ {proof.isPending && ( +

{S.graph.proofLoading}

+ )} + {steps && ( +
    + {steps.map((st) => ( +
  1. +
    + + {S.graph.proofStep(st.seq + 1)} + + + {st.subject} + — {st.predicate ?? "?"} → + {st.object ?? "?"} + + {st.retracted && ( + {S.graph.proofRetracted} + )} +
    +
    + {st.evidence.map((ev) => ( + +
    + {ev.quote ? `“${ev.quote}”` : S.graph.noQuote} +
    +
    + {S.graph.sectionRef(ev.filename, ev.seq + 1)} + {ev.stale && ( + + {S.graph.fromVersion(ev.doc_version)} + + )} +
    + + ))} + {st.evidence.length === 0 && ( +

    {S.graph.noEvidence}

    + )} +
    +
    2. + ))} +
+ )} + {/* 派生已失效、证明取不到：退回列表里带来的那几行文本 */} + {!proof.isPending && !steps && ( +
    + {d.premises.map((p, i) => ( +
  1. + {p} +
    2. + ))} {d.premises.length === 0 && ( -

    - {S.graph.derivedNoProof} -

    +
  2. {S.graph.derivedNoProof}
    3. )} -
+ )} ); @@ -2966,6 +3036,7 @@ function EntityPanel({ return ( Date: Thu, 3 Sep 2026 11:14:24 +0800 Subject: [PATCH 02/10] A contradiction points at an error upstream (B2a) (#238) * A contradiction points at an error upstream Co-Authored-By: Claude Fable 5.1 * A contradiction points at an error upstream (B2a) Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- crates/utopia-core/src/models.rs | 15 +- crates/utopia-reason/src/derive.rs | 405 +++++++++++++++- crates/utopia-reason/src/lib.rs | 8 +- crates/utopia-server/src/api/review_routes.rs | 94 +++- crates/utopia-store/src/reasoning.rs | 453 +++++++++++++++--- .../tests/a_contradiction_points_upstream.rs | 341 +++++++++++++ docs/decisions/0002-reasoning-engine.md | 2 + ...-the-open-seams-before-cutting-new-ones.md | 2 +- .../0017-a-contradiction-points-upstream.md | 205 ++++++++ docs/decisions/README.md | 3 +- .../0020_a_contradiction_points_upstream.sql | 34 ++ web/src/api.ts | 56 ++- web/src/i18n/en.ts | 28 ++ web/src/i18n/zh.ts | 23 + web/src/pages/Review.tsx | 147 +++++- web/src/styles.css | 2 + 16 files changed, 1722 insertions(+), 96 deletions(-) create mode 100644 crates/utopia-store/tests/a_contradiction_points_upstream.rs create mode 100644 docs/decisions/0017-a-contradiction-points-upstream.md create mode 100644 migrations/0020_a_contradiction_points_upstream.sql diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index 06e1a0b41..ab05c1848 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -867,10 +867,10 @@ pub struct ConceptMapping { /// /// **两条事实都展开成 主-谓-宾 文本**：Review 页要让人一眼看出矛盾在哪， /// 而两个 UUID 看不出任何东西。自反那一类两条相同——它就是一条事实。 -#[derive(Debug, Clone, Serialize, sqlx::FromRow)] +#[derive(Debug, Clone, Serialize)] pub struct AxiomViolation { pub id: Uuid, - /// self_loop | asymmetry | cycle | functional + /// self_loop | asymmetry | cycle | functional | signature | derived_contradiction pub kind: String, /// 判据来自哪条关系。人若判「公理写错了」，从这里进本体去改 pub predicate: Option, @@ -881,6 +881,12 @@ pub struct AxiomViolation { /// 环的长度（含首尾）。其余三类为 0——前端据此决定要不要显示「查看路径」 pub path_len: i32, pub detected_at: chrono::DateTime, + /// `derived_contradiction` 独有（0017）：推出来的那条三元组——它没有落库， + /// 只能在这里写出来。字段见 `reasoning::run`。其余种类是 `{}` + pub detail: serde_json::Value, + /// 审核线索（0017 §2）：`stale`（旧断言没写结束日期）、`duplicate`（有同名 + /// 实体）、`unsure`（抽取置信度低）。只给一条，没有就空 + pub hint: Option, } /// 本体自己的一处自相矛盾（见 `ontology_defects`）。 @@ -891,8 +897,11 @@ pub struct AxiomViolation { pub struct OntologyDefect { pub id: Uuid, /// symmetric_and_asymmetric | transitive_and_functional | subclass_cycle - /// | disjoint_with_ancestor | inherits_disjoint + /// | disjoint_with_ancestor | inherits_disjoint | inverse_of_itself + /// | inverse_not_mutual | sub_property_cycle | rules_disagree pub kind: String, + /// `rules_disagree` 独有（0017）：哪两条规则、撞在哪条公理上、几对、几个例子 + pub detail: serde_json::Value, /// 出问题那个对象的标签（类或谓词）。查不到就是它已经被删了 pub subject_label: Option, /// 另一方：互斥的那个类 diff --git a/crates/utopia-reason/src/derive.rs b/crates/utopia-reason/src/derive.rs index 73c8ab2e1..aa6c473dc 100644 --- a/crates/utopia-reason/src/derive.rs +++ b/crates/utopia-reason/src/derive.rs @@ -20,7 +20,7 @@ //! 前提 A `[2020,2023)`、前提 B `[2022,∞)` → 派生 `[2022,2023)`。交集为空 //! 就不推——两段没有重叠的时候,链本身在任何时刻都不成立。 -use crate::{Axioms, Edge, MAX_DEPTH}; +use crate::{Axioms, Edge, Kind, MAX_DEPTH}; use std::collections::{HashMap, HashSet}; use uuid::Uuid; @@ -31,7 +31,7 @@ use uuid::Uuid; /// ——悄悄截断会让「推完了」和「推了一部分」长得一模一样。 pub const MAX_DERIVED_PER_PREDICATE: usize = 20_000; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Rule { /// `A p B` ∧ `B p C` ⟹ `A p C` Transitive, @@ -379,6 +379,248 @@ pub fn validity( Some(acc) } +// ===================== 矛盾：派生撞上了什么（0017） ===================== + +/// 一条派生撞上了一条断言。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Clash { + /// `Derivation::facts` 里的下标 + pub derived: usize, + /// 撞在哪条公理上：`Functional`（含 inverse_functional）、`Asymmetry`、`SelfLoop` + pub axiom: Kind, + /// 被撞的断言。自环没有对方，取派生的最后一条前提 + pub against: Uuid, +} + +/// 两条规则加在一起产出了互相矛盾的派生。 +/// +/// **按规则对聚合，不逐对报**：`ceo_of ⊑ works_at` 加 `works_at` functional，每个有 +/// 两个 ceo 的组织就撞一对——根子是那两条声明，逐对进队列只会淹掉 Review。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuleClash { + /// (声明所在的谓词, 规则种类)，两条按 (谓词, 种类) 排过序，a ≤ b + pub a: (Uuid, Rule), + pub b: (Uuid, Rule), + pub axiom: Kind, + /// 互撞的派生对，按 `Derivation::facts` 的下标 + pub pairs: Vec<(usize, usize)>, +} + +/// 半开区间 `[from, to)`，两端可空 +type Span = (Option, Option); +/// (谓词, 一端) → 另一端的边：(另一端, 事实, 区间)。functional 两个方向各一份 +type ByEnd = HashMap<(Uuid, Uuid), Vec<(Uuid, Uuid, Span)>>; +/// 一条规则的身份：声明所在的谓词 + 规则种类 +type RuleSide = (Uuid, Rule); +/// 互撞的派生对，按 (规则 a, 规则 b, 撞在哪条公理上) 分组 +type Grouped = HashMap<(RuleSide, RuleSide, Kind), Vec<(usize, usize)>>; + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Contradictions { + pub with_assertions: Vec, + pub between_derivations: Vec, +} + +impl Contradictions { + /// 不该落地的派生下标：撞过断言的，和撞过别的派生的。**写图宁少勿错**（0002） + pub fn blocked(&self) -> HashSet { + let mut out: HashSet = self.with_assertions.iter().map(|c| c.derived).collect(); + for rc in &self.between_derivations { + for (i, j) in &rc.pairs { + out.insert(*i); + out.insert(*j); + } + } + out + } +} + +/// 拿公理量一遍派生：与断言撞的逐条列出，派生之间撞的按规则对聚合。 +/// +/// 只查四类——`functional`（含 inverse）、`asymmetric`、`irreflexive`——因为只有它们 +/// 能由**两条边**判出矛盾；传递环那类要走闭包，派生本身就是闭包的一部分，R0 对断言 +/// 查过就够了。functional 与 asymmetric 都要求**有效区间重叠**：Mira 走了 Devin +/// 接任，两条 `ceo_of` 区间不交，那是接任，不是矛盾。 +/// +/// 撞上断言的派生一律**不落地**（asserted > derived，硬性）；这一步把「让路」这件事 +/// 从静默变成可见——0002 那张表里写了没做的那一行。 +pub fn contradictions( + derivation: &Derivation, + edges: &[TimedEdge], + axioms: &HashMap, + spans: &HashMap, Option)>, +) -> Contradictions { + // 断言的三份索引：(谓词, 主) → 宾；(谓词, 宾) → 主；(谓词, 主, 宾) → 边 + let mut by_ps: ByEnd = HashMap::new(); + let mut by_po: ByEnd = HashMap::new(); + let mut by_spo: HashMap<(Uuid, Uuid, Uuid), Vec<(Uuid, Span)>> = HashMap::new(); + for e in edges { + let span = (e.from, e.to); + let x = e.edge; + by_ps + .entry((x.predicate, x.subject)) + .or_default() + .push((x.object, x.fact, span)); + by_po + .entry((x.predicate, x.object)) + .or_default() + .push((x.subject, x.fact, span)); + by_spo + .entry((x.predicate, x.subject, x.object)) + .or_default() + .push((x.fact, span)); + } + + let mut out = Contradictions::default(); + // 派生的区间：与落库那一侧同一个函数算，算不出的（前提区间不交）本来就不会落 + let derived_spans: Vec> = derivation + .facts + .iter() + .map(|d| validity(&d.premises, spans)) + .collect(); + + for (i, d) in derivation.facts.iter().enumerate() { + let Some(span) = derived_spans[i] else { + continue; + }; + let Some(ax) = axioms.get(&d.predicate) else { + continue; + }; + let Some(&last) = d.premises.last() else { + continue; + }; + if ax.irreflexive && d.subject == d.object { + out.with_assertions.push(Clash { + derived: i, + axiom: Kind::SelfLoop, + against: last, + }); + } + if ax.asymmetric { + if let Some(v) = by_spo.get(&(d.predicate, d.object, d.subject)) { + for (fact, sp) in v { + if overlap(span, *sp).is_some() { + out.with_assertions.push(Clash { + derived: i, + axiom: Kind::Asymmetry, + against: *fact, + }); + } + } + } + } + if ax.functional { + if let Some(v) = by_ps.get(&(d.predicate, d.subject)) { + for (obj, fact, sp) in v { + if *obj != d.object && overlap(span, *sp).is_some() { + out.with_assertions.push(Clash { + derived: i, + axiom: Kind::Functional, + against: *fact, + }); + } + } + } + } + if ax.inverse_functional { + if let Some(v) = by_po.get(&(d.predicate, d.object)) { + for (subj, fact, sp) in v { + if *subj != d.subject && overlap(span, *sp).is_some() { + out.with_assertions.push(Clash { + derived: i, + axiom: Kind::Functional, + against: *fact, + }); + } + } + } + } + } + + // 派生之间：同样三份索引，只不过键的是下标 + let mut d_ps: HashMap<(Uuid, Uuid), Vec> = HashMap::new(); + let mut d_po: HashMap<(Uuid, Uuid), Vec> = HashMap::new(); + let mut d_spo: HashMap<(Uuid, Uuid, Uuid), Vec> = HashMap::new(); + for (i, d) in derivation.facts.iter().enumerate() { + if derived_spans[i].is_none() { + continue; + } + d_ps.entry((d.predicate, d.subject)).or_default().push(i); + d_po.entry((d.predicate, d.object)).or_default().push(i); + d_spo + .entry((d.predicate, d.subject, d.object)) + .or_default() + .push(i); + } + let mut grouped: Grouped = HashMap::new(); + let mut note = |i: usize, j: usize, axiom: Kind| { + let (i, j) = if i < j { (i, j) } else { (j, i) }; + let ri = (derivation.facts[i].via, derivation.facts[i].rule); + let rj = (derivation.facts[j].via, derivation.facts[j].rule); + let (a, b) = if (ri.0, ri.1.as_str()) <= (rj.0, rj.1.as_str()) { + (ri, rj) + } else { + (rj, ri) + }; + grouped.entry((a, b, axiom)).or_default().push((i, j)); + }; + for (i, d) in derivation.facts.iter().enumerate() { + let Some(span) = derived_spans[i] else { + continue; + }; + let Some(ax) = axioms.get(&d.predicate) else { + continue; + }; + let overlapping = |j: usize| derived_spans[j].is_some_and(|s| overlap(span, s).is_some()); + if ax.asymmetric { + if let Some(v) = d_spo.get(&(d.predicate, d.object, d.subject)) { + for &j in v { + if j > i && overlapping(j) { + note(i, j, Kind::Asymmetry); + } + } + } + } + if ax.functional { + if let Some(v) = d_ps.get(&(d.predicate, d.subject)) { + for &j in v { + if j > i && derivation.facts[j].object != d.object && overlapping(j) { + note(i, j, Kind::Functional); + } + } + } + } + if ax.inverse_functional { + if let Some(v) = d_po.get(&(d.predicate, d.object)) { + for &j in v { + if j > i && derivation.facts[j].subject != d.subject && overlapping(j) { + note(i, j, Kind::Functional); + } + } + } + } + } + let mut rule_clashes: Vec = grouped + .into_iter() + .map(|((a, b, axiom), mut pairs)| { + pairs.sort_unstable(); + pairs.dedup(); + RuleClash { a, b, axiom, pairs } + }) + .collect(); + // 输出排过序——这条路的价值有一半在确定性 + rule_clashes.sort_by(|x, y| { + (x.a.0, x.a.1.as_str(), x.b.0, x.b.1.as_str()).cmp(&( + y.a.0, + y.a.1.as_str(), + y.b.0, + y.b.1.as_str(), + )) + }); + out.between_derivations = rule_clashes; + out +} + #[cfg(test)] mod tests { use super::*; @@ -796,4 +1038,163 @@ mod tests { let d = derive(&[ep(P, 1, 1, 1)], &ax); assert!(d.facts.is_empty(), "`A p A` 的逆还是 `A p A`——自环不推"); } + + // ---------- 矛盾（0017） ---------- + + /// 指定谓词的一条带区间的边 + fn et(pred: Uuid, fact: u8, s: u8, o: u8, from: Option, to: Option) -> TimedEdge { + TimedEdge { + edge: Edge { + fact: f(fact), + predicate: pred, + subject: n(s), + object: n(o), + }, + from, + to, + } + } + + fn spans_of(edges: &[TimedEdge]) -> HashMap, Option)> { + edges + .iter() + .map(|e| (e.edge.fact, (e.from, e.to))) + .collect() + } + + /// `ceo_of ⊑ works_at`，works_at functional：Mira 的 ceo_of 推出 works_at Acme， + /// 而账本里说她 works_at Globex——派生撞上断言，指名道姓 + #[test] + fn a_derivation_that_breaks_functional_names_the_assertion_it_hit() { + let ax = HashMap::from([ + ( + P, + Axioms { + sub_property_of: Some(Q), + ..Default::default() + }, + ), + ( + Q, + Axioms { + functional: true, + ..Default::default() + }, + ), + ]); + let edges = [ep(P, 1, 1, 2), ep(Q, 2, 1, 3)]; + let d = derive(&edges, &ax); + assert_eq!(d.facts.len(), 1); + let c = contradictions(&d, &edges, &ax, &spans_of(&edges)); + assert_eq!( + c.with_assertions, + vec![Clash { + derived: 0, + axiom: Kind::Functional, + against: f(2) + }] + ); + assert!(c.between_derivations.is_empty()); + assert_eq!(c.blocked(), HashSet::from([0])); + } + + /// 区间不交就不是矛盾：前任与继任 + #[test] + fn disjoint_intervals_are_succession_and_stay_silent() { + let ax = HashMap::from([ + ( + P, + Axioms { + sub_property_of: Some(Q), + ..Default::default() + }, + ), + ( + Q, + Axioms { + functional: true, + ..Default::default() + }, + ), + ]); + let edges = [ + et(P, 1, 1, 2, Some(10), Some(20)), + et(Q, 2, 1, 3, Some(30), None), + ]; + let d = derive(&edges, &ax); + let c = contradictions(&d, &edges, &ax, &spans_of(&edges)); + assert!(c.with_assertions.is_empty(), "{c:?}"); + } + + /// 对称与非对称：`A p B` 对称推出 `B p A`，而 p 又声明 asymmetric—— + /// 每条断言都撞上自己的镜像 + #[test] + fn a_symmetric_derivation_hits_the_asymmetric_assertion() { + let ax = HashMap::from([( + P, + Axioms { + symmetric: true, + asymmetric: true, + ..Default::default() + }, + )]); + let edges = [ep(P, 1, 1, 2)]; + let d = derive(&edges, &ax); + let c = contradictions(&d, &edges, &ax, &spans_of(&edges)); + assert_eq!(c.with_assertions.len(), 1); + assert_eq!(c.with_assertions[0].axiom, Kind::Asymmetry); + assert_eq!(c.with_assertions[0].against, f(1)); + } + + /// 两条派生互撞时按规则对聚合，而且都不落地 + #[test] + fn derivations_that_disagree_are_grouped_by_the_rules_that_made_them() { + let ax = HashMap::from([ + ( + P, + Axioms { + sub_property_of: Some(Q), + ..Default::default() + }, + ), + ( + Q, + Axioms { + functional: true, + ..Default::default() + }, + ), + ]); + // 1 ceo_of 2 与 1 ceo_of 3：两条 works_at 由同一条规则推出，互相排斥 + let edges = [ep(P, 1, 1, 2), ep(P, 2, 1, 3), ep(P, 3, 4, 5)]; + let d = derive(&edges, &ax); + assert_eq!(d.facts.len(), 3); + let c = contradictions(&d, &edges, &ax, &spans_of(&edges)); + assert!(c.with_assertions.is_empty()); + assert_eq!(c.between_derivations.len(), 1); + let rc = &c.between_derivations[0]; + assert_eq!(rc.a, (P, Rule::SubProperty)); + assert_eq!(rc.b, (P, Rule::SubProperty)); + assert_eq!(rc.axiom, Kind::Functional); + assert_eq!(rc.pairs.len(), 1); + // 第三条（4 works_at 5）没跟谁撞，照常落地 + assert_eq!(c.blocked().len(), 2); + assert!(!c.blocked().contains(&2)); + } + + /// 谓词上没有公理就没有矛盾可言 + #[test] + fn a_predicate_without_axioms_cannot_contradict() { + let ax = HashMap::from([( + P, + Axioms { + sub_property_of: Some(Q), + ..Default::default() + }, + )]); + let edges = [ep(P, 1, 1, 2), ep(Q, 2, 1, 3)]; + let d = derive(&edges, &ax); + let c = contradictions(&d, &edges, &ax, &spans_of(&edges)); + assert_eq!(c, Contradictions::default()); + } } diff --git a/crates/utopia-reason/src/lib.rs b/crates/utopia-reason/src/lib.rs index b12b6528d..b6c3ca5cc 100644 --- a/crates/utopia-reason/src/lib.rs +++ b/crates/utopia-reason/src/lib.rs @@ -67,7 +67,7 @@ pub struct Violation { pub path: Vec, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Kind { /// `A p A`，而 p 声明了 irreflexive SelfLoop, @@ -85,6 +85,11 @@ pub enum Kind { /// 是为了与其它四类走同一条落库、清陈、裁决的路——left 与 right 同一条事实， /// 与自反那类同款 Signature, + /// 一条派生撞上了一条断言（0017）：推出来的 `A p B` 与账本里的某条断言在 p 的 + /// 公理上不能并存。派生不落地，这一行把它摆到人面前。`left` 是被撞的断言， + /// `right` 是派生的最后一条前提，`path` 是全部前提；推出来的三元组本身在 + /// `axiom_violations.detail` 里——它没有落库，没有 id 可指 + DerivedContradiction, } impl Kind { @@ -95,6 +100,7 @@ impl Kind { Kind::Cycle => "cycle", Kind::Functional => "functional", Kind::Signature => "signature", + Kind::DerivedContradiction => "derived_contradiction", } } } diff --git a/crates/utopia-server/src/api/review_routes.rs b/crates/utopia-server/src/api/review_routes.rs index ba1a619e7..c6fc951d2 100644 --- a/crates/utopia-server/src/api/review_routes.rs +++ b/crates/utopia-server/src/api/review_routes.rs @@ -497,8 +497,11 @@ pub async fn decide_mapping( #[derive(Deserialize)] pub struct DecideViolationReq { - /// fact_retracted | axiom_relaxed | accepted + /// fact_retracted | fact_closed | axiom_relaxed | accepted pub resolution: String, + /// `fact_closed` 必填：旧断言在哪一天结束 + #[serde(default)] + pub close_at: Option>, } /// 人裁决一处公理违规。 @@ -518,16 +521,101 @@ pub async fn decide_violation( require_kb(&state, &user, kb_id, Role::Editor).await?; if !matches!( req.resolution.as_str(), - "fact_retracted" | "axiom_relaxed" | "accepted" + "fact_retracted" | "fact_closed" | "axiom_relaxed" | "accepted" ) { return Err(utopia_core::AppError::invalid( "bad_resolution", - "resolution 只能是 fact_retracted、axiom_relaxed 或 accepted", + "resolution 只能是 fact_retracted、fact_closed、axiom_relaxed 或 accepted", ) .into()); } + let row: Option<(String, Uuid)> = sqlx::query_as( + "SELECT kind, left_fact FROM axiom_violations + WHERE id = $1 AND kb_id = $2 AND status = 'open'", + ) + .bind(violation_id) + .bind(kb_id) + .fetch_optional(&state.pool) + .await + .map_err(utopia_core::AppError::Db)?; + let Some((kind, left)) = row else { + return Err(utopia_core::AppError::NotFound.into()); + }; + // 派生撞断言那一类（0017）的修法就在卡片上，端点替人执行：撤旧断言、或给它一个 + // 结束日期。其它几类仍只记决定——那些卡片上两条都是断言，撤哪条端点判不了 + let repaired = kind == "derived_contradiction"; + match (repaired, req.resolution.as_str()) { + (true, "fact_retracted") => { + let snap = fact_snapshot(&state, kb_id, left).await; + utopia_store::graph::reject_fact(&state.pool, kb_id, left).await?; + if let Some(d) = snap { + let _ = utopia_store::audit::record( + &state.pool, + Some(kb_id), + user.id, + "fact.reject", + "fact", + Some(left), + d, + ) + .await; + } + } + (true, "fact_closed") => { + let Some(at) = req.close_at else { + return Err(utopia_core::AppError::invalid( + "close_at_required", + "fact_closed 要给出结束日期", + ) + .into()); + }; + let open: Option<(Uuid,)> = sqlx::query_as( + "SELECT id FROM facts + WHERE id = $1 AND invalidated_at IS NULL AND valid_to IS NULL", + ) + .bind(left) + .fetch_optional(&state.pool) + .await + .map_err(utopia_core::AppError::Db)?; + if open.is_none() { + return Err(utopia_core::AppError::invalid( + "not_open", + "这条断言已有结束日期，或已被撤", + ) + .into()); + } + let snap = fact_snapshot(&state, kb_id, left).await; + utopia_store::temporal::close_superseded(&state.pool, left, at, "day").await?; + if let Some(mut d) = snap { + d["valid_to"] = json!(at.to_rfc3339()); + let _ = utopia_store::audit::record( + &state.pool, + Some(kb_id), + user.id, + "fact.close", + "fact", + Some(left), + d, + ) + .await; + } + } + (false, "fact_closed") => { + return Err(utopia_core::AppError::invalid( + "bad_resolution", + "fact_closed 只用于 derived_contradiction", + ) + .into()); + } + _ => {} + } utopia_store::reasoning::decide(&state.pool, kb_id, violation_id, &req.resolution, user.id) .await?; + // 路清了就让派生落地，人不必再去点一次「推一遍」。撤与闭合把断言挪开了， + // 认可则在 materialize 里放行 + if repaired { + utopia_store::reasoning::materialize(&state.pool, kb_id).await?; + } let _ = utopia_store::audit::record( &state.pool, Some(kb_id), diff --git a/crates/utopia-store/src/reasoning.rs b/crates/utopia-store/src/reasoning.rs index 37c42f54a..1b9764aac 100644 --- a/crates/utopia-store/src/reasoning.rs +++ b/crates/utopia-store/src/reasoning.rs @@ -12,13 +12,14 @@ //! 提案刷回待看，等于每跑一次就把人的否决抹掉一次。所以这里 `ON CONFLICT` //! 什么都不做——已经在库里的那一行，无论 open 还是 resolved，都按原样留着。 +use serde_json::json; use sqlx::PgPool; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use utopia_core::models::{AxiomViolation, DerivedFactView, OntologyDefect}; /// 规则种类的字面量。用 &'static str 而不是枚举:它直接进 SQL 也直接做键 type RuleKind = &'static str; use utopia_core::AppResult; -use utopia_reason::derive::TimedEdge; +use utopia_reason::derive::{Contradictions, Derivation, TimedEdge}; use utopia_reason::{check, Axioms, Edge, Kind, Violation}; use uuid::Uuid; @@ -35,37 +36,17 @@ pub struct Report { pub inserted: usize, /// 清掉的陈旧 open 行 pub cleared: usize, + /// 派生撞上断言的条数（0017），已含在 `found` 里 + pub contradictions: usize, + /// 撞上单谓词上限、没进队列的矛盾条数。**不为零时说明根子在规则**： + /// 一条谓词上上百条派生都撞了，逐条看是没有意义的 + pub contradictions_capped: usize, + /// 互撞的规则对数——进 `ontology_defects`，不进这张表 + pub rules_disagree: usize, } -/// 取这个库里所有能参与检查的边。 -/// -/// 三个过滤条件都是必要的： -/// -/// - `invalidated_at IS NULL`——被推翻的事实不该再报矛盾，它已经不是我们的断言了 -/// - `predicate_id IS NOT NULL`——没有谓词就没有公理可依（见 `facts.predicate_id`） -/// - `object_id IS NOT NULL`——属性事实的宾语是字面值，公理谈的是实体之间的关系 -async fn edges(pool: &PgPool, kb_id: Uuid) -> AppResult> { - let rows: Vec<(Uuid, Uuid, Uuid, Uuid)> = sqlx::query_as( - "SELECT id, predicate_id, subject_id, object_id - FROM facts - WHERE kb_id = $1 - AND invalidated_at IS NULL - AND predicate_id IS NOT NULL - AND object_id IS NOT NULL", - ) - .bind(kb_id) - .fetch_all(pool) - .await?; - Ok(rows - .into_iter() - .map(|(fact, predicate, subject, object)| Edge { - fact, - predicate, - subject, - object, - }) - .collect()) -} +/// 单个谓词上进队列的矛盾上限（0017 §1）。超出的部分只计数。 +const MAX_CLASHES_PER_PREDICATE: usize = 50; /// 取这个库的谓词公理。 /// @@ -229,7 +210,8 @@ pub async fn record_signature_breaks( /// 跑一遍检查，把结果落库。 pub async fn run(pool: &PgPool, kb_id: Uuid) -> AppResult { - let edges = edges(pool, kb_id).await?; + let (timed, spans, _) = timed_edges(pool, kb_id).await?; + let edges: Vec = timed.iter().map(|t| t.edge).collect(); let axioms = axioms(pool, kb_id).await?; let mut violations = check(&edges, &axioms); // 第五类不在纯逻辑引擎里：它要看实体的类型与谓词的 domain / range，那是库里的 @@ -243,10 +225,63 @@ pub async fn run(pool: &PgPool, kb_id: Uuid) -> AppResult { }); } + // 第六类（0017）：推出来却落不了地的派生。与 `materialize` 用同一个函数算， + // 所以这里报的正是那边拦下的——两边各算一套的话，队列会跟图对不上 + let derivation = utopia_reason::derive::derive(&timed, &axioms); + let clashes = utopia_reason::derive::contradictions(&derivation, &timed, &axioms, &spans); + let names = names_for(pool, &derivation, &clashes).await?; + let mut details: HashMap<(Uuid, Uuid), serde_json::Value> = HashMap::new(); + let mut per_pred: HashMap = HashMap::new(); + let mut contradictions_capped = 0usize; + for c in &clashes.with_assertions { + let d = &derivation.facts[c.derived]; + let Some(&last) = d.premises.last() else { + continue; + }; + let key = (c.against, last); + if details.contains_key(&key) { + continue; + } + let n = per_pred.entry(d.predicate).or_default(); + if *n >= MAX_CLASHES_PER_PREDICATE { + contradictions_capped += 1; + continue; + } + *n += 1; + let span = utopia_reason::derive::validity(&d.premises, &spans); + details.insert( + key, + json!({ + "axiom": c.axiom.as_str(), + "rule": d.rule.as_str(), + "via": d.via, + "via_label": names.predicate(d.via), + "subject_id": d.subject, + "subject": names.entity(d.subject), + "predicate_id": d.predicate, + "predicate": names.predicate(d.predicate), + "object_id": d.object, + "object": names.entity(d.object), + "valid_from": span.and_then(|s| s.0).map(|t| stamp(t).to_rfc3339()), + "valid_to": span.and_then(|s| s.1).map(|t| stamp(t).to_rfc3339()), + "premises": d.premises, + }), + ); + violations.push(Violation { + kind: Kind::DerivedContradiction, + left: c.against, + right: last, + path: d.premises.clone(), + }); + } + let mut report = Report { edges: edges.len(), predicates_with_axioms: axioms.len(), found: violations.len(), + contradictions: details.len(), + contradictions_capped, + rules_disagree: clashes.between_derivations.len(), ..Default::default() }; @@ -261,9 +296,13 @@ pub async fn run(pool: &PgPool, kb_id: Uuid) -> AppResult { right, path, } = v; + let detail = details + .get(&(*left, *right)) + .cloned() + .unwrap_or_else(|| json!({})); let id: Option<(Uuid,)> = sqlx::query_as( - "INSERT INTO axiom_violations (id, kb_id, kind, left_fact, right_fact, path) - VALUES ($1, $2, $3, $4, $5, $6) + "INSERT INTO axiom_violations (id, kb_id, kind, left_fact, right_fact, path, detail) + VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (kb_id, kind, left_fact, right_fact) DO NOTHING RETURNING id", ) @@ -273,6 +312,7 @@ pub async fn run(pool: &PgPool, kb_id: Uuid) -> AppResult { .bind(left) .bind(right) .bind(path) + .bind(&detail) .fetch_optional(&mut *tx) .await?; if id.is_some() { @@ -303,10 +343,144 @@ pub async fn run(pool: &PgPool, kb_id: Uuid) -> AppResult { .execute(&mut *tx) .await?; report.cleared = cleared.rows_affected() as usize; + + // 派生之间互撞的按规则对进 `ontology_defects`——根子是那两条声明，不是哪条事实。 + // 同一对谓词上可能有几种撞法（functional 与 asymmetric 各撞各的），唯一键只到 + // 谓词对，所以合成一行，几种撞法都写进 detail + let mut by_pair: HashMap<(Uuid, Uuid), Vec> = HashMap::new(); + let mut order: Vec<(Uuid, Uuid)> = Vec::new(); + for rc in &clashes.between_derivations { + let triple = |i: usize| { + let d = &derivation.facts[i]; + format!( + "{} · {} · {}", + names.entity(d.subject), + names.predicate(d.predicate), + names.entity(d.object) + ) + }; + let examples: Vec = rc + .pairs + .iter() + .take(3) + .map(|(i, j)| json!([triple(*i), triple(*j)])) + .collect(); + let key = (rc.a.0, rc.b.0); + if !by_pair.contains_key(&key) { + order.push(key); + } + by_pair.entry(key).or_default().push(json!({ + "rule_a": rc.a.1.as_str(), + "via_a": names.predicate(rc.a.0), + "rule_b": rc.b.1.as_str(), + "via_b": names.predicate(rc.b.0), + "axiom": rc.axiom.as_str(), + "count": rc.pairs.len(), + "examples": examples, + })); + } + let mut fresh_defects: Vec = Vec::with_capacity(order.len()); + for key in order { + let rules = by_pair.remove(&key).unwrap_or_default(); + let count: usize = rules + .iter() + .map(|r| r["count"].as_u64().unwrap_or(0) as usize) + .sum(); + // 已经有人认可过的那一行保持 resolved，只刷 detail：0017 说认可之后不再报 + let (id,): (Uuid,) = sqlx::query_as( + "INSERT INTO ontology_defects (id, kb_id, kind, subject, other, path, detail) + VALUES ($1, $2, 'rules_disagree', $3, $4, '{}', $5) + ON CONFLICT (kb_id, kind, subject, other) DO UPDATE SET detail = EXCLUDED.detail + RETURNING id", + ) + .bind(Uuid::now_v7()) + .bind(kb_id) + .bind(key.0) + .bind(key.1) + .bind(json!({ "count": count, "rules": rules })) + .fetch_one(&mut *tx) + .await?; + fresh_defects.push(id); + } + sqlx::query( + "DELETE FROM ontology_defects + WHERE kb_id = $1 AND kind = 'rules_disagree' AND status = 'open' + AND NOT (id = ANY($2))", + ) + .bind(kb_id) + .bind(&fresh_defects) + .execute(&mut *tx) + .await?; tx.commit().await?; Ok(report) } +/// 矛盾要写成人能读的话，而派生没有落库、没有文本可查——名字在这里补。 +struct Names { + entities: HashMap, + predicates: HashMap, +} + +impl Names { + fn entity(&self, id: Uuid) -> String { + self.entities + .get(&id) + .cloned() + .unwrap_or_else(|| "?".into()) + } + fn predicate(&self, id: Uuid) -> String { + self.predicates + .get(&id) + .cloned() + .unwrap_or_else(|| "?".into()) + } +} + +async fn names_for( + pool: &PgPool, + derivation: &Derivation, + clashes: &Contradictions, +) -> AppResult { + let mut ents: HashSet = HashSet::new(); + let mut preds: HashSet = HashSet::new(); + let mut want = |i: usize| { + let d = &derivation.facts[i]; + ents.insert(d.subject); + ents.insert(d.object); + preds.insert(d.predicate); + preds.insert(d.via); + }; + for c in &clashes.with_assertions { + want(c.derived); + } + for rc in &clashes.between_derivations { + for (i, j) in rc.pairs.iter().take(3) { + want(*i); + want(*j); + } + } + for rc in &clashes.between_derivations { + preds.insert(rc.a.0); + preds.insert(rc.b.0); + } + let ents: Vec = ents.into_iter().collect(); + let preds: Vec = preds.into_iter().collect(); + let entities: Vec<(Uuid, String)> = + sqlx::query_as("SELECT id, canonical_name FROM entities WHERE id = ANY($1)") + .bind(&ents) + .fetch_all(pool) + .await?; + let predicates: Vec<(Uuid, String)> = + sqlx::query_as("SELECT id, label FROM relation_types WHERE id = ANY($1)") + .bind(&preds) + .fetch_all(pool) + .await?; + Ok(Names { + entities: entities.into_iter().collect(), + predicates: predicates.into_iter().collect(), + }) +} + /// Review 页要看的:还没人表态的违规,连同两条事实的三元组文本。 /// /// 展开成文本在 SQL 里做而不是回来再查一遍:一页几十条,每条两个三元组, @@ -336,10 +510,20 @@ pub async fn open_violations( v.left_fact, l.text AS left_text, v.right_fact, rt.text AS right_text, coalesce(array_length(v.path, 1), 0) AS path_len, - v.detected_at + v.detected_at, v.detail, + lf.valid_to IS NULL AS left_open, + lf.confidence AS left_confidence, + EXISTS ( + SELECT 1 FROM entities e + JOIN entities x ON x.kb_id = e.kb_id AND x.id <> e.id + AND x.merged_into IS NULL + AND lower(x.canonical_name) = lower(e.canonical_name) + WHERE e.id IN (lf.subject_id, lf.object_id) + ) AS same_name_peers FROM axiom_violations v JOIN triple l ON l.id = v.left_fact JOIN triple rt ON rt.id = v.right_fact + JOIN facts lf ON lf.id = v.left_fact WHERE v.kb_id = $1 AND v.status = 'open' ORDER BY v.detected_at DESC LIMIT $2 OFFSET $3", @@ -348,7 +532,60 @@ pub async fn open_violations( .bind(limit) .bind(offset) .fetch_all(pool) - .await?) + .await? + .into_iter() + .map(|r: ViolationRow| { + let hint = if r.kind == "derived_contradiction" { + hint_for(&r).map(String::from) + } else { + None + }; + AxiomViolation { + id: r.id, + kind: r.kind, + predicate: r.predicate, + left_fact: r.left_fact, + left_text: r.left_text, + right_fact: r.right_fact, + right_text: r.right_text, + path_len: r.path_len, + detected_at: r.detected_at, + detail: r.detail, + hint, + } + }) + .collect()) +} + +#[derive(sqlx::FromRow)] +struct ViolationRow { + id: Uuid, + kind: String, + predicate: Option, + left_fact: Uuid, + left_text: String, + right_fact: Uuid, + right_text: String, + path_len: i32, + detected_at: chrono::DateTime, + detail: serde_json::Value, + left_open: bool, + left_confidence: f32, + same_name_peers: bool, +} + +/// 线索按最常见的错法排（0017 §2）：旧断言没写结束日期、两个同名实体、抽取本来就 +/// 没把握。一次只给一条——三条并列等于没给 +fn hint_for(r: &ViolationRow) -> Option<&'static str> { + if r.left_open && r.detail.get("valid_from").is_some_and(|v| !v.is_null()) { + Some("stale") + } else if r.same_name_peers { + Some("duplicate") + } else if r.left_confidence < 0.75 { + Some("unsure") + } else { + None + } } /// 人裁决一处违规。 @@ -463,7 +700,8 @@ pub async fn check_ontology(pool: &PgPool, kb_id: Uuid) -> AppResult 'rules_disagree' + AND NOT (id = ANY($2))", ) .bind(kb_id) .bind(&fresh) @@ -497,6 +735,84 @@ pub struct DeriveReport { /// 而下游靠它推出的 `employs` 反倒进了库——一条派生的前提凭空消失。 /// 数出来，别再让它静默一次 pub unruled: usize, + /// 推出来了却撞上断言或别的派生、这一轮拦下没落的（0017）。**拦下的每一条 + /// 都在 Review 里有对应的一行**——`run` 与这里用同一个函数算 + pub blocked: usize, +} + +/// 一次取数，三样东西：带区间的边、每条事实的区间、精度与置信度。 +/// `run` 与 `materialize` 共用——两边看到的边必须是同一批 +type TimedEdges = ( + Vec, + HashMap, Option)>, + HashMap, Option, f32)>, +); + +async fn timed_edges(pool: &PgPool, kb_id: Uuid) -> AppResult { + // 输入**只有断言**。派生住在另一张表，所以这里连过滤都不必写——那正是 + // 分表买到的东西：忘了排除的后果是推不出东西，不是把自己的输出喂回自己 + let rows: Vec = sqlx::query_as( + "SELECT id, predicate_id, subject_id, object_id, + valid_from, valid_to, valid_from_precision, valid_to_precision, confidence + FROM facts + WHERE kb_id = $1 + AND invalidated_at IS NULL + AND predicate_id IS NOT NULL + AND object_id IS NOT NULL", + ) + .bind(kb_id) + .fetch_all(pool) + .await?; + + let mut edges = Vec::with_capacity(rows.len()); + let mut meta: HashMap, Option, f32)> = HashMap::new(); + let mut spans: HashMap, Option)> = HashMap::new(); + for (id, pred, subj, obj, from, to, fp, tp, conf) in rows { + let (f, t) = (from.map(|x| x.timestamp()), to.map(|x| x.timestamp())); + edges.push(TimedEdge { + edge: Edge { + fact: id, + predicate: pred, + subject: subj, + object: obj, + }, + from: f, + to: t, + }); + spans.insert(id, (f, t)); + meta.insert(id, (fp, tp, conf)); + } + Ok((edges, spans, meta)) +} + +/// 人认可过并存的（派生三元组, 断言）对：这些派生下一轮照常落地（0017 §2）。 +async fn accepted_clashes( + pool: &PgPool, + kb_id: Uuid, +) -> AppResult> { + let rows: Vec<(Uuid, serde_json::Value)> = sqlx::query_as( + "SELECT left_fact, detail FROM axiom_violations + WHERE kb_id = $1 AND kind = 'derived_contradiction' AND resolution = 'accepted'", + ) + .bind(kb_id) + .fetch_all(pool) + .await?; + let id = |v: &serde_json::Value, k: &str| { + v.get(k) + .and_then(|x| x.as_str()) + .and_then(|s| s.parse::().ok()) + }; + Ok(rows + .into_iter() + .filter_map(|(against, d)| { + Some(( + id(&d, "subject_id")?, + id(&d, "predicate_id")?, + id(&d, "object_id")?, + against, + )) + }) + .collect()) } /// 派生事实的身份：三元组 + 区间。 @@ -610,52 +926,40 @@ type LiveRow = ( pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult { let ax = axioms(pool, kb_id).await?; let rules = compile_rules(pool, kb_id, &ax).await?; - - // 输入**只有断言**。派生住在另一张表，所以这里连过滤都不必写——那正是 - // 分表买到的东西：忘了排除的后果是推不出东西，不是把自己的输出喂回自己 - let rows: Vec = sqlx::query_as( - "SELECT id, predicate_id, subject_id, object_id, - valid_from, valid_to, valid_from_precision, valid_to_precision, confidence - FROM facts - WHERE kb_id = $1 - AND invalidated_at IS NULL - AND predicate_id IS NOT NULL - AND object_id IS NOT NULL", - ) - .bind(kb_id) - .fetch_all(pool) - .await?; - - let mut edges = Vec::with_capacity(rows.len()); - let mut meta: HashMap, Option, f32)> = HashMap::new(); - let mut spans: HashMap, Option)> = HashMap::new(); - for (id, pred, subj, obj, from, to, fp, tp, conf) in rows { - let (f, t) = (from.map(|x| x.timestamp()), to.map(|x| x.timestamp())); - edges.push(TimedEdge { - edge: Edge { - fact: id, - predicate: pred, - subject: subj, - object: obj, - }, - from: f, - to: t, - }); - spans.insert(id, (f, t)); - meta.insert(id, (fp, tp, conf)); - } + let (edges, spans, meta) = timed_edges(pool, kb_id).await?; let derivation = utopia_reason::derive::derive(&edges, &ax); + // asserted > derived 是硬性的（0002）：撞上断言的派生不落地。人认可过并存的 + // 除外；派生之间互撞的两边都不落，认可与否只影响报不报（0017） + let clashes = utopia_reason::derive::contradictions(&derivation, &edges, &ax, &spans); + let accepted = accepted_clashes(pool, kb_id).await?; + let mut blocked: HashSet = HashSet::new(); + for c in &clashes.with_assertions { + let d = &derivation.facts[c.derived]; + if !accepted.contains(&(d.subject, d.predicate, d.object, c.against)) { + blocked.insert(c.derived); + } + } + for rc in &clashes.between_derivations { + for (i, j) in &rc.pairs { + blocked.insert(*i); + blocked.insert(*j); + } + } let mut report = DeriveReport { rules: rules.len(), edges: edges.len(), derived: derivation.facts.len(), capped: derivation.capped.len(), + blocked: blocked.len(), ..Default::default() }; let mut wanted: HashMap = HashMap::new(); - for d in &derivation.facts { + for (i, d) in derivation.facts.iter().enumerate() { + if blocked.contains(&i) { + continue; + } let Some((from, to)) = utopia_reason::derive::validity(&d.premises, &spans) else { continue; }; @@ -776,9 +1080,9 @@ pub async fn open_defects( offset: i64, ) -> AppResult> { Ok(sqlx::query_as( - "SELECT d.id, d.kind, + "SELECT d.id, d.kind, d.detail, COALESCE(st.label, sr.label) AS subject_label, - ot.label AS other_label, + COALESCE(ot.label, orr.label) AS other_label, COALESCE( (SELECT array_agg(t.label ORDER BY x.ord) FROM unnest(d.path) WITH ORDINALITY AS x(id, ord) @@ -790,6 +1094,7 @@ pub async fn open_defects( LEFT JOIN entity_types st ON st.id = d.subject LEFT JOIN relation_types sr ON sr.id = d.subject LEFT JOIN entity_types ot ON ot.id = d.other + LEFT JOIN relation_types orr ON orr.id = d.other WHERE d.kb_id = $1 AND d.status = 'open' ORDER BY d.detected_at DESC LIMIT $2 OFFSET $3", diff --git a/crates/utopia-store/tests/a_contradiction_points_upstream.rs b/crates/utopia-store/tests/a_contradiction_points_upstream.rs new file mode 100644 index 000000000..2ff4dd13b --- /dev/null +++ b/crates/utopia-store/tests/a_contradiction_points_upstream.rs @@ -0,0 +1,341 @@ +//! 0017：派生撞上断言时，让路这件事从静默变成可见。 +//! +//! `ceo_of ⊑ works_at`，`works_at` functional。Mira `ceo_of` Acme 推出 Mira `works_at` +//! Acme，而账本里说她 `works_at` Globex。这里守四件事： +//! +//! 1. **派生不落地，而队列里有一行。** `run` 记一条 `derived_contradiction`，left 是被撞 +//! 的断言，right 是最后一条前提，detail 写着推出来的三元组；`materialize` 拦下它。 +//! 2. **修了就落。** 给旧断言一个结束日期，派生的区间与它不再重叠，下一轮落地， +//! 队列里那一行随之清掉。 +//! 3. **认可就落。** 人说两边都对，`accepted` 之后派生照常落地，那一行留着不再报。 +//! 4. **派生之间互撞按规则对聚合。** 两个 ceo 推出两条互斥的 works_at，进 +//! `ontology_defects` 一行 `rules_disagree`，两条派生都不落。 +//! +//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆，绝不碰已有的库。 + +use sqlx::PgPool; +use utopia_store::reasoning; +use uuid::Uuid; + +struct Fixture { + org: Uuid, + user: Uuid, + kb: Uuid, + ceo_of: Uuid, + works_at: Uuid, + mira: Uuid, + acme: Uuid, + globex: Uuid, + initech: Uuid, +} + +async fn seed(pool: &PgPool) -> anyhow::Result { + let (org, ws, kb, user) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + let etype = Uuid::now_v7(); + let (ceo_of, works_at) = (Uuid::now_v7(), Uuid::now_v7()); + let (mira, acme, globex, initech) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'contradiction-test')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'contradiction-test')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO users (id, org_id, email, display_name, password_hash) + VALUES ($1, $2, $1 || '@contradiction.test', 'c', 'x')", + ) + .bind(user) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases (id, workspace_id, name, materialize_inferences) + VALUES ($1, $2, 'contradiction-test', TRUE)", + ) + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, 'thing', 'Thing')", + ) + .bind(etype) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO relation_types (id, kb_id, key, label, functional) + VALUES ($1, $2, 'works_at', 'works at', TRUE)", + ) + .bind(works_at) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO relation_types (id, kb_id, key, label, sub_property_of) + VALUES ($1, $2, 'ceo_of', 'CEO of', $3)", + ) + .bind(ceo_of) + .bind(kb) + .bind(works_at) + .execute(pool) + .await?; + for (id, name) in [ + (mira, "Mira"), + (acme, "Acme"), + (globex, "Globex"), + (initech, "Initech"), + ] { + sqlx::query( + "INSERT INTO entities (id, kb_id, type_id, canonical_name) VALUES ($1, $2, $3, $4)", + ) + .bind(id) + .bind(kb) + .bind(etype) + .bind(name) + .execute(pool) + .await?; + } + Ok(Fixture { + org, + user, + kb, + ceo_of, + works_at, + mira, + acme, + globex, + initech, + }) +} + +async fn asserted( + pool: &PgPool, + f: &Fixture, + subject: Uuid, + predicate: Uuid, + object: Uuid, + from: Option<&str>, +) -> anyhow::Result { + let id = Uuid::now_v7(); + sqlx::query( + "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, confidence, + valid_from, valid_from_precision) + VALUES ($1, $2, $3, $4, $5, 0.9, $6::timestamptz, CASE WHEN $6 IS NULL THEN NULL ELSE 'day' END)", + ) + .bind(id) + .bind(f.kb) + .bind(subject) + .bind(predicate) + .bind(object) + .bind(from) + .execute(pool) + .await?; + Ok(id) +} + +async fn live_derived(pool: &PgPool, f: &Fixture) -> anyhow::Result> { + Ok(sqlx::query_as( + "SELECT subject_id, object_id FROM derived_facts + WHERE kb_id = $1 AND invalidated_at IS NULL ORDER BY subject_id, object_id", + ) + .bind(f.kb) + .fetch_all(pool) + .await?) +} + +async fn open_contradictions( + pool: &PgPool, + f: &Fixture, +) -> anyhow::Result> { + Ok(sqlx::query_as( + "SELECT id, left_fact, detail FROM axiom_violations + WHERE kb_id = $1 AND kind = 'derived_contradiction' AND status = 'open'", + ) + .bind(f.kb) + .fetch_all(pool) + .await?) +} + +#[tokio::test] +async fn a_contradiction_points_upstream() -> anyhow::Result<()> { + let Ok(url) = std::env::var("UTOPIA_DATABASE_URL") else { + eprintln!("跳过：未设 UTOPIA_DATABASE_URL"); + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let f = seed(&pool).await?; + + let run = async { + // Mira works_at Globex（没写结束日期）；Mira ceo_of Acme 自 2024 起 + let old = asserted(&pool, &f, f.mira, f.works_at, f.globex, Some("2020-01-01")).await?; + let ceo = asserted(&pool, &f, f.mira, f.ceo_of, f.acme, Some("2024-01-01")).await?; + + // 1. 派生不落地，队列里有一行 + let m = reasoning::materialize(&pool, f.kb).await?; + assert_eq!(m.derived, 1); + assert_eq!( + m.blocked, 1, + "the derivation that hits an assertion stays out" + ); + assert_eq!(m.inserted, 0); + assert!(live_derived(&pool, &f).await?.is_empty()); + + let r = reasoning::run(&pool, f.kb).await?; + assert_eq!(r.contradictions, 1); + assert_eq!(r.rules_disagree, 0); + let rows = open_contradictions(&pool, &f).await?; + assert_eq!(rows.len(), 1); + let (vid, left, detail) = &rows[0]; + assert_eq!(*left, old, "left is the assertion that was hit"); + assert_eq!(detail["axiom"], "functional"); + assert_eq!(detail["rule"], "sub_property"); + assert_eq!(detail["subject"], "Mira"); + assert_eq!(detail["predicate"], "works at"); + assert_eq!(detail["object"], "Acme"); + assert_eq!(detail["via_label"], "CEO of"); + assert_eq!(detail["premises"][0], serde_json::json!(ceo)); + let (right,): (Uuid,) = + sqlx::query_as("SELECT right_fact FROM axiom_violations WHERE id = $1") + .bind(vid) + .fetch_one(&pool) + .await?; + assert_eq!(right, ceo, "right is the last premise"); + + // Review 给的线索：旧断言没写结束日期、派生起得更晚 → stale + let page = reasoning::open_violations(&pool, f.kb, 50, 0).await?; + let card = page + .iter() + .find(|v| v.id == *vid) + .expect("card on the page"); + assert_eq!(card.hint.as_deref(), Some("stale")); + assert_eq!(card.detail["subject"], "Mira"); + + // 重跑幂等：还是那一行 + reasoning::run(&pool, f.kb).await?; + assert_eq!(open_contradictions(&pool, &f).await?.len(), 1); + + // 2. 修了就落：给旧断言一个结束日期，区间不再重叠 + sqlx::query( + "UPDATE facts SET valid_to = '2023-06-30'::timestamptz, valid_to_precision = 'day' + WHERE id = $1", + ) + .bind(old) + .execute(&pool) + .await?; + let m = reasoning::materialize(&pool, f.kb).await?; + assert_eq!(m.blocked, 0); + assert_eq!( + m.inserted, 1, + "once the assertion ends, the derivation lands" + ); + assert_eq!(live_derived(&pool, &f).await?, vec![(f.mira, f.acme)]); + reasoning::run(&pool, f.kb).await?; + assert!( + open_contradictions(&pool, &f).await?.is_empty(), + "the queue row clears with the contradiction" + ); + + // 3. 认可就落：把结束日期拿掉，矛盾回来；人说两边都对，派生照常落地 + sqlx::query("UPDATE facts SET valid_to = NULL, valid_to_precision = NULL WHERE id = $1") + .bind(old) + .execute(&pool) + .await?; + let m = reasoning::materialize(&pool, f.kb).await?; + assert_eq!(m.blocked, 1); + assert_eq!(m.invalidated, 1, "the landed derivation is withdrawn again"); + reasoning::run(&pool, f.kb).await?; + let rows = open_contradictions(&pool, &f).await?; + assert_eq!(rows.len(), 1); + reasoning::decide(&pool, f.kb, rows[0].0, "accepted", f.user).await?; + let m = reasoning::materialize(&pool, f.kb).await?; + assert_eq!(m.blocked, 0, "an accepted pair lands"); + assert_eq!(live_derived(&pool, &f).await?, vec![(f.mira, f.acme)]); + let r = reasoning::run(&pool, f.kb).await?; + assert_eq!(r.contradictions, 1, "still counted"); + assert!( + open_contradictions(&pool, &f).await?.is_empty(), + "but the accepted row stays resolved and nothing new is opened" + ); + + // 4. 派生之间互撞：先把旧断言闭合掉，让断言不再参与；再来一个 ceo_of Initech， + // 两条 works_at 由同一条规则推出、互斥——按规则对报一次，两条都不落 + sqlx::query( + "UPDATE facts SET valid_to = '2023-06-30'::timestamptz, valid_to_precision = 'day' + WHERE id = $1", + ) + .bind(old) + .execute(&pool) + .await?; + let ceo2 = asserted(&pool, &f, f.mira, f.ceo_of, f.initech, Some("2024-01-01")).await?; + let m = reasoning::materialize(&pool, f.kb).await?; + assert_eq!(m.derived, 2); + assert_eq!(m.blocked, 2, "both sides of a rule clash stay out"); + assert_eq!(m.invalidated, 1, "the one that had landed is withdrawn"); + assert!(live_derived(&pool, &f).await?.is_empty()); + let r = reasoning::run(&pool, f.kb).await?; + assert_eq!(r.contradictions, 0); + assert_eq!(r.rules_disagree, 1); + let defects: Vec<(Uuid, Option, serde_json::Value)> = sqlx::query_as( + "SELECT subject, other, detail FROM ontology_defects + WHERE kb_id = $1 AND kind = 'rules_disagree' AND status = 'open'", + ) + .bind(f.kb) + .fetch_all(&pool) + .await?; + assert_eq!(defects.len(), 1); + assert_eq!(defects[0].0, f.ceo_of); + assert_eq!(defects[0].1, Some(f.ceo_of)); + assert_eq!(defects[0].2["count"], 1); + assert_eq!(defects[0].2["rules"][0]["axiom"], "functional"); + assert_eq!(defects[0].2["rules"][0]["rule_a"], "sub_property"); + assert_eq!(defects[0].2["rules"][0]["via_a"], "CEO of"); + let page = reasoning::open_defects(&pool, f.kb, 50, 0).await?; + let card = page + .iter() + .find(|d| d.kind == "rules_disagree") + .expect("the rule clash is on the page"); + assert_eq!(card.subject_label.as_deref(), Some("CEO of")); + assert_eq!(card.other_label.as_deref(), Some("CEO of")); + + // 撤掉第二个 ceo：规则对的那一行清掉，第一条派生重新落地 + sqlx::query("UPDATE facts SET invalidated_at = now() WHERE id = $1") + .bind(ceo2) + .execute(&pool) + .await?; + reasoning::run(&pool, f.kb).await?; + let (n,): (i64,) = sqlx::query_as( + "SELECT count(*) FROM ontology_defects + WHERE kb_id = $1 AND kind = 'rules_disagree' AND status = 'open'", + ) + .bind(f.kb) + .fetch_one(&pool) + .await?; + assert_eq!(n, 0, "a rule clash clears when its derivations go"); + let m = reasoning::materialize(&pool, f.kb).await?; + assert_eq!(m.blocked, 0); + assert_eq!(live_derived(&pool, &f).await?, vec![(f.mira, f.acme)]); + anyhow::Ok(()) + } + .await; + + let _ = sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(f.org) + .execute(&pool) + .await; + run +} diff --git a/docs/decisions/0002-reasoning-engine.md b/docs/decisions/0002-reasoning-engine.md index 976eb23fb..d18a742bf 100644 --- a/docs/decisions/0002-reasoning-engine.md +++ b/docs/decisions/0002-reasoning-engine.md @@ -75,6 +75,8 @@ CREATE TABLE fact_derivations ( > **修订记录（2026-09-02）**：表格三行里只有第一行的前半兑现了——`asserted` 硬性优先，已断言的三元组不再派生（`derive.rs`）。 > 「记一条规则与事实矛盾的信号」**没有**，「派生 vs 派生进 Review」**没有**（同一三元组的多条推导只留第一条证明）。两者都还是待做。 +> +> **修订记录（2026-09-03）**：两行都兑现了，方案见 [0017](0017-a-contradiction-points-upstream.md)。派生撞断言：`derive::contradictions` 逐条算出，`run` 记成 `axiom_violations` 一种 `derived_contradiction`（单谓词封顶 50），`materialize` 用同一个函数拦下不落地；卡片给线索（旧断言没结束日期、同名实体、置信度低）与修法（闭合、撤回、认可）。派生撞派生：按规则对聚合进 `ontology_defects` 一种 `rules_disagree`，两边都不落——根子是那两条声明，不是哪条事实。 ### 3. 前提撤回时——双时态是最优解，不是负担 diff --git a/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md b/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md index 02fff13e2..e695fd251 100644 --- a/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md +++ b/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md @@ -67,7 +67,7 @@ Chat memory 与 MCP 两处在 A1 / A2 落地前标 in development。中英两份 ### B · 推理机补完——A 之后，与 C 并行 **B1 · R2 证明树。**〔已做，`feat/proof-tree`；实际是链不是树，理由见 0002 R2 的修订〕递归展开到叶子 chunk 的 API + 实体面板里可展开的树。数据结构（`fact_derivations`）已够。 -**B2 · 派生 vs 断言矛盾要有信号。** `axiom_violations` 加一种 kind（`derived_contradiction`），进 Review 同一档——0002 写了没做的那一行。 +**B2 · 派生 vs 断言矛盾要有信号。** `axiom_violations` 加一种 kind（`derived_contradiction`），进 Review 同一档——0002 写了没做的那一行。（完整方案见 [0017](0017-a-contradiction-points-upstream.md)；B2a 引擎与队列已做，B2b 图与面板上的可见性待做。） **B3 · `disjointWith` 进消解，合并路径复核签名。** `classify_type_drift` 改从 `entity_type_disjoint` 读（没声明就退回今天的行为，不硬编码）； `merge_entities` 搬事实前跑一遍与写入时相同的 domain / range 检查，违反的进 `axiom_violations` 而不是静默搬过去。这是 0009 与 0012 各自待做的同一件事。 〔**签名那一半已提前做了**（#190 / #196 逼出来的，A 线里插队）：`ontology::judge_direction` 一处判断，抽取与采纳共用；合并后对搬动过的事实报 `signature` 违规；R0 多一类。剩下 `disjointWith` 进消解那一半仍在这里。〕 diff --git a/docs/decisions/0017-a-contradiction-points-upstream.md b/docs/decisions/0017-a-contradiction-points-upstream.md new file mode 100644 index 000000000..faecada80 --- /dev/null +++ b/docs/decisions/0017-a-contradiction-points-upstream.md @@ -0,0 +1,205 @@ +# 0017 · A contradiction points at an error upstream + +- **Status**: B2a implemented (engine and queue: `derive::contradictions`, migration 0020, the Review card with clues and repairs) · B2b (visibility in the graph and the entity panel) still planned · B2 of 0016, wider than the one line written there: contradictions become visible everywhere, not only as a new kind in the queue +- **Written**: 2026-09-03 (conventions in [README](README.md)) +- **Related**: the two unbuilt rows of the "derived vs asserted" table in [0002](0002-reasoning-engine.md) §2; [0016](0016-close-the-open-seams-before-cutting-new-ones.md) B2; the proof chain (B1, #227) supplies the "premises expand to the sentence" half of the card below + +> This record changes one phrase in 0002. There, a contradiction "goes to Review for a ruling". +> In discussion (2026-09-03) it became clear what a person actually does when two edges +> disagree: follow the signal to **where something went wrong upstream** — a fact went +> stale, extraction misread, resolution merged the wrong pair, the ontology is too strict. +> So this queue is built as an audit, and every button is a repair. + +## The problem + +R1 derivation only yields to "this exact triple is already asserted". It does not consult +axioms: `part_of` transitivity derives `A part_of C` while the ledger asserts `C part_of A` +(asymmetric); `ceo_of` is functional and `Acme ceo_of Zhang San` is asserted, yet the +engine still lands `Acme ceo_of Li Si`. R0 scans `facts` only, derivations live in their +own table, so this class of contradiction is **invisible on both sides**: absent from +Review, absent from the graph. + +It is not the only invisible class. Temporal conflicts and axiom violations are known to +the Review page alone; on the graph and in the entity panel a contested edge looks exactly +like any other. + +## Criteria + +1. **A contradiction points at an error upstream.** The error is in one of four places: + stale knowledge (most common), a misread extraction, a wrong merge, an over-strict + ontology. The interface's job is to lay out the clues and offer the repairs. +2. **Write less to the graph rather than write wrong** (0002). A derivation that hits a + contradiction does not land; when two derivations collide, neither lands. +3. **Anything reported item by item needs an upper bound.** Only "one assertion against one + concrete edge" — a volume that can be predicted — goes into the queue individually, and + with a per-predicate cap; whatever is produced in batches is aggregated by its cause. +4. **A disputed fact is visible where it sits.** The assertion stays live, but someone + passing it on the graph or in the panel should see that it is questioned, and reach the + place to fix it in one step. + +## Decisions + +### 1. Detection: two tiers, pure logic in `utopia-reason` + +`derive::contradictions(derived, edges, axioms)` checks every derivation against the axioms +of its predicate, once against assertions and once against other derivations: + +| Axiom | What counts as a contradiction | +|---|---| +| functional | same subject and predicate, different object, **validity intervals overlap** (Mira leaving and Devin taking over is a succession; the half-open interval semantics of `validity` are reused) | +| inverse_functional | the dual | +| asymmetric | the reverse edge exists with overlapping validity | +| irreflexive | the derivation is a self-loop | + +**Derived vs asserted**: per item. The derivation does not land; one `axiom_violations` +row of kind `derived_contradiction`. Capped per predicate (same shape as R1's +`MAX_DERIVED_PER_PREDICATE`; 50 suggested); the overflow stays out of the queue and is +counted in the report. + +**Derived vs derived**: aggregated. The cause is a rule set that contradicts itself — +`ceo_of ⊑ works_at` together with `works_at` functional necessarily produces contradictions +in batches. Queuing every pair would flood Review, and a person facing a hundred identical +"two derivations disagree" cards cannot decide anything from them. So one row per **pair of +rules** goes into `ontology_defects`, kind `rules_disagree`, recording both rules, the +count, and two or three examples. It is the dynamic form of the existing static defects +`transitive_and_functional` and `symmetric_and_asymmetric`: invisible in the declarations, +surfacing once data arrives. Neither derivation lands. + +`run()` (R0) and `materialize` (R1) share the one function: R0 already holds the edges and +axioms and adds a `derive` + `contradictions` pass to report; R1 uses the same result to +decide what stays unlanded. **Both must compute it** — otherwise R0's "clear open rows not +recomputed this round" would sweep away what R1 wrote. + +### 2. The Review card: lay out the clues, lay out the repairs + +**The derived-vs-asserted card**, three parts: + +1. **The contradiction itself**: the derived edge (rule, premise chain expandable to the + sentence — from B1) beside the asserted edge (evidence quote, interval, confidence). + Most errors are visible at a glance. +2. **A diagnostic hint**, one line when a clue can be computed, none when it cannot: + - the assertion has no end date and the derivation starts after it → "looks stale: did + Zhang San's tenure end in 2024-07?" + - an entity on either side has same-name neighbours (the existing `same_name` machinery) + → "looks like a wrong merge: are these two Acmes one company?" + - the assertion's confidence is below 0.75 → "the extraction was unsure to begin with" + - none of the above → "read both sentences" +3. **Actions that are repairs**, all on existing endpoints; once a repair is made the + violation clears on the next recomputation, with no separate resolve step: + +| Action | What it repairs | Where it lands | +|---|---|---| +| Close the assertion at a date | stale knowledge | `POST /facts/{id}/close` (same as the temporal-conflict queue, with a date input) | +| Retract the assertion | misread extraction / wrong merge | `reject_fact`, append-only | +| Go to possible duplicates | wrong merge | the Duplicates queue in Review | +| The ontology is too strict | wrong definition | the relation on the Ontology page | +| Both hold; let the derivation through | the world is like that | `resolution = accepted`; the next `materialize` lets that pair land | + +**The derived-vs-derived card** (in the ontology-defects queue): "these two declarations +together produced N contradicting pairs; examples below — usually a sub-property attached +in the wrong place, or a functional declaration that is too strict". Two actions: go to the +Ontology page and change a declaration; or "accept" (contradicting derivations from this +pair of rules never land and are not reported again). Both sides are derived and the +assertions beneath may each be right, so there is no "the data is wrong" here; "both hold" +would mean giving up the meaning of functional, which belongs on the Ontology page, so +there is no such button either. + +### 3. Disputed facts are visible where they sit + +"Disputed" becomes one status with three sources: open temporal conflicts +(`fact_conflicts`), open axiom violations (`axiom_violations`, including the new kind), and +blocked derivations (the `detail` of `derived_contradiction` rows). B2 makes it a first-class +state of the browse pages and wires up the first two sources along the way — they are +equally invisible today. + +**Rows in the entity panel**: `EntityFact` already carries `stale` and `corrected`; a third +flag `contested: Option` is computed with one EXISTS. The chip +reads "disputed"; hover gives one sentence ("derived works_at Li Si contradicts this"); +click goes to the matching Review item. The row is not dimmed: the assertion is still live. + +**The Derived tab of the panel**: a new section, "derivations that did not land", read from +`axiom_violations.detail`, each row naming what blocked it and expanding to its premise +chain. Here a person sees "the engine could have drawn this edge, and what stopped it". + +**Edges on the graph**: + +- A contested assertion switches to the **alert colour**, edge and all, with no reliance on + a node ring. The request was explicit: a ring sits on the node while the edge stays grey, + and peripheral vision cannot tell them apart. +- A blocked derivation is drawn as a **ghost edge**: the same hue mixed toward `EDGE_DIM` + (under premultiplied blending alpha cannot darken an edge; only the RGB can be mixed — see + the note at `EDGE_DIM`), thinner. It follows the Derived toggle; clicking it opens the + entity panel at that row. sigma's default edge program does not draw dashes, and no custom + program is introduced for this. +- The hover label chip gets a "⚠" prefix, and the tooltip states the dispute. + +**Colour**: one new token, `--u-contest`, set to **#ff6a3d** (hot coral orange). It has to +stand apart from three colours already in use: derived edges are gold +(`rgb(231,197,124)`), warning chips are amber (`--u-warn` #f2b66d, too close to gold to +double as an edge colour), and danger is pink (`--u-danger` #ff9daf, reserved for +destructive actions). Coral orange is bright enough on the dark ground, its hue is far from +all three, and it is no common type colour (the type palette leans blue, green and violet). +Edge colour `rgba(255,106,61,0.55)`; ghost edge `lerp(#ff6a3d, EDGE_DIM, 0.55)`. This is +the only new colour; chips on the Ontology and Review pages use the same token. + +### 4. Data + +Migration `0020` (0019 was taken by #199 / #233): + +- `axiom_violations.kind` CHECK gains `derived_contradiction` +- `axiom_violations` gains `detail JSONB NOT NULL DEFAULT '{}'`: the derived triple (three + ids and three names), the rule kind, the premise ids — without it the interface cannot + say what was derived; `path` holds the premises for the proof chain but cannot draw the + ghost edge +- `axiom_violations.resolution` CHECK gains `fact_closed` (closing an assertion is a repair + that leaves its own trace, kept apart from `fact_retracted`) +- `ontology_defects.kind` CHECK gains `rules_disagree`, plus `detail JSONB` (both rules, the + count, the examples) + +`left_fact` / `right_fact` stay non-null and keep pointing at `facts`: for derived vs +asserted, `left` is the contradicted assertion and `right` the derivation's last premise; +`rules_disagree` does not live in this table. + +### 5. Interfaces + +- `GET /kbs/{id}/graph`: edges gain `contested: bool`; a new class of edge with + `blocked: true` (the ghosts) +- `GET /kbs/{id}/entities/{id}`: `facts[].contested`; `blocked: [...]` next to `derived` +- Review `violations` queue: rows gain `detail` and `hint` (the diagnostic as a code; wording + belongs to the frontend) +- `POST /kbs/{id}/review/violations/{id}`: `resolution` gains `fact_closed` (with + `close_at`); the server calls `close_fact` and then marks the row resolved + +### 6. Tests + +- `utopia-reason` unit tests: one per axiom, non-overlapping intervals are no contradiction, + derived-vs-derived aggregates by rule pair, the cap is counted +- Store integration: a functional assertion stands and a derivation collides → the + derivation stays out and one violation carries `detail`; retract the assertion and rerun → + the derivation lands and the violation clears; `accepted` → both coexist; `fact_closed` + runs the close path; R0 and R1 agree +- Browser: an edge turns coral, ghost edges follow the toggle, the panel chip jumps to the + Review item, the Review card shows three parts and five actions + +## Two cuts + +**B2a · engine and queue**: §1, §2, §4, and the Review parts of §5, with the integration +tests. Two days. +**B2b · visibility**: §3 plus the graph and panel interfaces, wiring existing temporal +conflicts and axiom violations along the way. A day and a half. + +B1 (#227) merges first after a rebase; B2a branches from it. + +## Open questions + +- **The per-predicate cap of 50 is a guess**, awaiting a bench number like R1's twenty + thousand. +- **Granularity of `accepted`**: per pair (one derivation against one assertion). An + assertion hit by several derivations needs several clicks; aggregating the exemption per + assertion would generalise "this axiom does not apply to this assertion" too far. Per + pair first, measure later. +- **Could ghost edges be too many?** Their number is bounded by the (capped) violation + count, so it should stay manageable; failing that, draw them only when a related node is + selected. +- **Does the disputed status need its own SSE event?** The `review` event is already sent; + graph and panel can refetch on it. No new event. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 81142c040..a32bd2d47 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -20,7 +20,7 @@ **行号会漂，文件名会换。** 正文里的 `file.rs:123` 是成文时的坐标，不保证仍然准确；迁移在 #130 / #131 从 53 份折成 10 份、一个域一份，所以 2026-08-31 之前写的迁移文件名都要按域重找。引用时优先写函数名、表名、常量名。 -**语言**：中文，与代码注释一致（UI 与 README 保持英文）。目前的读者是维护者，不是外部贡献者；将来需要时再译。 +**语言**：2026-09-03 起新记录用英文（0017 起），读者已经包括外部贡献者；此前的十六篇仍是中文，与代码注释一致，是否回译等有人需要时再定。 ## 索引 @@ -42,6 +42,7 @@ | 0014 | [身份跟着人，范围跟着令牌](0014-identity-from-the-person-scope-from-the-token.md) | 已实施（#180）· MCP 只读五工具 · 令牌页在账户层（A2）· 误导性的占位 crate 已删 | | 0015 | [记下一句话，不等于断言一个事实](0015-recording-a-sentence-is-not-asserting-a-fact.md) | 已实施 · 记忆抽出的事实进 `pending_facts`，Review 新档 + 跟在 remember 步骤后的确认卡 · `remember` 重新打开 · MCP 放开写是下一刀 | | 0016 | [先把开着的口子收上，再开新的](0016-close-the-open-seams-before-cutting-new-ones.md) | 规划中 · v0.1.0 之后的排期：A 收口 → B 推理机 ∥ C 尺子与本体 → D 语义层 → E 企业交付；模拟引擎后置 | +| 0017 | [A contradiction points at an error upstream](0017-a-contradiction-points-upstream.md) | B2a 已实现（引擎与队列：逐条封顶、按规则对聚合、卡片给线索与修法）· B2b 待做：争议在图和面板上原地可见（新警戒色）| ## 不是决策记录的那些 diff --git a/migrations/0020_a_contradiction_points_upstream.sql b/migrations/0020_a_contradiction_points_upstream.sql new file mode 100644 index 000000000..f893e5564 --- /dev/null +++ b/migrations/0020_a_contradiction_points_upstream.sql @@ -0,0 +1,34 @@ +-- 派生撞上断言时，让路这件事从静默变成可见（docs/decisions/0017）。 +-- +-- 0002 定了 asserted > derived：推出来的事实撞上账本里的断言就不落地。此前那一步 +-- 什么都不留——`ceo_of ⊑ works_at` 推出的 works_at 没了，人不知道有过这回事，也就 +-- 不知道该去看看是抽取错了、旧断言该闭合、还是两个「Mira」其实是一个人。 +-- +-- 一致性检查多一种 `derived_contradiction`：left 是被撞的断言，right 是派生的最后一条 +-- 前提，path 是全部前提；推出来的三元组本身没有落库、没有 id 可指，放进 `detail`。 +-- 出路多一条 `fact_closed`——最常见的修法是给旧断言一个结束日期。 +-- +-- 派生之间互撞（两条规则加在一起产出互斥的结论）按规则对聚合进 `ontology_defects`， +-- 一种 `rules_disagree`，`detail` 记规则对与几个例子。逐对进 Review 只会淹掉队列。 + +ALTER TABLE axiom_violations + DROP CONSTRAINT axiom_violations_kind_check, + ADD CONSTRAINT axiom_violations_kind_check CHECK (kind IN ( + 'self_loop', 'asymmetry', 'cycle', 'functional', 'signature', + 'derived_contradiction' + )), + DROP CONSTRAINT axiom_violations_resolution_check, + ADD CONSTRAINT axiom_violations_resolution_check CHECK (resolution IN ( + 'fact_retracted', 'fact_closed', 'axiom_relaxed', 'accepted' + )), + ADD COLUMN detail JSONB NOT NULL DEFAULT '{}'::jsonb; + +ALTER TABLE ontology_defects + DROP CONSTRAINT ontology_defects_kind_check, + ADD CONSTRAINT ontology_defects_kind_check CHECK (kind IN ( + 'symmetric_and_asymmetric', 'transitive_and_functional', 'subclass_cycle', + 'disjoint_with_ancestor', 'inherits_disjoint', + 'inverse_of_itself', 'inverse_not_mutual', 'sub_property_cycle', + 'rules_disagree' + )), + ADD COLUMN detail JSONB NOT NULL DEFAULT '{}'::jsonb; diff --git a/web/src/api.ts b/web/src/api.ts index 9fa6adf77..05d28ed1d 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -473,9 +473,33 @@ export interface MappingRevision { changed_at: string; } /** 一处公理违规（0002 R0）。判据来自本体自己声明的公理，没声明就不报 */ +/** derived_contradiction 独有（0017）：推出来的那条三元组——它没有落库， + * 只能在这里写出来。其它种类是 `{}` */ +export interface ViolationDetail { + axiom?: "functional" | "asymmetry" | "self_loop"; + rule?: "transitive" | "symmetric" | "inverse" | "sub_property"; + via_label?: string; + subject?: string; + predicate?: string; + object?: string; + valid_from?: string | null; + valid_to?: string | null; + premises?: string[]; +} +export type ViolationResolution = + | "fact_retracted" + | "fact_closed" + | "axiom_relaxed" + | "accepted"; export interface AxiomViolation { id: string; - kind: "self_loop" | "asymmetry" | "cycle" | "functional"; + kind: + | "self_loop" + | "asymmetry" + | "cycle" + | "functional" + | "signature" + | "derived_contradiction"; /** 判据来自哪条关系。判「公理写错了」时从这里进本体去改 */ predicate: string | null; left_fact: string; @@ -486,6 +510,10 @@ export interface AxiomViolation { /** 环的长度；其余三类为 0 */ path_len: number; detected_at: string; + detail: ViolationDetail; + /** 审核线索（0017 §2），一次只给一条：旧断言没写结束日期、有同名实体、 + * 抽取置信度低。没有就空 */ + hint: "stale" | "duplicate" | "unsure" | null; } /** 本体自己的一处自相矛盾。**与 AxiomViolation 不是一回事**：那个说 * 「事实与定义抵触」，这个说「定义自己站不住」，后者更根本 */ @@ -500,11 +528,27 @@ export interface OntologyDefect { // 0017 加的三类：都在谓词上，前两类关于逆，第三类是子属性成环 | "inverse_of_itself" | "inverse_not_mutual" - | "sub_property_cycle"; + | "sub_property_cycle" + // 0017：两条规则加在一起产出互斥的派生，按规则对聚合报一次 + | "rules_disagree"; subject_label: string | null; other_label: string | null; path_labels: string[]; detected_at: string; + detail: DefectDetail; +} +/** rules_disagree 独有：哪两条规则、撞在哪条公理上、几对、几个例子 */ +export interface DefectDetail { + count?: number; + rules?: { + rule_a: string; + via_a: string; + rule_b: string; + via_b: string; + axiom: string; + count: number; + examples: [string, string][]; + }[]; } export interface FactReviewItem { id: string; @@ -1720,11 +1764,15 @@ export const api = { decideViolation: ( kbId: string, violationId: string, - resolution: "fact_retracted" | "axiom_relaxed" | "accepted", + resolution: ViolationResolution, + closeAt?: string, ) => request<{ ok: boolean }>( `/api/v1/kbs/${kbId}/review/violations/${violationId}`, - { method: "POST", body: JSON.stringify({ resolution }) }, + { + method: "POST", + body: JSON.stringify({ resolution, close_at: closeAt ?? null }), + }, ), confirmFact: (kbId: string, factId: string) => request<{ ok: boolean }>(`/api/v1/kbs/${kbId}/facts/${factId}/confirm`, { diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index c2cce2958..367e56994 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -1303,6 +1303,16 @@ export const en = { defectInverseSelf: "Its own inverse — say symmetric instead", defectInverseNotMutual: "The inverse does not point back", defectSubPropertyCycle: "subPropertyOf runs in a circle", + defectRulesDisagree: "Two rules produce contradicting derivations", + rulesDisagreeCount: (n: number) => + `${n} pair(s) of derivations held back until this is settled`, + rulesDisagreeRule: ( + a: string, + va: string, + b: string, + vb: string, + axiom: string, + ) => `${a} on ${va} with ${b} on ${vb}, against ${axiom}`, defectNeverInstantiable: "no instance can ever satisfy it", defectFixed: "I fixed the ontology", defectAccepted: "Leave it", @@ -1325,6 +1335,24 @@ export const en = { /** 签名违规（#190 / #196）：一条事实的主语或宾语落在谓词声明的类型之外—— * 抽取时会掰正，采纳与合并这两条路从前绕过了检查 */ violationSignature: "Subject or object outside the declared types", + /** 0017：派生撞上断言。卡片是一次审核，线索指向上游的错 */ + violationDerived: "A derivation contradicts an assertion", + derivedLine: (s: string, p: string, o: string) => + `Derived: ${s} · ${p} · ${o}`, + derivedBy: (rule: string, via: string) => `by ${rule} on ${via}`, + assertedLine: (t: string) => `Asserted: ${t}`, + hintStale: + "The assertion has no end date and the derivation starts later. It may simply have ended.", + hintDuplicate: + "Two entities share this name. They may be the same one.", + hintUnsure: + "The assertion was extracted with low confidence. Read its sentence.", + hintReadBoth: "Read both sentences and decide which one is wrong.", + closeAssertion: "Give the assertion an end date", + retractAssertion: "Retract the assertion", + seeDuplicates: "See duplicates", + openOntology: "Open the ontology", + letBothStand: "Let both stand", violationVia: (p: string) => `via ${p}`, violationPath: (n: number) => `${n} facts in the cycle`, retractFact: "Data is wrong", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index c5915ed25..f30986c00 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -1177,6 +1177,15 @@ export const zh: Strings = { defectInverseSelf: "自己是自己的逆——写成「对称」更直白", defectInverseNotMutual: "逆关系没有指回来", defectSubPropertyCycle: "子属性绕成了环", + defectRulesDisagree: "两条规则推出互相抵触的结论", + rulesDisagreeCount: (n: number) => `${n} 对派生等这里定了再落地`, + rulesDisagreeRule: ( + a: string, + va: string, + b: string, + vb: string, + axiom: string, + ) => `${va} 上的 ${a} 与 ${vb} 上的 ${b}，撞在 ${axiom} 上`, defectNeverInstantiable: "这个类永远不可能有实例", defectFixed: "已去本体里改了", defectAccepted: "先放着", @@ -1195,6 +1204,20 @@ export const zh: Strings = { violationCycle: "传递链绕成了环", violationFunctional: "该只有一个值，却有两个", violationSignature: "主语或宾语不在关系声明的类型里", + violationDerived: "推出来的与断言相抵触", + derivedLine: (s: string, p: string, o: string) => + `推出：${s} · ${p} · ${o}`, + derivedBy: (rule: string, via: string) => `由 ${via} 上的 ${rule}`, + assertedLine: (t: string) => `断言：${t}`, + hintStale: "这条断言没写结束日期，而推出来的那条起得更晚——它可能只是结束了。", + hintDuplicate: "有两个同名实体，它们可能是同一个。", + hintUnsure: "这条断言抽取时把握不大，去读一下原句。", + hintReadBoth: "读一下两边的原句，判断哪条错了。", + closeAssertion: "给断言一个结束日期", + retractAssertion: "撤掉断言", + seeDuplicates: "去看重复实体", + openOntology: "去本体页", + letBothStand: "两边都成立", violationVia: (p: string) => `依据 ${p}`, violationPath: (n: number) => `环上 ${n} 条事实`, retractFact: "数据错了", diff --git a/web/src/pages/Review.tsx b/web/src/pages/Review.tsx index 5c73892ac..6ca977c25 100644 --- a/web/src/pages/Review.tsx +++ b/web/src/pages/Review.tsx @@ -14,6 +14,7 @@ import { type ReviewHistoryEvent, type ReviewItem, type ReviewSide, + type ViolationResolution, } from "../api"; import { PendingFactRow, useCanDecide } from "./PendingFacts"; import { S } from "../i18n"; @@ -481,7 +482,9 @@ function DefectRow({ inverse_of_itself: S.review.defectInverseSelf, inverse_not_mutual: S.review.defectInverseNotMutual, sub_property_cycle: S.review.defectSubPropertyCycle, + rules_disagree: S.review.defectRulesDisagree, }[d.kind]; + const rules = d.kind === "rules_disagree" ? (d.detail.rules ?? []) : []; // 后两类的后果值得写出来：不可满足的类不会报错，它只是永远空着 const unsatisfiable = d.kind === "disjoint_with_ancestor" || d.kind === "inherits_disjoint"; @@ -506,6 +509,23 @@ function DefectRow({ {S.review.defectNeverInstantiable}

)} + {rules.length > 0 && ( +
+
{S.review.rulesDisagreeCount(d.detail.count ?? 0)}
+ {rules.map((r, i) => ( +
+
+ {S.review.rulesDisagreeRule(r.rule_a, r.via_a, r.rule_b, r.via_b, r.axiom)} +
+ {r.examples.map(([x, y], j) => ( +
+ {x} · {y} +
+ ))} +
+ ))} +
+ )}
+ + + + +
+ + ); +} + /* ---------- 页面：左栏分类 + 单类内容区 ---------- */ type Sel = @@ -788,10 +917,12 @@ export function Review() { mutationFn: ({ id, resolution, + closeAt, }: { id: string; - resolution: "fact_retracted" | "axiom_relaxed" | "accepted"; - }) => api.decideViolation(kb!.id, id, resolution), + resolution: ViolationResolution; + closeAt?: string; + }) => api.decideViolation(kb!.id, id, resolution, closeAt), onSettled: invalidate, }); // 检查是同步的纯计算,所以直接 mutate 不排队。跑完把报告留在按钮旁边—— @@ -1177,9 +1308,11 @@ export function Review() { violationAction.isPending && violationAction.variables?.id === v.id } - onDecide={(resolution) => - violationAction.mutate({ id: v.id, resolution }) + onDecide={(resolution, closeAt) => + violationAction.mutate({ id: v.id, resolution, closeAt }) } + onDuplicates={() => select("duplicates")} + onOntology={() => navigate({ to: "/ontology" })} /> ))} diff --git a/web/src/styles.css b/web/src/styles.css index 40cdb6aa6..85397ce83 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -31,6 +31,8 @@ --u-warn: #f2b66d; /* 浅玫瑰：深底上的危险"文字/描边/点"专用（够亮才可读） */ --u-danger: #ff9daf; + /* 0017：争议色。派生撞上断言、被挡下的派生——警告与危险之外的第三种 */ + --u-contest: #ff6a3d; /* 深红：危险"实底按钮"专用（浅玫瑰做底会发粉；亮红在单色 chrome 上嗓门过大），配白字 */ --u-danger-solid: #c9353a; --u-violet: #c4a5ff; From f18c560160be6e3489ca983ce076558216e24ba3 Mon Sep 17 00:00:00 2001 From: WaylandYang <145302500+WaylandYang@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:26:57 +0800 Subject: [PATCH 03/10] The lakehouse is one protocol away: Trino, Databricks and Snowflake as query engines (#239) Co-authored-by: Claude Fable 5.1 --- Cargo.lock | 111 +++++- README.md | 4 +- crates/utopia-server/Cargo.toml | 6 + crates/utopia-server/src/api/chat.rs | 4 +- .../src/api/datasource_routes.rs | 19 +- crates/utopia-server/src/api/tools.rs | 3 +- crates/utopia-server/src/query_engine.rs | 205 ---------- crates/utopia-server/src/query_engine/conn.rs | 253 +++++++++++++ .../src/query_engine/databricks.rs | 271 ++++++++++++++ crates/utopia-server/src/query_engine/mod.rs | 349 ++++++++++++++++++ .../src/query_engine/postgres.rs | 93 +++++ .../src/query_engine/snowflake.rs | 257 +++++++++++++ .../utopia-server/src/query_engine/trino.rs | 266 +++++++++++++ crates/utopia-store/src/datasources.rs | 21 +- ...0018-the-lakehouse-is-one-protocol-away.md | 58 +++ docs/decisions/README.md | 1 + migrations/0021_lakehouse_engines.sql | 7 + web/src/i18n/en.ts | 8 +- web/src/i18n/zh.ts | 7 +- web/src/pages/Settings.tsx | 10 +- 20 files changed, 1716 insertions(+), 237 deletions(-) delete mode 100644 crates/utopia-server/src/query_engine.rs create mode 100644 crates/utopia-server/src/query_engine/conn.rs create mode 100644 crates/utopia-server/src/query_engine/databricks.rs create mode 100644 crates/utopia-server/src/query_engine/mod.rs create mode 100644 crates/utopia-server/src/query_engine/postgres.rs create mode 100644 crates/utopia-server/src/query_engine/snowflake.rs create mode 100644 crates/utopia-server/src/query_engine/trino.rs create mode 100644 docs/decisions/0018-the-lakehouse-is-one-protocol-away.md create mode 100644 migrations/0021_lakehouse_engines.sql diff --git a/Cargo.lock b/Cargo.lock index 4fd495552..83ce8b63b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,6 +116,16 @@ dependencies = [ "password-hash", ] +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "async-compression" version = "0.4.43" @@ -324,6 +334,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -934,6 +950,24 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + [[package]] name = "debug_unsafe" version = "0.1.4" @@ -1354,6 +1388,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.34" @@ -1427,6 +1476,7 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -1569,6 +1619,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + [[package]] name = "hex" version = "0.4.3" @@ -1727,7 +1783,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -2128,7 +2184,7 @@ version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ - "base64", + "base64 0.22.1", "ed25519-dalek", "getrandom 0.2.17", "hmac", @@ -2555,6 +2611,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "object" version = "0.39.1" @@ -2572,7 +2638,7 @@ checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" dependencies = [ "async-trait", "aws-lc-rs", - "base64", + "base64 0.22.1", "bytes", "chrono", "crc-fast", @@ -2806,7 +2872,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -3305,7 +3371,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -3875,7 +3941,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "crc", @@ -3952,7 +4018,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "bytes", @@ -3996,7 +4062,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "chrono", @@ -4212,7 +4278,7 @@ checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07" dependencies = [ "aho-corasick", "arc-swap", - "base64", + "base64 0.22.1", "bitpacking", "bon", "byteorder", @@ -4922,6 +4988,7 @@ dependencies = [ "async-trait", "axum", "axum-extra", + "base64 0.23.1", "chrono", "dotenvy", "feed-rs", @@ -4929,6 +4996,7 @@ dependencies = [ "futures-util", "jsonwebtoken", "object_store", + "percent-encoding", "quick-xml 0.42.0", "reqwest", "serde", @@ -4940,6 +5008,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "url", "utopia-core", "utopia-extract", "utopia-ingest", @@ -4947,6 +5016,7 @@ dependencies = [ "utopia-search", "utopia-store", "uuid", + "wiremock", ] [[package]] @@ -5506,6 +5576,29 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64 0.22.1", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/README.md b/README.md index a46c5af17..a71c26b23 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ One Rust binary and one Postgres. Full-text search is embedded in the binary, ve | **Entity resolution and review** | Duplicates are resolved in three stages: exact name or alias, embedding similarity, then a model's call on the doubtful pairs. Every merge can be undone. Uncertain cases go to a review queue: low-confidence extractions, suspected duplicates and cardinality conflicts. | | **Reasoning and derivation** | Ontology axioms compile into rules: transitivity, symmetry, inverses and relation hierarchy derive new facts by forward chaining. Derivation is off by default, since a wrong axiom derives wrong facts. A derived fact is marked as such on the graph, carries validity and confidence like any other, and shows what it was derived from. When it contradicts an asserted fact, the asserted one stands. | | **Conflict detection** | Three kinds of conflict, three sets of choices. A new fact that clashes with an older one: close the old, keep both, or reject the new. Data that breaks an axiom (self-loop, asymmetry, transitive cycle, cardinality): retract the fact, relax the axiom, or accept both. The ontology itself is checked first, because violations of a self-contradictory ontology are noise. | -| **Ontology-driven querying** | Mount a Postgres database on a base and chat can query it alongside the documents. The agent proposes how its tables map onto the ontology, and you confirm. The method behind it, [Ontology2SQL](https://github.com/deeplethe/ontology2sql), is state of the art on BIRD Mini-Dev for SQLite and PostgreSQL ([submission](https://github.com/bird-bench/bird-bench.github.io/pull/218)). | +| **Ontology-driven querying** | Mount a database on a base (Postgres, Trino for Iceberg / Delta Lake / Hive, Databricks, Snowflake) and chat can query it alongside the documents. The agent proposes how its tables map onto the ontology, and you confirm. The method behind it, [Ontology2SQL](https://github.com/deeplethe/ontology2sql), is state of the art on BIRD Mini-Dev for SQLite and PostgreSQL ([submission](https://github.com/bird-bench/bird-bench.github.io/pull/218)). | | **Multi-user and permissions** | Each knowledge base has its own members and roles: owner, admin, editor and viewer. Open bases are readable by everyone in the deployment, restricted ones only by invitation. The first account registered becomes the system administrator. | | **Decision ledger** | Confirming or rejecting a fact, merging or reverting an entity, rebuilding the graph: each leaves a record of who, when, and what the object looked like at the time. The ledger is append-only, and a record outlives its object, even the base it belonged to. | | **[Decision intelligence (in development)](#roadmap)** | Record a decision, replay both what was understood and the course it took, and reason over overlaid scenarios. | @@ -103,7 +103,7 @@ cd web && pnpm install && pnpm dev - [ ] **Decision reasoning**: constraint computation, and replaying a decision after the fact - [ ] **Execution gate**: checking an agent's calls against ontology rules and symbolic logic -- [ ] **Lakehouse for mapping and querying**: mapping exploration and Ontology2SQL over Iceberg / Delta Lake, Databricks, Snowflake and MaxCompute +- [ ] **MaxCompute**: mapping exploration and Ontology2SQL over Alibaba Cloud MaxCompute (Iceberg / Delta Lake via Trino, Databricks and Snowflake are in, awaiting a run against a real cluster) - [ ] **More sources**: MySQL, ClickHouse and Doris drivers; S3, WebDAV, Notion and Feishu connectors - [ ] **Time to the moment**: an `instant` precision beside year / month / day, for sources that carry a real timestamp. Today a connector rounds it to a UTC day, which can shift an event across midnight by one day - [ ] **Agent memory over MCP**: episode writes, the retrieve endpoint, and the MCP server diff --git a/crates/utopia-server/Cargo.toml b/crates/utopia-server/Cargo.toml index ffd2b5462..628e03dbf 100644 --- a/crates/utopia-server/Cargo.toml +++ b/crates/utopia-server/Cargo.toml @@ -41,3 +41,9 @@ async-trait = "0.1.92" sqlparser = "0.62.0" object_store = { version = "0.14.1", features = ["aws", "azure", "gcp"] } quick-xml.workspace = true +url = "2.5.8" +percent-encoding = "2.3.2" +base64 = "0.23.1" + +[dev-dependencies] +wiremock = "0.6.5" diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rs index 2c73d2696..f6ef8237c 100644 --- a/crates/utopia-server/src/api/chat.rs +++ b/crates/utopia-server/src/api/chat.rs @@ -71,7 +71,7 @@ fn tools_schema(can_write: bool, data_source_names: &[String]) -> serde_json::Va }, "sql": { "type": "string", - "description": "One SELECT/WITH statement (PostgreSQL dialect)." + "description": "One SELECT/WITH statement in the source's own SQL dialect (PostgreSQL, Trino, Databricks or Snowflake; the schema document names the engine)." }, "purpose": { "type": "string", @@ -435,7 +435,7 @@ pub async fn chat( }; if !ds_names.is_empty() { system_prompt.push_str(&format!( - "\nData: query_data runs read-only SQL (PostgreSQL dialect) against: {}. \ + "\nData: query_data runs read-only SQL (in each source's own dialect) against: {}. \ For questions about numbers/metrics, search for the source's schema document \ first, then query. State units and the time range you used in the answer.", ds_names.join(", ") diff --git a/crates/utopia-server/src/api/datasource_routes.rs b/crates/utopia-server/src/api/datasource_routes.rs index 9a3e9e822..8956a4ed0 100644 --- a/crates/utopia-server/src/api/datasource_routes.rs +++ b/crates/utopia-server/src/api/datasource_routes.rs @@ -53,10 +53,25 @@ pub async fn create( Json(body): Json, ) -> ApiResult> { require_admin(&user)?; + // 引擎跟着 scheme 走，界面只有一个连接串输入框；body.engine 只为兼容旧调用留着 + let engine = crate::query_engine::engine_from_conn(&body.conn_string).ok_or_else(|| { + utopia_core::AppError::invalid( + "unsupported_conn_scheme", + format!( + "Connection string must start with one of: postgres://, trino://, databricks://, snowflake:// (engines: {})", + crate::query_engine::ENGINES.join(", ") + ), + ) + })?; + let _ = &body.engine; + // 连接串的形状在登记时就校验（缺令牌、缺 warehouse……），错误信息里带写法； + // 否则要等到「测试」才知道，而那一步只回 ok:false + crate::query_engine::engine_for(engine, &body.conn_string) + .map_err(|e| utopia_core::AppError::invalid("bad_conn_string", e.to_string()))?; let id = utopia_store::datasources::create( &state.pool, &body.name, - &body.engine, + engine, &body.conn_string, user.id, ) @@ -228,7 +243,7 @@ async fn sync_schema_doc(state: &AppState, kb_id: Uuid, ds_id: Uuid) -> anyhow:: .await?; let mut md = format!( - "# Data source: {name}\n\nTables and columns available for SQL queries against this source.\n" + "# Data source: {name}\n\nEngine: {engine}. Tables and columns available for SQL queries against this source; write SQL in this engine's dialect.\n" ); let mut current = String::new(); let mut tables = 0usize; diff --git a/crates/utopia-server/src/api/tools.rs b/crates/utopia-server/src/api/tools.rs index 22c41d9f4..98a52a2c1 100644 --- a/crates/utopia-server/src/api/tools.rs +++ b/crates/utopia-server/src/api/tools.rs @@ -410,8 +410,9 @@ pub(super) fn charter_source_json(n: usize, h: &utopia_search::DocsSection) -> s /// 问数执行：安全闸（解析白名单）→ 引擎执行（只读会话 + 强制 LIMIT + 超时）→ JSON 行。 async fn run_query(state: &AppState, ds_id: Uuid, sql: &str) -> anyhow::Result { - let guarded = crate::query_engine::guard_sql(sql)?; let (engine, conn) = utopia_store::datasources::engine_and_conn(&state.pool, ds_id).await?; + // 闸门按引擎选方言：Databricks 的反引号、Snowflake 的 :: 转型都得先过得了解析 + let guarded = crate::query_engine::guard_sql_for(&engine, sql)?; let result = crate::query_engine::engine_for(&engine, &conn)? .execute(&guarded) .await?; diff --git a/crates/utopia-server/src/query_engine.rs b/crates/utopia-server/src/query_engine.rs deleted file mode 100644 index 01a015bc2..000000000 --- a/crates/utopia-server/src/query_engine.rs +++ /dev/null @@ -1,205 +0,0 @@ -//! 问数查询引擎：trait 接缝（BlobStore 同手法）+ 引擎无关的安全闸。 -//! -//! 引擎按协议族扩，不按产品名扩：postgres（本文件）→ mysql 线协议族（白捡 -//! TiDB/OceanBase/Doris/StarRocks）→ HTTP 族（ClickHouse、Trino——后者一个顶起 -//! Iceberg/Delta/Hive 整个湖仓生态）。挂载模型与注册表引擎无关，加引擎零迁移。 -//! -//! 安全闸（纵深防御，不信任模型）： -//! 1. sqlparser 解析：仅放行单条 SELECT/WITH（含 CTE），拒绝 DML/DDL/多语句/SELECT INTO -//! 2. 强制外包一层 LIMIT（cap+1 探测截断） -//! 3. 会话级只读 + 语句超时（引擎各自的机制，parser 万一漏网也写不进去） -//! 4. 结果统一为 JSON Lines（各引擎都有原生 JSON 行输出，也是模型最好消化的格式） - -use sqlparser::ast::Statement; -use sqlparser::dialect::PostgreSqlDialect; -use sqlparser::parser::Parser; -use sqlx::postgres::PgPoolOptions; -use sqlx::Row; -use std::time::Duration; - -/// 行数上限（外包 LIMIT cap+1，第 201 行只用来判断截断）。 -pub const ROW_CAP: usize = 200; -const STATEMENT_TIMEOUT_SECS: u32 = 10; - -pub struct QueryResult { - /// 每行一个 JSON 对象文本（键序 = 查询列序） - pub rows: Vec, - pub truncated: bool, -} - -#[derive(Debug)] -pub struct SchemaColumn { - pub schema: String, - pub table: String, - pub column: String, - pub data_type: String, - pub comment: Option, -} - -#[async_trait::async_trait] -pub trait QueryEngine: Send + Sync { - async fn test(&self) -> anyhow::Result<()>; - async fn fetch_schema(&self) -> anyhow::Result>; - /// 执行已过闸的 SELECT。实现自身仍需强制只读会话与超时（纵深防御）。 - async fn execute(&self, sql: &str) -> anyhow::Result; -} - -/// 引擎工厂。conn 凭据只在服务端流转。 -pub fn engine_for(engine: &str, conn: &str) -> anyhow::Result> { - match engine { - "postgres" => Ok(Box::new(PostgresEngine { - conn: conn.to_string(), - })), - other => anyhow::bail!("Unsupported engine: {other}"), - } -} - -/// 安全闸第 1 层：解析并校验，返回规整后的语句文本。 -pub fn guard_sql(sql: &str) -> anyhow::Result { - let cleaned = sql.trim().trim_end_matches(';').trim(); - if cleaned.is_empty() { - anyhow::bail!("Empty SQL"); - } - let statements = Parser::parse_sql(&PostgreSqlDialect {}, cleaned) - .map_err(|e| anyhow::anyhow!("SQL parse error: {e}"))?; - if statements.len() != 1 { - anyhow::bail!("Exactly one statement is allowed"); - } - match &statements[0] { - Statement::Query(_) => Ok(cleaned.to_string()), - other => anyhow::bail!( - "Read-only: only SELECT/WITH queries are allowed (got {})", - statement_kind(other) - ), - } -} - -fn statement_kind(s: &Statement) -> &'static str { - match s { - Statement::Insert { .. } => "INSERT", - Statement::Update { .. } => "UPDATE", - Statement::Delete { .. } => "DELETE", - Statement::Drop { .. } => "DROP", - Statement::CreateTable { .. } | Statement::CreateView { .. } => "CREATE", - Statement::AlterTable { .. } => "ALTER", - Statement::Truncate { .. } => "TRUNCATE", - Statement::Copy { .. } => "COPY", - _ => "a non-SELECT statement", - } -} - -// --------------------------------------------------------------------------- -// Postgres 族（顺带覆盖 Greenplum/Timescale 等 PG 兼容系） -// --------------------------------------------------------------------------- - -pub struct PostgresEngine { - conn: String, -} - -impl PostgresEngine { - async fn pool(&self) -> anyhow::Result { - Ok(PgPoolOptions::new() - .max_connections(1) - .acquire_timeout(Duration::from_secs(5)) - .connect(&self.conn) - .await?) - } -} - -#[async_trait::async_trait] -impl QueryEngine for PostgresEngine { - async fn test(&self) -> anyhow::Result<()> { - let pool = self.pool().await?; - sqlx::query("SELECT 1").execute(&pool).await?; - pool.close().await; - Ok(()) - } - - async fn fetch_schema(&self) -> anyhow::Result> { - let pool = self.pool().await?; - let rows: Vec<(String, String, String, String, Option)> = sqlx::query_as( - "SELECT c.table_schema, c.table_name, c.column_name, - c.data_type, pgd.description - FROM information_schema.columns c - LEFT JOIN pg_catalog.pg_statio_all_tables st - ON st.schemaname = c.table_schema AND st.relname = c.table_name - LEFT JOIN pg_catalog.pg_description pgd - ON pgd.objoid = st.relid AND pgd.objsubid = c.ordinal_position - WHERE c.table_schema NOT IN ('pg_catalog', 'information_schema') - ORDER BY c.table_schema, c.table_name, c.ordinal_position", - ) - .fetch_all(&pool) - .await?; - pool.close().await; - Ok(rows - .into_iter() - .map(|(schema, table, column, data_type, comment)| SchemaColumn { - schema, - table, - column, - data_type, - comment, - }) - .collect()) - } - - async fn execute(&self, sql: &str) -> anyhow::Result { - let pool = self.pool().await?; - // 纵深防御第 3 层：会话级只读 + 超时（parser 漏网也写不进去、跑不死库） - sqlx::query("SET default_transaction_read_only = on") - .execute(&pool) - .await?; - sqlx::query(&format!( - "SET statement_timeout = '{STATEMENT_TIMEOUT_SECS}s'" - )) - .execute(&pool) - .await?; - // 第 2 层：外包 LIMIT；row_to_json 让 PG 全权处理类型→JSON（文本键序保留列序） - let wrapped = format!( - "SELECT row_to_json(_q)::text AS _j FROM ( {sql} ) AS _q LIMIT {}", - ROW_CAP + 1 - ); - let fetched = sqlx::query(&wrapped).fetch_all(&pool).await?; - pool.close().await; - - let truncated = fetched.len() > ROW_CAP; - let rows = fetched - .into_iter() - .take(ROW_CAP) - .map(|r| r.try_get::("_j").unwrap_or_else(|_| "{}".into())) - .collect(); - Ok(QueryResult { rows, truncated }) - } -} - -#[cfg(test)] -mod tests { - use super::guard_sql; - - #[test] - fn allows_select_and_cte() { - assert!(guard_sql("SELECT region, sum(amount) FROM orders GROUP BY 1").is_ok()); - assert!(guard_sql("WITH t AS (SELECT 1 AS x) SELECT * FROM t;").is_ok()); - } - - #[test] - fn rejects_writes_and_ddl() { - for bad in [ - "UPDATE orders SET amount = 0", - "DELETE FROM orders", - "INSERT INTO orders (region) VALUES ('east')", - "DROP TABLE orders", - "TRUNCATE orders", - "CREATE TABLE t (id int)", - "ALTER TABLE orders ADD COLUMN x int", - ] { - assert!(guard_sql(bad).is_err(), "should reject: {bad}"); - } - } - - #[test] - fn rejects_multi_statement() { - assert!(guard_sql("SELECT 1; DROP TABLE orders").is_err()); - assert!(guard_sql("").is_err()); - } -} diff --git a/crates/utopia-server/src/query_engine/conn.rs b/crates/utopia-server/src/query_engine/conn.rs new file mode 100644 index 000000000..1d3cd5192 --- /dev/null +++ b/crates/utopia-server/src/query_engine/conn.rs @@ -0,0 +1,253 @@ +//! 连接串解析。一个输入框、四种 scheme；这里把 URL 拆成各引擎要的字段。 +//! +//! 写法沿用 `postgres://user:pass@host/db` 的形状：凭据在 userinfo 里，HTTP 族的 +//! 令牌放 password 位（`databricks://:TOKEN@…`），路径是「目录 / 库 / schema」， +//! 引擎特有的开关走 query。`ssl=false` 让 HTTP 族走明文——给本地代理与测试用， +//! 线上的三家都只认 https。 + +use percent_encoding::percent_decode_str; +use url::Url; + +fn decode(s: &str) -> String { + percent_decode_str(s).decode_utf8_lossy().into_owned() +} + +fn query(u: &Url, key: &str) -> Option { + u.query_pairs() + .find(|(k, _)| k == key) + .map(|(_, v)| v.into_owned()) + .filter(|v| !v.is_empty()) +} + +fn ssl_off(u: &Url) -> bool { + matches!( + query(u, "ssl").as_deref(), + Some("false") | Some("0") | Some("off") | Some("no") + ) +} + +fn segments(u: &Url) -> Vec { + u.path_segments() + .map(|s| s.filter(|x| !x.is_empty()).map(decode).collect()) + .unwrap_or_default() +} + +/// 令牌：password 位优先；没有 password 时 username 位也算（`databricks://TOKEN@host` +/// 少打一个冒号是最常见的手滑）；最后看 `?token=` +fn token_of(u: &Url) -> Option { + u.password() + .map(decode) + .filter(|s| !s.is_empty()) + .or_else(|| Some(decode(u.username())).filter(|s| !s.is_empty())) + .or_else(|| query(u, "token")) +} + +fn base_of(u: &Url, https: bool, default_port: u16) -> anyhow::Result { + let host = u + .host_str() + .ok_or_else(|| anyhow::anyhow!("{}://: a host is required", u.scheme()))?; + let port = u.port().unwrap_or(default_port); + let scheme = if https { "https" } else { "http" }; + // 默认端口不写进 URL：reqwest 照样能连，日志里也干净 + let explicit = match (https, port) { + (true, 443) | (false, 80) => String::new(), + _ => format!(":{port}"), + }; + Ok(format!("{scheme}://{host}{explicit}")) +} + +/// `trino://user[:password]@host[:port]/[catalog[/schema]][?ssl=true|false]` +/// +/// 明文 http 是 Trino 的默认（8080）；带密码、`ssl=true`、或端口 443 / 8443 时走 https—— +/// Trino 自己也拒绝在明文上收密码。`presto://` 是同一个协议的旧名。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrinoConn { + pub base: String, + pub user: String, + pub password: Option, + pub catalog: Option, + pub schema: Option, +} + +impl TrinoConn { + pub fn parse(conn: &str) -> anyhow::Result { + let u = Url::parse(conn.trim())?; + let user = decode(u.username()); + if user.is_empty() { + anyhow::bail!("trino://: a user is required (it becomes X-Trino-User), e.g. trino://alice@host:8080/hive/default"); + } + let password = u.password().map(decode).filter(|s| !s.is_empty()); + let https = !ssl_off(&u) + && (password.is_some() + || query(&u, "ssl").as_deref() == Some("true") + || matches!(u.port(), Some(443) | Some(8443))); + let base = base_of(&u, https, if https { 443 } else { 8080 })?; + let segs = segments(&u); + Ok(Self { + base, + user, + password, + catalog: segs.first().cloned(), + schema: segs.get(1).cloned(), + }) + } +} + +/// `databricks://:TOKEN@workspace-host/sql/1.0/warehouses/WAREHOUSE_ID[?catalog=main&schema=default]` +/// +/// 路径就是 JDBC 里的 httpPath，从控制台复制过来不用改；`?warehouse=ID` 也认。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DatabricksConn { + pub base: String, + pub token: String, + pub warehouse_id: String, + pub catalog: Option, + pub schema: Option, +} + +impl DatabricksConn { + pub fn parse(conn: &str) -> anyhow::Result { + let u = Url::parse(conn.trim())?; + let base = base_of(&u, !ssl_off(&u), if ssl_off(&u) { 80 } else { 443 })?; + let token = token_of(&u).ok_or_else(|| { + anyhow::anyhow!("databricks://: a personal access token is required, e.g. databricks://:TOKEN@host/sql/1.0/warehouses/ID") + })?; + let segs = segments(&u); + let from_path = segs + .iter() + .position(|s| s == "warehouses") + .and_then(|i| segs.get(i + 1).cloned()); + let warehouse_id = query(&u, "warehouse").or(from_path).ok_or_else(|| { + anyhow::anyhow!("databricks://: a SQL warehouse is required — the /sql/1.0/warehouses/ID path or ?warehouse=ID") + })?; + Ok(Self { + base, + token, + warehouse_id, + catalog: query(&u, "catalog"), + schema: query(&u, "schema"), + }) + } +} + +/// `snowflake://:TOKEN@account.snowflakecomputing.com/[DATABASE[/SCHEMA]][?warehouse=WH&role=R&token_type=pat|oauth]` +/// +/// SQL API 不收密码，只收令牌：默认当作 programmatic access token，`token_type=oauth` +/// 换成 OAuth 令牌。密钥对 JWT 要本地签名，这一版不做。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnowflakeConn { + pub base: String, + pub token: String, + /// `X-Snowflake-Authorization-Token-Type` 的值 + pub token_type: &'static str, + pub database: Option, + pub schema: Option, + pub warehouse: Option, + pub role: Option, +} + +impl SnowflakeConn { + pub fn parse(conn: &str) -> anyhow::Result { + let u = Url::parse(conn.trim())?; + let base = base_of(&u, !ssl_off(&u), if ssl_off(&u) { 80 } else { 443 })?; + let token = token_of(&u).ok_or_else(|| { + anyhow::anyhow!("snowflake://: a programmatic access token or OAuth token is required, e.g. snowflake://:TOKEN@account.snowflakecomputing.com/DB/SCHEMA?warehouse=WH") + })?; + let token_type = match query(&u, "token_type") + .as_deref() + .map(str::to_ascii_lowercase) + .as_deref() + { + None | Some("pat") | Some("programmatic_access_token") => "PROGRAMMATIC_ACCESS_TOKEN", + Some("oauth") => "OAUTH", + Some(other) => { + anyhow::bail!("snowflake://: unknown token_type '{other}' (pat or oauth)") + } + }; + let segs = segments(&u); + Ok(Self { + base, + token, + token_type, + database: segs.first().cloned(), + schema: segs.get(1).cloned(), + warehouse: query(&u, "warehouse"), + role: query(&u, "role"), + }) + } +} + +#[cfg(test)] +mod tests { + use super::{DatabricksConn, SnowflakeConn, TrinoConn}; + + #[test] + fn trino_defaults_to_plain_http_and_upgrades_when_it_must() { + let c = TrinoConn::parse("trino://alice@lake.internal:8080/iceberg/sales").unwrap(); + assert_eq!(c.base, "http://lake.internal:8080"); + assert_eq!(c.user, "alice"); + assert_eq!(c.catalog.as_deref(), Some("iceberg")); + assert_eq!(c.schema.as_deref(), Some("sales")); + + let c = TrinoConn::parse("trino://alice:s%40cret@lake.internal/hive").unwrap(); + assert_eq!(c.base, "https://lake.internal"); + assert_eq!(c.password.as_deref(), Some("s@cret")); + + let c = TrinoConn::parse("trino://alice@lake.internal:8443/hive").unwrap(); + assert_eq!(c.base, "https://lake.internal:8443"); + + let c = TrinoConn::parse("presto://bob@127.0.0.1:9000?ssl=false").unwrap(); + assert_eq!(c.base, "http://127.0.0.1:9000"); + assert_eq!(c.catalog, None); + + assert!(TrinoConn::parse("trino://lake.internal/hive").is_err()); + } + + #[test] + fn databricks_reads_the_http_path_from_the_console() { + let c = DatabricksConn::parse( + "databricks://:dapi123@dbc-abc.cloud.databricks.com/sql/1.0/warehouses/9f2a?catalog=main&schema=sales", + ) + .unwrap(); + assert_eq!(c.base, "https://dbc-abc.cloud.databricks.com"); + assert_eq!(c.token, "dapi123"); + assert_eq!(c.warehouse_id, "9f2a"); + assert_eq!(c.catalog.as_deref(), Some("main")); + assert_eq!(c.schema.as_deref(), Some("sales")); + + let c = DatabricksConn::parse("databricks://dapi123@host?warehouse=w1").unwrap(); + assert_eq!(c.token, "dapi123"); + assert_eq!(c.warehouse_id, "w1"); + + assert!(DatabricksConn::parse("databricks://host/sql/1.0/warehouses/w1").is_err()); + assert!(DatabricksConn::parse("databricks://:t@host").is_err()); + } + + #[test] + fn snowflake_takes_a_token_and_the_session_knobs() { + let c = SnowflakeConn::parse( + "snowflake://:tok@xy12345.eu-central-1.snowflakecomputing.com/ANALYTICS/PUBLIC?warehouse=WH&role=ANALYST", + ) + .unwrap(); + assert_eq!( + c.base, + "https://xy12345.eu-central-1.snowflakecomputing.com" + ); + assert_eq!(c.token, "tok"); + assert_eq!(c.token_type, "PROGRAMMATIC_ACCESS_TOKEN"); + assert_eq!(c.database.as_deref(), Some("ANALYTICS")); + assert_eq!(c.schema.as_deref(), Some("PUBLIC")); + assert_eq!(c.warehouse.as_deref(), Some("WH")); + assert_eq!(c.role.as_deref(), Some("ANALYST")); + + let c = + SnowflakeConn::parse("snowflake://:tok@acct.snowflakecomputing.com?token_type=oauth") + .unwrap(); + assert_eq!(c.token_type, "OAUTH"); + assert!(SnowflakeConn::parse( + "snowflake://:tok@acct.snowflakecomputing.com?token_type=jwt" + ) + .is_err()); + assert!(SnowflakeConn::parse("snowflake://acct.snowflakecomputing.com/DB").is_err()); + } +} diff --git a/crates/utopia-server/src/query_engine/databricks.rs b/crates/utopia-server/src/query_engine/databricks.rs new file mode 100644 index 000000000..7f9eb3793 --- /dev/null +++ b/crates/utopia-server/src/query_engine/databricks.rs @@ -0,0 +1,271 @@ +//! Databricks SQL Statement Execution API（`/api/2.0/sql/statements`）。 +//! 一个 SQL warehouse 后面是 Unity Catalog 的整个湖仓（Delta 为主）， +//! 令牌是 personal access token。结果要 INLINE + JSON_ARRAY：值全是字符串， +//! 按 manifest 里的列类型还原成数与布尔。 + +use super::conn::DatabricksConn; +use super::{ + coerce, rows_to_json_lines, sql_literal, truncate_rows, wrap_limit, QueryEngine, QueryResult, + SchemaColumn, HTTP_POLL_BUDGET, ROW_CAP, +}; +use serde::Deserialize; +use serde_json::json; +use std::time::{Duration, Instant}; + +pub struct DatabricksEngine { + conn: DatabricksConn, +} + +#[derive(Deserialize)] +struct StatementResponse { + statement_id: Option, + status: Status, + manifest: Option, + result: Option, +} + +#[derive(Deserialize)] +struct Status { + state: String, + error: Option, +} + +#[derive(Deserialize)] +struct StatusError { + message: Option, + error_code: Option, +} + +#[derive(Deserialize)] +struct Manifest { + schema: Option, +} + +#[derive(Deserialize)] +struct Schema { + columns: Vec, +} + +#[derive(Deserialize)] +struct ColumnInfo { + name: String, + type_text: Option, +} + +#[derive(Deserialize)] +struct ResultData { + data_array: Option>>, +} + +impl DatabricksEngine { + pub fn new(conn: DatabricksConn) -> Self { + Self { conn } + } + + async fn run(&self, sql: &str) -> anyhow::Result<(Vec, Vec>)> { + let client = super::http()?; + let mut body = json!({ + "warehouse_id": self.conn.warehouse_id, + "statement": sql, + "wait_timeout": "30s", + "on_wait_timeout": "CONTINUE", + "disposition": "INLINE", + "format": "JSON_ARRAY", + "row_limit": ROW_CAP + 1, + }); + if let Some(c) = &self.conn.catalog { + body["catalog"] = json!(c); + } + if let Some(s) = &self.conn.schema { + body["schema"] = json!(s); + } + let mut resp: StatementResponse = client + .post(format!("{}/api/2.0/sql/statements", self.conn.base)) + .bearer_auth(&self.conn.token) + .json(&body) + .send() + .await? + .error_for_status()? + .json() + .await?; + let started = Instant::now(); + loop { + match resp.status.state.as_str() { + "SUCCEEDED" => break, + "PENDING" | "RUNNING" => { + let id = resp + .statement_id + .clone() + .ok_or_else(|| anyhow::anyhow!("Databricks returned no statement_id"))?; + if started.elapsed() > HTTP_POLL_BUDGET { + anyhow::bail!( + "Databricks statement did not finish within {}s", + HTTP_POLL_BUDGET.as_secs() + ); + } + tokio::time::sleep(Duration::from_secs(1)).await; + resp = client + .get(format!("{}/api/2.0/sql/statements/{id}", self.conn.base)) + .bearer_auth(&self.conn.token) + .send() + .await? + .error_for_status()? + .json() + .await?; + } + other => { + let e = resp.status.error.as_ref(); + let code = e + .and_then(|e| e.error_code.clone()) + .map(|c| format!("{c}: ")) + .unwrap_or_default(); + let msg = e + .and_then(|e| e.message.clone()) + .unwrap_or_else(|| format!("statement ended in state {other}")); + anyhow::bail!("{code}{msg}"); + } + } + } + let columns: Vec = resp + .manifest + .and_then(|m| m.schema) + .map(|s| s.columns) + .unwrap_or_default(); + let raw_rows = resp.result.and_then(|r| r.data_array).unwrap_or_default(); + let rows = raw_rows + .into_iter() + .map(|row| { + row.iter() + .enumerate() + .map(|(i, v)| { + let ty = columns + .get(i) + .and_then(|c| c.type_text.as_deref()) + .unwrap_or(""); + coerce(ty, v) + }) + .collect() + }) + .collect(); + Ok((columns.into_iter().map(|c| c.name).collect(), rows)) + } +} + +#[async_trait::async_trait] +impl QueryEngine for DatabricksEngine { + async fn test(&self) -> anyhow::Result<()> { + self.run("SELECT 1").await.map(|_| ()) + } + + async fn fetch_schema(&self) -> anyhow::Result> { + // 带 catalog 就查那个 catalog 的 information_schema；不带就是会话默认的 + let prefix = self + .conn + .catalog + .as_deref() + .map(|c| format!("`{}`.", c.replace('`', "``"))) + .unwrap_or_default(); + let schema_filter = self + .conn + .schema + .as_deref() + .map(|s| format!(" AND table_schema = {}", sql_literal(s))) + .unwrap_or_default(); + let sql = format!( + "SELECT table_schema, table_name, column_name, data_type, comment \ + FROM {prefix}information_schema.columns \ + WHERE table_schema <> 'information_schema'{schema_filter} \ + ORDER BY table_schema, table_name, ordinal_position" + ); + let (_, rows) = self.run(&sql).await?; + Ok(rows.into_iter().map(super::trino::schema_row).collect()) + } + + async fn execute(&self, sql: &str) -> anyhow::Result { + let (columns, rows) = self.run(&wrap_limit(sql)).await?; + let (rows, truncated) = truncate_rows(rows); + Ok(QueryResult { + rows: rows_to_json_lines(&columns, &rows), + truncated, + }) + } +} + +#[cfg(test)] +mod tests { + use super::super::conn::DatabricksConn; + use super::super::QueryEngine; + use super::DatabricksEngine; + use serde_json::json; + use wiremock::matchers::{header, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn conn(server: &MockServer) -> DatabricksConn { + DatabricksConn::parse(&format!( + "databricks://:dapi-test@{}/sql/1.0/warehouses/wh1?catalog=main&ssl=false", + server.uri().trim_start_matches("http://") + )) + .unwrap() + } + + #[tokio::test] + async fn polls_until_succeeded_and_restores_types() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/2.0/sql/statements")) + .and(header("authorization", "Bearer dapi-test")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "statement_id": "s1", + "status": { "state": "PENDING" } + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/api/2.0/sql/statements/s1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "statement_id": "s1", + "status": { "state": "SUCCEEDED" }, + "manifest": { "schema": { "columns": [ + { "name": "region", "type_text": "STRING", "position": 0 }, + { "name": "total", "type_text": "DECIMAL(12,2)", "position": 1 }, + { "name": "active", "type_text": "BOOLEAN", "position": 2 } + ] } }, + "result": { "data_array": [ ["east", "12.50", "true"], ["west", null, "false"] ] } + }))) + .expect(1) + .mount(&server) + .await; + + let out = DatabricksEngine::new(conn(&server)) + .execute("SELECT region, total, active FROM orders") + .await + .unwrap(); + assert_eq!( + out.rows, + vec![ + r#"{"region":"east","total":12.5,"active":true}"#, + r#"{"region":"west","total":null,"active":false}"# + ] + ); + } + + #[tokio::test] + async fn a_failed_statement_reports_the_message() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/2.0/sql/statements")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "statement_id": "s2", + "status": { "state": "FAILED", "error": { "error_code": "BAD_REQUEST", "message": "TABLE_OR_VIEW_NOT_FOUND: nope" } } + }))) + .mount(&server) + .await; + let err = DatabricksEngine::new(conn(&server)) + .execute("SELECT * FROM nope") + .await + .unwrap_err() + .to_string(); + assert!(err.contains("TABLE_OR_VIEW_NOT_FOUND"), "{err}"); + } +} diff --git a/crates/utopia-server/src/query_engine/mod.rs b/crates/utopia-server/src/query_engine/mod.rs new file mode 100644 index 000000000..2bd171322 --- /dev/null +++ b/crates/utopia-server/src/query_engine/mod.rs @@ -0,0 +1,349 @@ +//! 问数查询引擎：trait 接缝（BlobStore 同手法）+ 引擎无关的安全闸。 +//! +//! 引擎按协议族扩，不按产品名扩：postgres 线协议（`postgres.rs`）→ HTTP 族—— +//! `trino.rs` 一个顶起 Iceberg / Delta / Hive 整个湖仓生态，`databricks.rs`、 +//! `snowflake.rs` 各走自家的 SQL REST API。挂载模型与注册表引擎无关，加引擎只放宽 +//! 一条 CHECK。连接串是唯一的输入：引擎由 scheme 决定（[`engine_from_conn`]）， +//! 剩下的部分各引擎自己拆（`conn.rs`），凭据只在服务端流转。 +//! +//! 安全闸（纵深防御，不信任模型）： +//! 1. sqlparser 解析：仅放行单条 SELECT/WITH（含 CTE），拒绝 DML/DDL/多语句/SELECT INTO。 +//! 按引擎选方言；sqlparser 没有 Trino 方言，Generic 是它的超集 +//! 2. 强制外包一层 LIMIT（cap+1 探测截断） +//! 3. 会话级只读 + 语句超时（引擎各自的机制，parser 万一漏网也写不进去）。 +//! HTTP 族没有会话，只有语句超时——只读靠第 1 层，这是它们比线协议少的那一层 +//! 4. 结果统一为 JSON Lines：PG 让库自己转；HTTP 族拿到列名与值后在这里拼，列序保留 + +mod conn; +mod databricks; +mod postgres; +mod snowflake; +mod trino; + +use sqlparser::ast::Statement; +use sqlparser::dialect::{DatabricksDialect, GenericDialect, PostgreSqlDialect, SnowflakeDialect}; +use sqlparser::parser::Parser; +use std::time::Duration; + +/// 行数上限（外包 LIMIT cap+1，第 201 行只用来判断截断）。 +pub const ROW_CAP: usize = 200; +pub(crate) const STATEMENT_TIMEOUT_SECS: u32 = 10; +/// HTTP 族：单次请求的超时，与整条语句从提交到拿完结果的轮询预算 +pub(crate) const HTTP_REQUEST_TIMEOUT: Duration = Duration::from_secs(20); +pub(crate) const HTTP_POLL_BUDGET: Duration = Duration::from_secs(30); + +/// 注册表里 `engine` 列的取值。迁移里的 CHECK 与这张表要一致 +pub const ENGINES: &[&str] = &["postgres", "trino", "databricks", "snowflake"]; + +#[derive(Debug)] +pub struct QueryResult { + /// 每行一个 JSON 对象文本（键序 = 查询列序） + pub rows: Vec, + pub truncated: bool, +} + +#[derive(Debug)] +pub struct SchemaColumn { + pub schema: String, + pub table: String, + pub column: String, + pub data_type: String, + pub comment: Option, +} + +#[async_trait::async_trait] +pub trait QueryEngine: Send + Sync { + async fn test(&self) -> anyhow::Result<()>; + async fn fetch_schema(&self) -> anyhow::Result>; + /// 执行已过闸的 SELECT。实现自身仍需强制只读会话与超时（纵深防御）。 + async fn execute(&self, sql: &str) -> anyhow::Result; +} + +/// scheme → 引擎名。界面只有一个连接串输入框，这里是它唯一的分派点。 +pub fn engine_from_conn(conn: &str) -> Option<&'static str> { + let scheme = conn.trim().split("://").next()?.to_ascii_lowercase(); + match scheme.as_str() { + "postgres" | "postgresql" => Some("postgres"), + "trino" | "presto" => Some("trino"), + "databricks" => Some("databricks"), + "snowflake" => Some("snowflake"), + _ => None, + } +} + +/// 引擎工厂。conn 凭据只在服务端流转。 +pub fn engine_for(engine: &str, conn: &str) -> anyhow::Result> { + match engine { + "postgres" => Ok(Box::new(postgres::PostgresEngine::new(conn))), + "trino" => Ok(Box::new(trino::TrinoEngine::new(conn::TrinoConn::parse( + conn, + )?))), + "databricks" => Ok(Box::new(databricks::DatabricksEngine::new( + conn::DatabricksConn::parse(conn)?, + ))), + "snowflake" => Ok(Box::new(snowflake::SnowflakeEngine::new( + conn::SnowflakeConn::parse(conn)?, + ))), + other => anyhow::bail!("Unsupported engine: {other}"), + } +} + +/// 安全闸第 1 层：按引擎方言解析并校验，返回规整后的语句文本。 +pub fn guard_sql_for(engine: &str, sql: &str) -> anyhow::Result { + let cleaned = sql.trim().trim_end_matches(';').trim(); + if cleaned.is_empty() { + anyhow::bail!("Empty SQL"); + } + let parsed = match engine { + "databricks" => Parser::parse_sql(&DatabricksDialect {}, cleaned), + "snowflake" => Parser::parse_sql(&SnowflakeDialect {}, cleaned), + "trino" => Parser::parse_sql(&GenericDialect {}, cleaned), + _ => Parser::parse_sql(&PostgreSqlDialect {}, cleaned), + }; + let statements = parsed.map_err(|e| anyhow::anyhow!("SQL parse error: {e}"))?; + if statements.len() != 1 { + anyhow::bail!("Exactly one statement is allowed"); + } + match &statements[0] { + Statement::Query(_) => Ok(cleaned.to_string()), + other => anyhow::bail!( + "Read-only: only SELECT/WITH queries are allowed (got {})", + statement_kind(other) + ), + } +} + +fn statement_kind(s: &Statement) -> &'static str { + match s { + Statement::Insert { .. } => "INSERT", + Statement::Update { .. } => "UPDATE", + Statement::Delete { .. } => "DELETE", + Statement::CreateTable { .. } => "CREATE TABLE", + Statement::Drop { .. } => "DROP", + Statement::AlterTable { .. } => "ALTER TABLE", + Statement::Truncate { .. } => "TRUNCATE", + _ => "a non-SELECT statement", + } +} + +/// 第 2 层：外包一层 LIMIT。三个 HTTP 引擎都认这个写法；PG 有自己的 row_to_json 版本 +pub(crate) fn wrap_limit(sql: &str) -> String { + format!("SELECT * FROM ( {sql} ) AS _q LIMIT {}", ROW_CAP + 1) +} + +/// 第 201 行只用来判断截断，不交给模型 +pub(crate) fn truncate_rows(mut rows: Vec) -> (Vec, bool) { + let truncated = rows.len() > ROW_CAP; + rows.truncate(ROW_CAP); + (rows, truncated) +} + +/// HTTP 族共用：「列名 + 行值」拼成 JSON Lines。手拼而不是 `serde_json::Map`， +/// 后者不开 `preserve_order` 就按键排序，而列序是查询写下的顺序，模型读表靠它 +pub(crate) fn rows_to_json_lines( + columns: &[String], + rows: &[Vec], +) -> Vec { + rows.iter() + .map(|row| { + let mut line = String::from("{"); + for (i, col) in columns.iter().enumerate() { + if i > 0 { + line.push(','); + } + line.push_str(&serde_json::to_string(col).unwrap_or_else(|_| "\"?\"".into())); + line.push(':'); + let value = row.get(i).cloned().unwrap_or(serde_json::Value::Null); + line.push_str(&value.to_string()); + } + line.push('}'); + line + }) + .collect() +} + +/// Databricks 的 JSON_ARRAY 与 Snowflake 的 data 把每个值都给成字符串（或 null）。 +/// 按列类型把数与布尔还原，其余留字符串——模型对 `"42"` 和 `42` 的算术不一样 +pub(crate) fn coerce(type_name: &str, raw: &serde_json::Value) -> serde_json::Value { + let serde_json::Value::String(s) = raw else { + return raw.clone(); + }; + let ty = type_name.to_ascii_uppercase(); + const NUMERIC: &[&str] = &[ + "INT", "LONG", "SHORT", "BYTE", "FLOAT", "DOUBLE", "DECIMAL", "NUMBER", "FIXED", "REAL", + "NUMERIC", + ]; + // INTERVAL 也含 "INT"：解析不成数就原样留下，不会误伤 + if NUMERIC.iter().any(|k| ty.contains(k)) { + if let Ok(n) = s.parse::() { + return n.into(); + } + if let Ok(f) = s.parse::() { + if let Some(n) = serde_json::Number::from_f64(f) { + return serde_json::Value::Number(n); + } + } + } + if ty.starts_with("BOOL") { + match s.as_str() { + "true" | "TRUE" => return true.into(), + "false" | "FALSE" => return false.into(), + _ => {} + } + } + raw.clone() +} + +/// 单引号字面量的转义：schema 名进 information_schema 的 WHERE 子句 +pub(crate) fn sql_literal(s: &str) -> String { + format!("'{}'", s.replace('\'', "''")) +} + +/// HTTP 族共用的客户端。 +/// +/// **代理策略是显式的**：回环地址与 `NO_PROXY` 里的主机直连，其余按 `HTTPS_PROXY` / +/// `HTTP_PROXY` / `ALL_PROXY` 走。不用 reqwest 的系统代理探测——Windows 上它读注册表， +/// 而注册表里 `127.*` 这种绕过写法它认不全，本机的替身服务会被送进代理拿回 502。 +/// 服务进程该看环境变量，这条规矩与 docker-compose 里的写法一致 +pub(crate) fn http() -> anyhow::Result { + Ok(reqwest::Client::builder() + .timeout(HTTP_REQUEST_TIMEOUT) + .user_agent("utopia") + .proxy(reqwest::Proxy::custom(|url: &reqwest::Url| proxy_for(url))) + .build()?) +} + +fn proxy_for(url: &reqwest::Url) -> Option { + let host = url.host_str()?; + let loopback = host.eq_ignore_ascii_case("localhost") + || host + .trim_matches(|c| c == '[' || c == ']') + .parse::() + .map(|ip| ip.is_loopback()) + .unwrap_or(false); + if loopback || no_proxy_matches(host) { + return None; + } + let keys: &[&str] = if url.scheme() == "https" { + &["HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"] + } else { + &["HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy"] + }; + keys.iter() + .find_map(|k| std::env::var(k).ok()) + .filter(|v| !v.trim().is_empty()) + .and_then(|v| reqwest::Url::parse(v.trim()).ok()) +} + +/// `NO_PROXY=localhost,127.0.0.1,.internal,corp.example` 的常见写法：整名相等， +/// 或者以点开头的后缀匹配 +fn no_proxy_matches(host: &str) -> bool { + let raw = std::env::var("NO_PROXY") + .or_else(|_| std::env::var("no_proxy")) + .unwrap_or_default(); + raw.split(',') + .map(str::trim) + .filter(|p| !p.is_empty() && *p != "*") + .any(|p| { + let p = p.trim_start_matches('.'); + host.eq_ignore_ascii_case(p) + || host + .to_ascii_lowercase() + .ends_with(&format!(".{}", p.to_ascii_lowercase())) + }) + || raw.split(',').any(|p| p.trim() == "*") +} + +#[cfg(test)] +mod tests { + use super::{coerce, engine_from_conn, guard_sql_for, rows_to_json_lines}; + use serde_json::json; + + fn guard_sql(sql: &str) -> anyhow::Result { + guard_sql_for("postgres", sql) + } + + #[test] + fn allows_select_and_cte() { + assert!(guard_sql("SELECT region, sum(amount) FROM orders GROUP BY 1").is_ok()); + assert!(guard_sql("WITH t AS (SELECT 1 AS x) SELECT * FROM t;").is_ok()); + } + + #[test] + fn rejects_writes_and_ddl() { + for bad in [ + "UPDATE orders SET amount = 0", + "DELETE FROM orders", + "INSERT INTO orders (region) VALUES ('east')", + "DROP TABLE orders", + "TRUNCATE orders", + "CREATE TABLE t (id int)", + "ALTER TABLE orders ADD COLUMN x int", + ] { + assert!(guard_sql(bad).is_err(), "should reject: {bad}"); + } + } + + #[test] + fn rejects_multi_statement() { + assert!(guard_sql("SELECT 1; DROP TABLE orders").is_err()); + assert!(guard_sql("").is_err()); + } + + #[test] + fn every_dialect_keeps_the_same_gate() { + for engine in ["postgres", "trino", "databricks", "snowflake"] { + assert!( + guard_sql_for(engine, "SELECT a FROM t WHERE b > 1").is_ok(), + "{engine}" + ); + assert!(guard_sql_for(engine, "DELETE FROM t").is_err(), "{engine}"); + assert!( + guard_sql_for(engine, "SELECT 1; SELECT 2").is_err(), + "{engine}" + ); + } + // 各家的方言细节：反引号、双冒号转型，都要过得去 + assert!(guard_sql_for("databricks", "SELECT `region` FROM main.sales.orders").is_ok()); + assert!(guard_sql_for("snowflake", "SELECT amount::number FROM db.public.orders").is_ok()); + assert!(guard_sql_for("trino", "SELECT count(*) FROM hive.default.orders").is_ok()); + } + + #[test] + fn engine_follows_the_scheme() { + assert_eq!(engine_from_conn("postgres://u:p@h/db"), Some("postgres")); + assert_eq!(engine_from_conn("postgresql://u:p@h/db"), Some("postgres")); + assert_eq!(engine_from_conn("trino://u@h:8443/hive"), Some("trino")); + assert_eq!(engine_from_conn("presto://u@h/hive"), Some("trino")); + assert_eq!( + engine_from_conn("databricks://:t@h/sql/1.0/warehouses/x"), + Some("databricks") + ); + assert_eq!( + engine_from_conn("snowflake://:t@a.snowflakecomputing.com/db"), + Some("snowflake") + ); + assert_eq!(engine_from_conn("mysql://u@h/db"), None); + assert_eq!(engine_from_conn("garbage"), None); + } + + #[test] + fn json_lines_keep_column_order() { + let cols = vec!["zeta".to_string(), "alpha".to_string()]; + let rows = vec![vec![json!(1), json!("x")], vec![json!(null)]]; + assert_eq!( + rows_to_json_lines(&cols, &rows), + vec![r#"{"zeta":1,"alpha":"x"}"#, r#"{"zeta":null,"alpha":null}"#] + ); + } + + #[test] + fn strings_come_back_as_numbers_when_the_column_says_so() { + assert_eq!(coerce("DOUBLE", &json!("12.5")), json!(12.5)); + assert_eq!(coerce("fixed", &json!("42")), json!(42)); + assert_eq!(coerce("BOOLEAN", &json!("true")), json!(true)); + assert_eq!(coerce("STRING", &json!("42")), json!("42")); + assert_eq!(coerce("INTERVAL", &json!("1 day")), json!("1 day")); + assert_eq!(coerce("DOUBLE", &json!(null)), json!(null)); + } +} diff --git a/crates/utopia-server/src/query_engine/postgres.rs b/crates/utopia-server/src/query_engine/postgres.rs new file mode 100644 index 000000000..5bba3a4ef --- /dev/null +++ b/crates/utopia-server/src/query_engine/postgres.rs @@ -0,0 +1,93 @@ +//! Postgres 族（顺带覆盖 Greenplum / Timescale 等 PG 兼容系）。线协议直连， +//! 是四个引擎里唯一有会话可设只读的那个。 + +use super::{QueryEngine, QueryResult, SchemaColumn, ROW_CAP, STATEMENT_TIMEOUT_SECS}; +use sqlx::postgres::PgPoolOptions; +use sqlx::Row; +use std::time::Duration; + +pub struct PostgresEngine { + conn: String, +} + +impl PostgresEngine { + pub fn new(conn: &str) -> Self { + Self { + conn: conn.to_string(), + } + } + + async fn pool(&self) -> anyhow::Result { + Ok(PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(Duration::from_secs(5)) + .connect(&self.conn) + .await?) + } +} + +#[async_trait::async_trait] +impl QueryEngine for PostgresEngine { + async fn test(&self) -> anyhow::Result<()> { + let pool = self.pool().await?; + sqlx::query("SELECT 1").execute(&pool).await?; + pool.close().await; + Ok(()) + } + + async fn fetch_schema(&self) -> anyhow::Result> { + let pool = self.pool().await?; + let rows: Vec<(String, String, String, String, Option)> = sqlx::query_as( + "SELECT c.table_schema, c.table_name, c.column_name, + c.data_type, pgd.description + FROM information_schema.columns c + LEFT JOIN pg_catalog.pg_statio_all_tables st + ON st.schemaname = c.table_schema AND st.relname = c.table_name + LEFT JOIN pg_catalog.pg_description pgd + ON pgd.objoid = st.relid AND pgd.objsubid = c.ordinal_position + WHERE c.table_schema NOT IN ('pg_catalog', 'information_schema') + ORDER BY c.table_schema, c.table_name, c.ordinal_position", + ) + .fetch_all(&pool) + .await?; + pool.close().await; + Ok(rows + .into_iter() + .map(|(schema, table, column, data_type, comment)| SchemaColumn { + schema, + table, + column, + data_type, + comment, + }) + .collect()) + } + + async fn execute(&self, sql: &str) -> anyhow::Result { + let pool = self.pool().await?; + // 纵深防御第 3 层：会话级只读 + 超时（parser 漏网也写不进去、跑不死库） + sqlx::query("SET default_transaction_read_only = on") + .execute(&pool) + .await?; + sqlx::query(&format!( + "SET statement_timeout = '{STATEMENT_TIMEOUT_SECS}s'" + )) + .execute(&pool) + .await?; + // 第 2 层：外包 LIMIT；row_to_json 让 PG 全权处理类型→JSON（文本键序保留列序） + let wrapped = format!( + "SELECT row_to_json(_q)::text AS _j FROM ( {sql} ) AS _q LIMIT {}", + ROW_CAP + 1 + ); + let fetched = sqlx::query(&wrapped).fetch_all(&pool).await?; + pool.close().await; + + let truncated = fetched.len() > ROW_CAP; + let rows = fetched + .into_iter() + .take(ROW_CAP) + .map(|r| r.try_get::("_j").unwrap_or_else(|_| "{}".into())) + .collect(); + Ok(QueryResult { rows, truncated }) + } +} diff --git a/crates/utopia-server/src/query_engine/snowflake.rs b/crates/utopia-server/src/query_engine/snowflake.rs new file mode 100644 index 000000000..ff8106117 --- /dev/null +++ b/crates/utopia-server/src/query_engine/snowflake.rs @@ -0,0 +1,257 @@ +//! Snowflake SQL API v2（`/api/v2/statements`）。同步提交（`async=false`）拿不完的 +//! 语句回 202，沿 statementHandle 轮询。值全是字符串，按 rowType 还原数与布尔。 +//! +//! 只收令牌，不收密码：programmatic access token 或 OAuth。密钥对 JWT 要本地签名， +//! 这一版不做——见 `conn.rs`。 + +use super::conn::SnowflakeConn; +use super::{ + coerce, rows_to_json_lines, sql_literal, truncate_rows, wrap_limit, QueryEngine, QueryResult, + SchemaColumn, HTTP_POLL_BUDGET, STATEMENT_TIMEOUT_SECS, +}; +use reqwest::StatusCode; +use serde::Deserialize; +use serde_json::json; +use std::time::{Duration, Instant}; + +pub struct SnowflakeEngine { + conn: SnowflakeConn, +} + +#[derive(Deserialize)] +struct StatementResponse { + #[serde(rename = "resultSetMetaData")] + meta: Option, + data: Option>>, + message: Option, + code: Option, + #[serde(rename = "statementHandle")] + handle: Option, +} + +#[derive(Deserialize)] +struct Meta { + #[serde(rename = "rowType")] + row_type: Vec, +} + +#[derive(Deserialize)] +struct RowType { + name: String, + #[serde(rename = "type")] + ty: String, +} + +impl SnowflakeEngine { + pub fn new(conn: SnowflakeConn) -> Self { + Self { conn } + } + + fn request(&self, r: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + r.bearer_auth(&self.conn.token) + .header("X-Snowflake-Authorization-Token-Type", self.conn.token_type) + .header("Accept", "application/json") + } + + async fn run(&self, sql: &str) -> anyhow::Result<(Vec, Vec>)> { + let client = super::http()?; + let mut body = json!({ + "statement": sql, + "timeout": STATEMENT_TIMEOUT_SECS, + "parameters": { "MULTI_STATEMENT_COUNT": "1" }, + }); + for (key, value) in [ + ("database", &self.conn.database), + ("schema", &self.conn.schema), + ("warehouse", &self.conn.warehouse), + ("role", &self.conn.role), + ] { + if let Some(v) = value { + body[key] = json!(v); + } + } + let mut http = self + .request(client.post(format!("{}/api/v2/statements?async=false", self.conn.base))) + .json(&body) + .send() + .await?; + let started = Instant::now(); + // 202 = 还在跑；其余非 2xx 的 body 里带 message + while http.status() == StatusCode::ACCEPTED { + let partial: StatementResponse = http.json().await?; + let handle = partial.handle.ok_or_else(|| { + anyhow::anyhow!("Snowflake returned 202 without a statementHandle") + })?; + if started.elapsed() > HTTP_POLL_BUDGET { + anyhow::bail!( + "Snowflake statement did not finish within {}s", + HTTP_POLL_BUDGET.as_secs() + ); + } + tokio::time::sleep(Duration::from_secs(1)).await; + http = self + .request(client.get(format!("{}/api/v2/statements/{handle}", self.conn.base))) + .send() + .await?; + } + if !http.status().is_success() { + let status = http.status(); + let text = http.text().await.unwrap_or_default(); + let msg = serde_json::from_str::(&text) + .ok() + .and_then(|r| r.message) + .unwrap_or(text); + anyhow::bail!("Snowflake {status}: {msg}"); + } + let resp: StatementResponse = http.json().await?; + if let (Some(code), Some(message)) = (&resp.code, &resp.message) { + // 2xx 里也可能带业务错误码；090001 是 "statement executed successfully" + if code != "090001" && resp.meta.is_none() { + anyhow::bail!("Snowflake {code}: {message}"); + } + } + let types: Vec = resp.meta.map(|m| m.row_type).unwrap_or_default(); + let rows = resp + .data + .unwrap_or_default() + .into_iter() + .map(|row| { + row.iter() + .enumerate() + .map(|(i, v)| coerce(types.get(i).map(|t| t.ty.as_str()).unwrap_or(""), v)) + .collect() + }) + .collect(); + Ok((types.into_iter().map(|t| t.name).collect(), rows)) + } +} + +#[async_trait::async_trait] +impl QueryEngine for SnowflakeEngine { + async fn test(&self) -> anyhow::Result<()> { + self.run("SELECT 1").await.map(|_| ()) + } + + async fn fetch_schema(&self) -> anyhow::Result> { + let database = self.conn.database.as_deref().ok_or_else(|| { + anyhow::anyhow!("snowflake://: put the database in the connection string (snowflake://:TOKEN@account/DATABASE) so the schema can be read") + })?; + let schema_filter = self + .conn + .schema + .as_deref() + .map(|s| format!(" AND table_schema = {}", sql_literal(s))) + .unwrap_or_default(); + let sql = format!( + "SELECT table_schema, table_name, column_name, data_type, comment \ + FROM \"{}\".information_schema.columns \ + WHERE table_schema <> 'INFORMATION_SCHEMA'{schema_filter} \ + ORDER BY table_schema, table_name, ordinal_position", + database.replace('"', "\"\"") + ); + let (_, rows) = self.run(&sql).await?; + Ok(rows.into_iter().map(super::trino::schema_row).collect()) + } + + async fn execute(&self, sql: &str) -> anyhow::Result { + let (columns, rows) = self.run(&wrap_limit(sql)).await?; + let (rows, truncated) = truncate_rows(rows); + Ok(QueryResult { + rows: rows_to_json_lines(&columns, &rows), + truncated, + }) + } +} + +#[cfg(test)] +mod tests { + use super::super::conn::SnowflakeConn; + use super::super::QueryEngine; + use super::SnowflakeEngine; + use serde_json::json; + use wiremock::matchers::{header, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn conn(server: &MockServer) -> SnowflakeConn { + SnowflakeConn::parse(&format!( + "snowflake://:pat-test@{}/ANALYTICS/PUBLIC?warehouse=WH&ssl=false", + server.uri().trim_start_matches("http://") + )) + .unwrap() + } + + #[tokio::test] + async fn a_synchronous_answer_is_typed_by_row_type() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/v2/statements")) + .and(header("authorization", "Bearer pat-test")) + .and(header( + "X-Snowflake-Authorization-Token-Type", + "PROGRAMMATIC_ACCESS_TOKEN", + )) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "resultSetMetaData": { "numRows": 1, "rowType": [ + { "name": "REGION", "type": "text" }, + { "name": "TOTAL", "type": "fixed", "scale": 2 } + ] }, + "data": [ ["east", "42.10"] ], + "code": "090001", + "statementHandle": "h1", + "message": "Statement executed successfully." + }))) + .expect(1) + .mount(&server) + .await; + let out = SnowflakeEngine::new(conn(&server)) + .execute("SELECT region, total FROM orders") + .await + .unwrap(); + assert_eq!(out.rows, vec![r#"{"REGION":"east","TOTAL":42.1}"#]); + } + + #[tokio::test] + async fn a_202_is_polled_until_the_answer_arrives() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/v2/statements")) + .respond_with(ResponseTemplate::new(202).set_body_json(json!({ + "code": "333334", "statementHandle": "h2", "message": "Asynchronous execution in progress." + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/api/v2/statements/h2")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "resultSetMetaData": { "rowType": [ { "name": "N", "type": "fixed" } ] }, + "data": [ ["1"] ], "code": "090001", "statementHandle": "h2" + }))) + .expect(1) + .mount(&server) + .await; + let out = SnowflakeEngine::new(conn(&server)) + .execute("SELECT 1 AS n") + .await + .unwrap(); + assert_eq!(out.rows, vec![r#"{"N":1}"#]); + } + + #[tokio::test] + async fn an_error_body_is_surfaced() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/v2/statements")) + .respond_with(ResponseTemplate::new(422).set_body_json(json!({ + "code": "002003", "message": "SQL compilation error: Object 'NOPE' does not exist" + }))) + .mount(&server) + .await; + let err = SnowflakeEngine::new(conn(&server)) + .execute("SELECT * FROM nope") + .await + .unwrap_err() + .to_string(); + assert!(err.contains("does not exist"), "{err}"); + } +} diff --git a/crates/utopia-server/src/query_engine/trino.rs b/crates/utopia-server/src/query_engine/trino.rs new file mode 100644 index 000000000..4b2c40dc3 --- /dev/null +++ b/crates/utopia-server/src/query_engine/trino.rs @@ -0,0 +1,266 @@ +//! Trino（旧名 Presto）：REST 协议 `POST /v1/statement`，然后沿 `nextUri` 一页页取。 +//! 一个引擎顶起整个湖仓——Iceberg / Delta / Hive / Hudi 都是它的 catalog， +//! 换格式不换协议。Starburst 同协议。 +//! +//! 没有会话可设只读：超时靠 `X-Trino-Session: query_max_execution_time`， +//! 只读靠 `guard_sql_for`。 + +use super::conn::TrinoConn; +use super::{ + rows_to_json_lines, sql_literal, truncate_rows, wrap_limit, QueryEngine, QueryResult, + SchemaColumn, HTTP_POLL_BUDGET, STATEMENT_TIMEOUT_SECS, +}; +use base64::Engine as _; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; +use serde::Deserialize; +use std::time::Instant; + +pub struct TrinoEngine { + conn: TrinoConn, +} + +#[derive(Deserialize)] +struct Column { + name: String, +} + +#[derive(Deserialize)] +struct Page { + #[serde(rename = "nextUri")] + next_uri: Option, + columns: Option>, + data: Option>>, + error: Option, +} + +#[derive(Deserialize)] +struct TrinoError { + message: String, + #[serde(rename = "errorName")] + error_name: Option, +} + +impl TrinoEngine { + pub fn new(conn: TrinoConn) -> Self { + Self { conn } + } + + fn headers(&self) -> anyhow::Result { + let mut h = HeaderMap::new(); + h.insert("X-Trino-User", HeaderValue::from_str(&self.conn.user)?); + h.insert("X-Trino-Source", HeaderValue::from_static("utopia")); + h.insert( + "X-Trino-Session", + HeaderValue::from_str(&format!( + "query_max_execution_time={STATEMENT_TIMEOUT_SECS}s" + ))?, + ); + if let Some(c) = &self.conn.catalog { + h.insert("X-Trino-Catalog", HeaderValue::from_str(c)?); + } + if let Some(s) = &self.conn.schema { + h.insert("X-Trino-Schema", HeaderValue::from_str(s)?); + } + if let Some(p) = &self.conn.password { + let raw = format!("{}:{p}", self.conn.user); + let token = base64::engine::general_purpose::STANDARD.encode(raw); + h.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Basic {token}"))?, + ); + } + Ok(h) + } + + /// 提交并沿 nextUri 收完：列在第一个带 columns 的页上，数据分页累积 + async fn run(&self, sql: &str) -> anyhow::Result<(Vec, Vec>)> { + let client = super::http()?; + let headers = self.headers()?; + let mut page: Page = client + .post(format!("{}/v1/statement", self.conn.base)) + .headers(headers.clone()) + .body(sql.to_string()) + .send() + .await? + .error_for_status()? + .json() + .await?; + let started = Instant::now(); + let mut columns: Option> = None; + let mut rows = Vec::new(); + loop { + if let Some(e) = page.error { + let name = e.error_name.map(|n| format!("{n}: ")).unwrap_or_default(); + anyhow::bail!("{name}{}", e.message); + } + if columns.is_none() { + columns = page + .columns + .take() + .map(|cs| cs.into_iter().map(|c| c.name).collect()); + } + if let Some(d) = page.data.take() { + rows.extend(d); + } + let Some(next) = page.next_uri.take() else { + break; + }; + if started.elapsed() > HTTP_POLL_BUDGET { + anyhow::bail!( + "Trino query did not finish within {}s", + HTTP_POLL_BUDGET.as_secs() + ); + } + page = client + .get(&next) + .headers(headers.clone()) + .send() + .await? + .error_for_status()? + .json() + .await?; + } + Ok((columns.unwrap_or_default(), rows)) + } +} + +#[async_trait::async_trait] +impl QueryEngine for TrinoEngine { + async fn test(&self) -> anyhow::Result<()> { + self.run("SELECT 1").await.map(|_| ()) + } + + async fn fetch_schema(&self) -> anyhow::Result> { + let catalog = self.conn.catalog.as_deref().ok_or_else(|| { + anyhow::anyhow!("trino://: put the catalog in the connection string (trino://user@host/CATALOG) so the schema can be read") + })?; + let schema_filter = self + .conn + .schema + .as_deref() + .map(|s| format!(" AND table_schema = {}", sql_literal(s))) + .unwrap_or_default(); + let sql = format!( + "SELECT table_schema, table_name, column_name, data_type, comment \ + FROM \"{}\".information_schema.columns \ + WHERE table_schema <> 'information_schema'{schema_filter} \ + ORDER BY table_schema, table_name, ordinal_position", + catalog.replace('"', "\"\"") + ); + let (_, rows) = self.run(&sql).await?; + Ok(rows.into_iter().map(schema_row).collect()) + } + + async fn execute(&self, sql: &str) -> anyhow::Result { + let (columns, rows) = self.run(&wrap_limit(sql)).await?; + let (rows, truncated) = truncate_rows(rows); + Ok(QueryResult { + rows: rows_to_json_lines(&columns, &rows), + truncated, + }) + } +} + +/// information_schema 的一行 → SchemaColumn（值可能是 null，comment 常是） +pub(crate) fn schema_row(row: Vec) -> SchemaColumn { + let text = |i: usize| -> String { + row.get(i) + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string() + }; + SchemaColumn { + schema: text(0), + table: text(1), + column: text(2), + data_type: text(3), + comment: row + .get(4) + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .map(str::to_string), + } +} + +#[cfg(test)] +mod tests { + use super::super::conn::TrinoConn; + use super::super::QueryEngine; + use super::TrinoEngine; + use serde_json::json; + use wiremock::matchers::{body_string_contains, header, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + #[tokio::test] + async fn follows_next_uri_and_keeps_column_order() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(header("X-Trino-User", "alice")) + .and(header("X-Trino-Catalog", "hive")) + .and(body_string_contains("LIMIT 201")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "q1", + "nextUri": format!("{}/v1/statement/q1/1", server.uri()), + "stats": { "state": "QUEUED" } + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/statement/q1/1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "q1", + "columns": [ { "name": "region", "type": "varchar" }, { "name": "total", "type": "double" } ], + "data": [ ["east", 12.5], ["west", 3] ], + "stats": { "state": "FINISHED" } + }))) + .expect(1) + .mount(&server) + .await; + + let uri = server.uri(); + let conn = TrinoConn::parse(&format!( + "trino://alice@{}/hive/default?ssl=false", + uri.trim_start_matches("http://") + )) + .unwrap(); + let out = TrinoEngine::new(conn) + .execute("SELECT region, total FROM orders") + .await + .unwrap(); + assert_eq!( + out.rows, + vec![ + r#"{"region":"east","total":12.5}"#, + r#"{"region":"west","total":3}"# + ] + ); + assert!(!out.truncated); + } + + #[tokio::test] + async fn a_trino_error_page_becomes_an_error() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "q2", + "error": { "message": "line 1:8: Table 'hive.default.nope' does not exist", "errorName": "TABLE_NOT_FOUND" }, + "stats": { "state": "FAILED" } + }))) + .mount(&server) + .await; + let conn = TrinoConn::parse(&format!( + "trino://alice@{}/hive?ssl=false", + server.uri().trim_start_matches("http://") + )) + .unwrap(); + let err = TrinoEngine::new(conn) + .execute("SELECT * FROM nope") + .await + .unwrap_err() + .to_string(); + assert!(err.contains("TABLE_NOT_FOUND"), "{err}"); + } +} diff --git a/crates/utopia-store/src/datasources.rs b/crates/utopia-store/src/datasources.rs index 5fad63897..6e3ec3a9e 100644 --- a/crates/utopia-store/src/datasources.rs +++ b/crates/utopia-store/src/datasources.rs @@ -18,17 +18,16 @@ type DataSourceRow = ( Option, ); -/// 连接串 → 无凭据摘要（host:port/db）。解析失败给占位符，绝不回显原串。 +/// 连接串 → 无凭据摘要（host[:port]/path）。解析失败给占位符，绝不回显原串。 +/// 端口没写就不补：四种 scheme 的默认端口各不相同，补错比不补更误导 pub fn conn_summary(conn: &str) -> String { url::Url::parse(conn) .ok() .map(|u| { format!( - "{}:{}{}", + "{}{}{}", u.host_str().unwrap_or("?"), - u.port() - .map(|p| p.to_string()) - .unwrap_or_else(|| "5432".into()), + u.port().map(|p| format!(":{p}")).unwrap_or_default(), u.path() ) }) @@ -77,16 +76,12 @@ pub async fn create( "Data source name is required", )); } - if engine != "postgres" { - return Err(AppError::invalid( - "only_postgres", - "Only the postgres engine is supported for now", - )); - } - if !conn_string.starts_with("postgres://") && !conn_string.starts_with("postgresql://") { + // 引擎由调用方按连接串的 scheme 定（`query_engine::engine_from_conn`）； + // 允许的取值在迁移 0020 的 CHECK 里，这里不再复制一份 + if engine.is_empty() || conn_string.trim().is_empty() { return Err(AppError::invalid( "bad_conn_string", - "Connection string must start with postgres://", + "A connection string is required", )); } let id = Uuid::now_v7(); diff --git a/docs/decisions/0018-the-lakehouse-is-one-protocol-away.md b/docs/decisions/0018-the-lakehouse-is-one-protocol-away.md new file mode 100644 index 000000000..91ccdd460 --- /dev/null +++ b/docs/decisions/0018-the-lakehouse-is-one-protocol-away.md @@ -0,0 +1,58 @@ +# 0018 · The lakehouse is one protocol away + +- **Status**: implemented · `trino` / `databricks` / `snowflake` engines in `query_engine/` (migration `0021` widens the `engine` CHECK) · every engine is covered by protocol replays only (wiremock); **none has run against a real cluster** (#240, #241, #242) · MaxCompute is not done, see the last section +- **Written**: 2026-09-03 (conventions in the [README](README.md)) +- **Related**: [0011](0011-a-mapping-is-not-a-fact.md) placed data sources at the deployment level and mounts at the base level; this record leaves that layer alone. [0016](0016-close-the-open-seams-before-cutting-new-ones.md) D4 put the MySQL wire protocol ahead of the lakehouse; the first section explains why the order flipped + +> The roadmap line reads "Iceberg / Delta Lake, Databricks, Snowflake and MaxCompute": four names, three kinds of thing. The first two are table formats, the next two are services, the last is a service on another cloud. Treating them as four engines would be writing code per product name. This record pins down what an engine is first, then decides which ones to build. + +## Engines follow protocols + +The header of `query_engine` (written for 0011) already gave the direction: the Postgres wire protocol, then the MySQL wire family, then the HTTP family. This step lands the HTTP family and skips MySQL. 0016 D4's "TiDB / OceanBase / Doris / StarRocks for free" is a fine list, but the lakehouse is what is wanted now, and on the protocol axis it is closer than it looks: + +| Wanted | What it is | Which protocol that is for us | +|---|---|---| +| Iceberg, Delta Lake, Hive, Hudi | table formats plus a catalog, with no query endpoint of their own | one **Trino** catalog each; `POST /v1/statement` with `nextUri` paging | +| Databricks | a Delta lakehouse behind a SQL warehouse | its own **SQL Statement Execution API** (`/api/2.0/sql/statements`) | +| Snowflake | a cloud warehouse that also reads Iceberg | its own **SQL API v2** (`/api/v2/statements`) | +| MaxCompute | Alibaba Cloud's warehouse | signed REST, asynchronous instances, results through Tunnel | + +The first three rows are all "JSON in, JSON out, Bearer or Basic auth". `reqwest` is already a dependency; each engine is about two hundred lines. The binary still carries no native database driver, which is the promise in the README's first sentence. It is also why the answer to Iceberg is Trino rather than an Iceberg reader: reading Iceberg directly pulls in Arrow, Parquet, object-storage SDKs and a query planner, and that is a different product. + +## The connection string is the only input + +The data-source page has a name and a connection string. Three new engines add no dropdown: **the scheme picks the engine** (`engine_from_conn`), and each engine parses the rest (`conn.rs`). The shape follows `postgres://user:pass@host/db`: credentials in the userinfo, the path is "catalog / database / schema", engine-specific switches go in the query string: + +``` +trino://alice[:password]@host[:8080]/catalog[/schema][?ssl=true] +databricks://:TOKEN@workspace-host/sql/1.0/warehouses/ID[?catalog=main&schema=default] +snowflake://:TOKEN@account.snowflakecomputing.com/DB[/SCHEMA][?warehouse=WH&role=R&token_type=pat|oauth] +``` + +The Databricks path is the httpPath shown in the console, so it can be pasted as is. All three tokens sit in the password position; `TOKEN@` with the colon missing is the most common slip, so the username position is accepted too. The shape is validated at registration, and the error carries the expected form. `ssl=false` exists for local proxies and stand-ins; the three services themselves only speak https. + +Left out on purpose: Snowflake key-pair JWT (local RSA signing, a dependency for a second login method, wait for someone to need it) and Trino Kerberos / OAuth2 (same reasoning). Passwords and tokens are the whole surface. + +## The HTTP family has three of the four gates + +0011 set up defense in depth: parse and admit only SELECT, wrap a LIMIT, a read-only session with a timeout, JSON Lines out. The HTTP family **has no session**, so the third gate is a timeout alone (Trino's `query_max_execution_time` session property, Databricks' `wait_timeout`, Snowflake's `timeout`), and read-only rests entirely on the first gate. The first gate therefore parses with each engine's dialect: `DatabricksDialect`, `SnowflakeDialect`, and `GenericDialect` for Trino (sqlparser has no Trino dialect; Generic is a superset). One test runs the same three checks under all four dialects: SELECT passes, DELETE fails, two statements fail. + +The fourth gate is assembled here for the HTTP family. Databricks' `JSON_ARRAY` and Snowflake's `data` return every value as a string; `coerce` restores numbers and booleans from the manifest / rowType column types, otherwise a model handed `"42"` stops doing arithmetic. Key order is assembled by hand instead of through `serde_json::Map`, which sorts keys unless `preserve_order` is on, and column order is the order the query wrote. + +## Loopback goes direct + +reqwest's system-proxy detection on Windows reads the registry and did not honor a `127.*` bypass, so a stand-in on the loopback address went through the proxy and came back as 502. The engine client now carries an explicit policy: loopback and `NO_PROXY` hosts go direct, everything else follows `HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY`. A server process reads its environment; that matches how docker-compose configures it. The other connectors keep reqwest's default. + +## Replays are the only tests so far + +No stand-in for any of the three runs on this machine: Docker Hub cannot be reached here (see memory), and Databricks and Snowflake are cloud-only anyway. The tests replay each vendor's documented protocol with wiremock: Trino's two `nextUri` pages, Databricks' PENDING → SUCCEEDED polling, Snowflake's 202 → 200, and each error body. **They prove our reading of the protocol, and only that.** Until one real cluster has answered, the README keeps these three marked as awaiting a real run, handled the way #214 / #215 handle GCS and Notion: an issue per engine labeled help wanted. + +## MaxCompute waits + +It is the one name of the four that is not "JSON in, JSON out": requests are signed with an AccessKey (HMAC-SHA1 over canonicalized headers), SQL runs as an asynchronous instance, and results come either through Tunnel (another protocol) or `GetInstanceResult` as CSV capped at ten thousand rows. Together that is a connector's worth of work, and this machine has no account that could sign a request, so the result could only be "probably like this". It stays on the roadmap until someone with an account arrives, or until its MySQL-compatible entry (MCQA) can ride the MySQL wire protocol of 0016 D4. + +## Open questions + +- **How much schema to fetch.** All three expose `information_schema.columns`, and a lakehouse catalog can hold thousands of tables; `sync_schema_doc` caps at 200. With a schema in the connection string only that schema is read, otherwise the whole catalog. Whether that is enough waits for a real cluster. +- **The type-restoration table** in `coerce` is hand-written from the three vendors' docs. Snowflake's `fixed` with a scale returns `"42.10"`, which becomes 42.1 and loses the trailing zero; harmless for a model, possibly not for an "exact definition". Revisit when the semantic layer keeps evidence (0016 D1) and decide whether to keep the raw string alongside. +- **Trino's `ssl` inference**: a password, `ssl=true`, or port 443 / 8443 means https, anything else is plaintext. That is trino-python's rule, and someone who gets it wrong sees a TLS error instead of a hint. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index a32bd2d47..d14c9f943 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -43,6 +43,7 @@ | 0015 | [记下一句话，不等于断言一个事实](0015-recording-a-sentence-is-not-asserting-a-fact.md) | 已实施 · 记忆抽出的事实进 `pending_facts`，Review 新档 + 跟在 remember 步骤后的确认卡 · `remember` 重新打开 · MCP 放开写是下一刀 | | 0016 | [先把开着的口子收上，再开新的](0016-close-the-open-seams-before-cutting-new-ones.md) | 规划中 · v0.1.0 之后的排期：A 收口 → B 推理机 ∥ C 尺子与本体 → D 语义层 → E 企业交付；模拟引擎后置 | | 0017 | [A contradiction points at an error upstream](0017-a-contradiction-points-upstream.md) | B2a 已实现（引擎与队列：逐条封顶、按规则对聚合、卡片给线索与修法）· B2b 待做：争议在图和面板上原地可见（新警戒色）| +| 0018 | [The lakehouse is one protocol away](0018-the-lakehouse-is-one-protocol-away.md) | 已实施 · 问数引擎扩到 HTTP 族：Trino（Iceberg / Delta / Hive）、Databricks、Snowflake，scheme 决定引擎，只有回放测试 · MaxCompute 未做 | ## 不是决策记录的那些 diff --git a/migrations/0021_lakehouse_engines.sql b/migrations/0021_lakehouse_engines.sql new file mode 100644 index 000000000..3781a6a69 --- /dev/null +++ b/migrations/0021_lakehouse_engines.sql @@ -0,0 +1,7 @@ +-- 问数引擎扩到 HTTP 协议族：trino（Iceberg / Delta / Hive 都是它的 catalog）、 +-- databricks（SQL Statement API）、snowflake（SQL API v2）。 +-- 挂载模型与注册表引擎无关（0006 的判断仍成立），这里只放宽 engine 的取值； +-- 允许的名字与 `query_engine::ENGINES` 同一张表。 +ALTER TABLE data_sources DROP CONSTRAINT data_sources_engine_check; +ALTER TABLE data_sources ADD CONSTRAINT data_sources_engine_check + CHECK (engine IN ('postgres', 'trino', 'databricks', 'snowflake')); diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 367e56994..1be5d4365 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -876,7 +876,13 @@ export const en = { "Read-only database connections for asking questions about your data in Chat. " + "Register connections here; each knowledge base mounts the ones it may query.", name: "Name", - connString: "Connection string (postgres://user:pass@host:5432/db)", + connString: "Connection string — the scheme picks the engine", + // 四种写法各一行；令牌放 password 位，Databricks 的路径就是控制台里的 httpPath + connSchemes: + "postgres://user:pass@host:5432/db\n" + + "trino://user[:pass]@host:8080/catalog[/schema] (Iceberg, Delta Lake, Hive)\n" + + "databricks://:TOKEN@host/sql/1.0/warehouses/ID?catalog=main\n" + + "snowflake://:TOKEN@account.snowflakecomputing.com/DB/SCHEMA?warehouse=WH", add: "Add data source", test: "Test", testOk: "Connected", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index f30986c00..7971a3d55 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -800,7 +800,12 @@ export const zh: Strings = { "只读的数据库连接，用于在「对话」里就你的数据提问。" + "在这里登记连接；每个知识库各自挂载允许查询的那些。", name: "名称", - connString: "连接串（postgres://user:pass@host:5432/db）", + connString: "连接串，前缀决定引擎", + connSchemes: + "postgres://user:pass@host:5432/db\n" + + "trino://user[:pass]@host:8080/catalog[/schema] （Iceberg、Delta Lake、Hive）\n" + + "databricks://:TOKEN@host/sql/1.0/warehouses/ID?catalog=main\n" + + "snowflake://:TOKEN@account.snowflakecomputing.com/DB/SCHEMA?warehouse=WH", add: "添加数据源", test: "测试", testOk: "已连接", diff --git a/web/src/pages/Settings.tsx b/web/src/pages/Settings.tsx index 4a4aa9c09..f61040aa9 100644 --- a/web/src/pages/Settings.tsx +++ b/web/src/pages/Settings.tsx @@ -639,7 +639,12 @@ function DataSourcesAdmin() {
-
{d.name}
+
+ {d.name} + + {d.engine} + +
{d.summary}
@@ -695,6 +700,9 @@ function DataSourcesAdmin() { value={conn} onChange={(e) => setConn(e.target.value)} /> +

+ {S.settings.datasources.connSchemes} +

+
+ + {S.graph.blockedBy(b.against_text)} + + + navigate({ + to: "/kb/$kbId/review", + params: { kbId }, + search: { queue: "violations", item: b.violation_id }, + }) + } + className="ml-auto shrink-0 text-neutral-500 hover:text-neutral-300 hover:underline underline-offset-2" + > + {S.graph.blockedReview} → + +
+ {open && ( +
+ {proof.isPending && ( +

{S.graph.proofLoading}

+ )} + {proof.data?.steps && ( + + )} +
+ )} +
+ ); +} + +/** 争议 chip（0017 §3）：有一条 open 的违规或冲突指着这条断言。行不压暗—— + * 它仍然活着。点它去 Review 对应那一档，并把那张卡点亮 */ +function ContestedChip({ + kbId, + c, +}: { + kbId: string; + c: NonNullable; +}) { + const navigate = useNavigate(); + const queue = c.kind === "temporal_conflict" ? "conflicts" : "violations"; + return ( + { + ev.stopPropagation(); + navigate({ + to: "/kb/$kbId/review", + params: { kbId }, + search: { queue, item: c.ref_id }, + }); + }} + onKeyDown={(ev) => { + if (ev.key === "Enter") { + ev.stopPropagation(); + navigate({ + to: "/kb/$kbId/review", + params: { kbId }, + search: { queue, item: c.ref_id }, + }); + } + }} + className="u-chip u-chip-contest shrink-0 !text-[10px] !px-1.5 cursor-pointer" + title={S.graph.contestedHint(c.kind, c.derived ?? null)} + > + {S.graph.contestedChip} + + ); +} + function EntityPanel({ kbId, entityId, exiting, + intent, onClose, onNavigate, }: { @@ -2593,6 +2783,8 @@ function EntityPanel({ entityId: string; /** 正在演退场：还挂在 DOM 上，但已经不接受点击 */ exiting: boolean; + /** 打开时停在哪一档、展开哪一行；读一次就清掉 */ + intent?: MutableRefObject<{ view: "derived"; open: string } | null>; onClose: () => void; onNavigate: (entityId: string) => void; }) { @@ -2604,6 +2796,8 @@ function EntityPanel({ // 推出来的那些。**单独一个键，不掺进 facts**——混在一个列表里，用户看不出 // 「文档里写的」和「引擎推的」的区别 const derived = detail.data?.derived ?? []; + // 没落地的（0017 §3）：推出来了，撞上一条断言 + const blocked = detail.data?.blocked ?? []; /* 按「方向 + 谓词 + 规则」分组，骨架与 Relations 的 groups 一致。 规则挂在组上而不是每一行：它对整组都成立，逐行重复既冗余， 那个琥珀色小字还会跟派生边抢色相 */ @@ -2635,6 +2829,13 @@ function EntityPanel({ const [view, setView] = useState< "relations" | "timeline" | "history" | "derived" >("relations"); + useEffect(() => { + const it = intent?.current; + if (!it) return; + intent.current = null; + setView(it.view); + setOpenFact(it.open); + }, [entityId, intent]); const e: GraphNode | undefined = detail.data?.entity; @@ -2919,8 +3120,10 @@ function EntityPanel({
{(["relations", "timeline", "history", "derived"] as const) // 推出来的那一档：**没有派生就不出现**。一个没开推理的库不该看到 - // 一个永远是空的标签页 - .filter((v) => v !== "derived" || derived.length > 0) + // 一个永远是空的标签页。没落地的也算——那正是这一档要说的事 + .filter( + (v) => v !== "derived" || derived.length > 0 || blocked.length > 0, + ) .map((v) => (
))} + {blocked.length > 0 && ( +
+
+ {S.graph.blockedTitle} + + {blocked.length} + +
+

+ {S.graph.blockedHint} +

+ {blocked.map((b) => ( + + setOpenFact( + openFact === b.violation_id ? null : b.violation_id, + ) + } + onNavigate={onNavigate} + /> + ))} +
+ )} )} {view !== "history" && @@ -3212,6 +3443,9 @@ function TimelineRow({ {S.graph.staleFactChip} )} + {fact.contested && ( + + )}
{open && } @@ -3296,6 +3530,7 @@ function FactRow({ {S.graph.staleFactChip} )} + {fact.contested && } {interval && ( {interval} diff --git a/web/src/pages/Review.tsx b/web/src/pages/Review.tsx index 6ca977c25..47e7d5651 100644 --- a/web/src/pages/Review.tsx +++ b/web/src/pages/Review.tsx @@ -1,6 +1,6 @@ import { useEffect, useState } from "react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { useNavigate } from "@tanstack/react-router"; +import { useNavigate, useSearch } from "@tanstack/react-router"; import { ArrowUpRight } from "lucide-react"; import { api, @@ -830,7 +830,11 @@ export function Review() { const { kb } = useKb(); const queryClient = useQueryClient(); const navigate = useNavigate(); - const [sel, setSel] = useState(null); + // 面板的争议 chip 带着 queue / item 跳过来：先落到那一档，再把那张卡点亮 + const search = useSearch({ from: "/app/kb/$kbId/review" }); + const [sel, setSel] = useState( + QUEUE_ORDER.includes(search.queue as Sel) ? (search.queue as Sel) : null, + ); const [page, setPage] = useState(0); // 队列变化经 SSE 事件流推送（useKbEvents 挂在 Shell），无需轮询。 @@ -1301,8 +1305,15 @@ export function Review() {
)} {asViolations().map((v) => ( - + select("duplicates")} onOntology={() => navigate({ to: "/ontology" })} /> +
))} )} diff --git a/web/src/router.tsx b/web/src/router.tsx index 6de3aceeb..d511346e6 100644 --- a/web/src/router.tsx +++ b/web/src/router.tsx @@ -143,6 +143,13 @@ const mappingsRoute = createRoute({ const reviewRoute = createRoute({ getParentRoute: () => kbRoute, path: "review", + // 从实体面板的争议 chip 跳过来：落到对应那一档，并点亮那一张卡（0017 §3） + validateSearch: ( + search: Record, + ): { queue?: string; item?: string } => ({ + queue: typeof search.queue === "string" ? search.queue : undefined, + item: typeof search.item === "string" ? search.item : undefined, + }), component: Review, }); diff --git a/web/src/styles.css b/web/src/styles.css index 85397ce83..e6b0719c3 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -413,6 +413,11 @@ select.input-dark option { background: rgba(242, 182, 109, 0.12); color: var(--u-warn); } +/* 争议（0017 §3）：与图上的争议边同一个色相 */ +.u-chip-contest { + background: rgba(255, 106, 61, 0.14); + color: var(--u-contest); +} .u-chip-danger { background: rgba(255, 157, 175, 0.12); color: var(--u-danger); From df4e7709eea48538602bd396dea72a709e7ed2f0 Mon Sep 17 00:00:00 2001 From: WaylandYang <145302500+WaylandYang@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:41:31 +0800 Subject: [PATCH 06/10] A declared disjointness keeps same names apart (B3) (#245) Co-authored-by: Claude Fable 5.1 --- crates/utopia-store/src/resolution.rs | 77 +++++-- ...declared_disjointness_keeps_names_apart.rs | 198 ++++++++++++++++++ docs/decisions/0009-no-type-is-a-type.md | 14 +- ...-the-open-seams-before-cutting-new-ones.md | 8 +- docs/decisions/README.md | 4 +- 5 files changed, 280 insertions(+), 21 deletions(-) create mode 100644 crates/utopia-store/tests/a_declared_disjointness_keeps_names_apart.rs diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs index f82377d9a..9d886a663 100644 --- a/crates/utopia-store/src/resolution.rs +++ b/crates/utopia-store/src/resolution.rs @@ -107,10 +107,12 @@ pub fn recall_keys(name: &str) -> Vec { /// 易混具体类型：抽取常在这几类间摇摆（一个团队算组织还是项目？平台算项目还是产品？）。 /// 同名跨这组类型 → 照建实体（宁分勿合），但入队审核对交 LLM/人工裁决。 /// -/// **这张表该从本体读，今天还没有。** `owl:disjointWith` 已经落库（`entity_type_disjoint`， -/// 有导入有编辑），但消费者只有推理机的本体自检；消解这边仍按这三个硬编码的 key 判。 -/// 没装包的库里这三个 key 不存在，于是这一档永不命中，所有跨类型同名判 `Disjoint`—— -/// 变严不变松，不会错合。改从本体读是 0016 的 B3。 +/// **本体说了算，这张表只是没声明时的退路。** 判两个类能不能指同一个东西，先看 +/// `owl:disjointWith`（`entity_type_disjoint`，含继承：Person ⟂ Organization 就让 +/// Corporation ⟂ Person）——声明了互斥的一律分开，哪怕它们在类层级上是一家； +/// 没声明的再看类层级（同一支系当易混，#226），最后才是这三个硬 key。 +/// 没装包也没声明的库里这三个 key 不存在，于是所有跨类型同名判 `Disjoint`—— +/// 变严不变松，不会错合（0016 B3） pub const CONFUSABLE_TYPE_KEYS: &[&str] = &["organization", "project", "product"]; /// 单次消解最多入队的漂移审核对（防同名大组刷爆审核队列）。 @@ -352,6 +354,9 @@ const CONTAIN_SCAN_LIMIT: i64 = 16; /// 所以这里靠 `kb_id` 收窄行集并设上限，且只在**新建实体时**跑一次， /// 不是每条 mention。大库上如果不够，正解是建一张「后缀键」表走等值查， /// 而不是加模糊索引。 +/// 包含扫描的一行：(id, 本名, 类型 key, 类型 id, 画像)。类型 id 用来比本体声明的互斥 +type ContainRow = (Uuid, String, Option, Option, Option); + async fn containment_reviews( pool: &PgPool, kb_id: Uuid, @@ -379,9 +384,11 @@ async fn containment_reviews( // 启明 X7 加速卡→concept vs 启明 X7 推理加速卡→product）， // 按类型相等去查，这两对一个都捞不到。相容性交给下面的 classify_type_drift。 // 多取一些行，因为硬互斥的会在 Rust 侧被筛掉 + // 本体声明了跟这个类互斥的那些类（含继承），一次取出，逐行比 id + let disjoint = declared_disjoint_from(pool, kb_id, type_id).await?; // 第三列可空：未分类实体也要参与包含关系扫描（0009） - let rows: Vec<(Uuid, String, Option, Option)> = sqlx::query_as( - "SELECT e.id, e.canonical_name, t.key, e.profile_embedding + let rows: Vec = sqlx::query_as( + "SELECT e.id, e.canonical_name, t.key, e.type_id, e.profile_embedding FROM entities e LEFT JOIN entity_types t ON t.id = e.type_id WHERE e.kb_id = $1 AND e.merged_into IS NULL AND e.id <> $2 @@ -419,12 +426,15 @@ async fn containment_reviews( Ok(rows .into_iter() // 哪些类型对可能指同一个东西，既有规则已经想清楚了，别另发明一套： - // person vs organization 永不合并，concept 兜底与谁都可能是一个 - .filter(|(_, _, type_key, _)| { - classify_type_drift(mention_key.as_deref(), type_key.as_deref()) != TypeDrift::Disjoint + // 本体声明互斥的永不合并，person vs organization 永不合并， + // concept 兜底与谁都可能是一个 + .filter(|(_, _, type_key, other_type, _)| { + !other_type.is_some_and(|t| disjoint.contains(&t)) + && classify_type_drift(mention_key.as_deref(), type_key.as_deref()) + != TypeDrift::Disjoint }) .take(MAX_CONTAIN_REVIEWS) - .map(|(id, other_name, _, emb)| { + .map(|(id, other_name, _, _, emb)| { // 分数只是给队列排序用的参考，**不参与是否合并的判断**—— // 那个判断本来就不在这条路上 let score = ctx @@ -466,6 +476,43 @@ fn drift_reason(mention_key: Option<&str>, other_key: Option<&str>, sim: Option< } } +/// 本体声明了跟这个类互斥的全部类（0016 B3）。 +/// +/// **互斥是继承的**：Person ⟂ Organization 一条声明，就让 Person 的每个子类跟 +/// Organization 的每个子类都互斥。所以先沿父链往上收集这个类的祖先，取它们声明的 +/// 互斥对象，再沿子链往下展开。表里两个方向各存一行，问一个方向就够。 +/// +/// 没判出类型（`None`）时没有类可问，回空集：那一侧本来就走召回候选那一档 +async fn declared_disjoint_from( + pool: &PgPool, + kb_id: Uuid, + type_id: Option, +) -> AppResult> { + let Some(type_id) = type_id else { + return Ok(HashSet::new()); + }; + let rows: Vec<(Uuid,)> = sqlx::query_as( + "WITH RECURSIVE up(id) AS ( + SELECT $2::uuid + UNION + SELECT p.parent_id FROM entity_type_parents p JOIN up ON p.child_id = up.id + ), hit(id) AS ( + SELECT d.b_id FROM entity_type_disjoint d JOIN up ON d.a_id = up.id + WHERE d.kb_id = $1 + ), down(id) AS ( + SELECT id FROM hit + UNION + SELECT p.child_id FROM entity_type_parents p JOIN down ON p.parent_id = down.id + ) + SELECT id FROM down", + ) + .bind(kb_id) + .bind(type_id) + .fetch_all(pool) + .await?; + Ok(rows.into_iter().map(|(id,)| id).collect()) +} + /// 两个类是不是一家的：一方是另一方的祖先，或者两者共有一个**不是根**的祖先。 /// /// `CONFUSABLE_TYPE_KEYS` 那张三 key 的硬表是给没装包的库准备的；装了 schema.org @@ -561,12 +608,18 @@ async fn resolve_type_drift( .fetch_all(pool) .await?; + // 本体声明了互斥的类，一次取出（含继承）。声明优先于下面所有启发式 + let disjoint = declared_disjoint_from(pool, kb_id, type_id).await?; let mut recall_cands: Vec<&CrossCandidate> = Vec::new(); let mut review_cands: Vec<&CrossCandidate> = Vec::new(); for c in &cross { let mut drift = classify_type_drift(mention_key.as_deref(), c.type_key.as_deref()); - // 硬表判不上的，再看类层级：同一支系下的同名当易混，进审阅队列 - if drift == TypeDrift::Disjoint { + if c.type_id.is_some_and(|t| disjoint.contains(&t)) { + // 本体说这两类互斥：哪怕硬表说易混、类层级说一家，也分开。 + // 声明是人写下的判断，启发式只是没声明时的猜测 + drift = TypeDrift::Disjoint; + } else if drift == TypeDrift::Disjoint { + // 硬表判不上的，再看类层级：同一支系下的同名当易混，进审阅队列 if let (Some(a), Some(b)) = (type_id, c.type_id) { if types_are_kin(pool, a, b).await? { drift = TypeDrift::Review; diff --git a/crates/utopia-store/tests/a_declared_disjointness_keeps_names_apart.rs b/crates/utopia-store/tests/a_declared_disjointness_keeps_names_apart.rs new file mode 100644 index 000000000..112f8323e --- /dev/null +++ b/crates/utopia-store/tests/a_declared_disjointness_keeps_names_apart.rs @@ -0,0 +1,198 @@ +//! 0016 B3：`owl:disjointWith` 进消解——本体声明了互斥的两个类，同名也不进审阅队列。 +//! +//! 消解判「两个类能不能指同一个东西」有三层：硬表 `CONFUSABLE_TYPE_KEYS`、类层级 +//! （同一支系当易混，#226）、本体声明的互斥。这里守的是**声明优先于前两层**： +//! +//! 1. 没声明时行为不变：organization vs project 照硬表进队列，corporation vs +//! federal_agency 照类层级（共有非根祖先 organization）进队列。 +//! 2. 声明 organization ⟂ project 之后，同名的 organization / project 分开，不进队列。 +//! 3. 声明 corporation ⟂ agency 之后，federal_agency（agency 的子类）跟 corporation +//! 也分开——**互斥是继承的**，声明在父类上就够。 +//! +//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆，绝不碰已有的库。 + +use sqlx::PgPool; +use utopia_store::{ontology, resolution}; +use uuid::Uuid; + +struct Fx { + org: Uuid, + kb: Uuid, + organization: Uuid, + project: Uuid, + corporation: Uuid, + agency: Uuid, + federal_agency: Uuid, +} + +async fn seed(pool: &PgPool) -> anyhow::Result { + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + // 顶上要有一个根：类层级那条规则里「共有的祖先」不算根（schema.org 里万物皆 + // Thing，算上它 Person 与 Organization 也成了一家），所以 organization 得有父类 + let (thing, organization, project, corporation, agency, federal_agency) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'disjoint-test')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'disjoint-test')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'disjoint-test')", + ) + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + for (id, key) in [ + (thing, "thing"), + (organization, "organization"), + (project, "project"), + (corporation, "corporation"), + (agency, "agency"), + (federal_agency, "federal_agency"), + ] { + sqlx::query("INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, $3, $3)") + .bind(id) + .bind(kb) + .bind(key) + .execute(pool) + .await?; + } + for (child, parent) in [ + (organization, thing), + (project, thing), + (corporation, organization), + (agency, organization), + (federal_agency, agency), + ] { + sqlx::query("INSERT INTO entity_type_parents (child_id, parent_id) VALUES ($1, $2)") + .bind(child) + .bind(parent) + .execute(pool) + .await?; + } + Ok(Fx { + org, + kb, + organization, + project, + corporation, + agency, + federal_agency, + }) +} + +async fn entity(pool: &PgPool, f: &Fx, name: &str, type_id: Uuid) -> anyhow::Result { + let id = Uuid::now_v7(); + sqlx::query( + "INSERT INTO entities (id, kb_id, type_id, canonical_name) VALUES ($1, $2, $3, $4)", + ) + .bind(id) + .bind(f.kb) + .bind(type_id) + .bind(name) + .execute(pool) + .await?; + Ok(id) +} + +/// 消解一条 mention，回它挂上的「类型漂移」审核对指向谁 +async fn drift_reviews( + pool: &PgPool, + f: &Fx, + name: &str, + type_id: Uuid, +) -> anyhow::Result> { + let r = resolution::resolve_mention(pool, f.kb, Some(type_id), name, None).await?; + assert!( + r.created, + "a cross-type same name is a new entity: keep apart, never merge" + ); + Ok(r.reviews + .iter() + .filter(|x| x.reason.starts_with("type_drift|")) + .map(|x| x.other_id) + .collect()) +} + +#[tokio::test] +async fn a_declared_disjointness_keeps_names_apart() -> anyhow::Result<()> { + let Ok(url) = std::env::var("UTOPIA_DATABASE_URL") else { + eprintln!("跳过：未设 UTOPIA_DATABASE_URL"); + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let f = seed(&pool).await?; + + let run = async { + // 1. 没声明：硬表与类层级照旧 + let orion = entity(&pool, &f, "Orion", f.organization).await?; + assert_eq!( + drift_reviews(&pool, &f, "Orion", f.project).await?, + vec![orion], + "organization vs project is confusable by the hard-coded list" + ); + let acme = entity(&pool, &f, "Acme", f.corporation).await?; + assert_eq!( + drift_reviews(&pool, &f, "Acme", f.federal_agency).await?, + vec![acme], + "corporation vs federal_agency share the ancestor organization: kin, so Review" + ); + + // 2. 声明 organization ⟂ project：硬表说易混，本体说互斥——本体赢 + ontology::set_disjoint_for(&pool, f.kb, f.organization, &[f.project]).await?; + let _vega = entity(&pool, &f, "Vega", f.organization).await?; + assert!( + drift_reviews(&pool, &f, "Vega", f.project) + .await? + .is_empty(), + "a declared disjointness wins over the hard-coded list" + ); + + // 3. 声明 corporation ⟂ agency：federal_agency 是 agency 的子类，互斥继承下来， + // 类层级说一家也不算 + ontology::set_disjoint_for(&pool, f.kb, f.corporation, &[f.agency]).await?; + let _beta = entity(&pool, &f, "Beta", f.corporation).await?; + assert!( + drift_reviews(&pool, &f, "Beta", f.federal_agency) + .await? + .is_empty(), + "a disjointness declared on the parent reaches the child and wins over kinship" + ); + // 反过来问也一样：表里两个方向各一行，继承沿另一头的祖先链走 + let _gamma = entity(&pool, &f, "Gamma", f.federal_agency).await?; + assert!( + drift_reviews(&pool, &f, "Gamma", f.corporation) + .await? + .is_empty(), + "the declaration holds from either side" + ); + + // 4. 取消声明，回到没声明时的行为——编辑必须能撤 + ontology::set_disjoint_for(&pool, f.kb, f.corporation, &[]).await?; + let delta = entity(&pool, &f, "Delta", f.corporation).await?; + assert_eq!( + drift_reviews(&pool, &f, "Delta", f.federal_agency).await?, + vec![delta], + "with the declaration gone, kinship sends the pair to Review again" + ); + Ok::<_, anyhow::Error>(()) + } + .await; + + let _ = sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(f.org) + .execute(&pool) + .await; + run +} diff --git a/docs/decisions/0009-no-type-is-a-type.md b/docs/decisions/0009-no-type-is-a-type.md index 9498662c7..38a928d4d 100644 --- a/docs/decisions/0009-no-type-is-a-type.md +++ b/docs/decisions/0009-no-type-is-a-type.md @@ -2,9 +2,11 @@ - **Status**: Implemented · `entities.type_id` and `entity_retypes.from_type_id` are nullable; the nine builtin classes and the seeding function left with the seed relations - (#110 / #125 / #128 / [0011](0011-a-mapping-is-not-a-fact.md)) · `owl:disjointWith` is - stored but resolution still reads the hard-coded `CONFUSABLE_TYPE_KEYS`; since #226 same-name - entities of kin classes (ancestor, descendant or a shared non-root ancestor) go to Review · `metric` / + (#110 / #125 / #128 / [0011](0011-a-mapping-is-not-a-fact.md)) · `owl:disjointWith` now + reaches resolution (0016 B3): a declared disjointness, inherited down both hierarchies, keeps + same-name entities apart ahead of every heuristic; since #226 same-name entities of kin classes + (ancestor, descendant or a shared non-root ancestor) go to Review; `CONFUSABLE_TYPE_KEYS` stays + as the fallback when nothing is declared · `metric` / `dimension` are created on demand by mapping exploration (#231); a semantic-layer pack is still planned - **Written**: 2026-08-30 · condensed into English 2026-09-03 @@ -82,6 +84,12 @@ in the ontology as a class, as if someone had decided it. - 2026-09-02: `owl:disjointWith` now has a table (`entity_type_disjoint`), import and an edit endpoint, but its only consumer is R0's ontology self-check; `CONFUSABLE_TYPE_KEYS` is still three hard-coded keys (0016 B3). +- 2026-09-03: resolution reads the table (0016 B3). `declared_disjoint_from` collects the + classes declared disjoint with the mention's class or any ancestor, expanded to their + descendants, in one recursive query; both cross-type paths (type drift and containment) + treat a hit as `Disjoint` before the kinship check and the hard-coded list. Nothing + declared → today's behavior, unchanged. The list is now a fallback, and decision 6 reads + accordingly. - 2026-09-02: `metric` / `dimension` had a visible cost — mapping exploration looked up `entity_types.key IN ('metric','dimension')` and `continue`d when missing, so a KB without those classes silently produced zero mappings. 2026-09-03: exploration now creates both as diff --git a/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md b/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md index b35d5fd51..40a9d6c94 100644 --- a/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md +++ b/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md @@ -1,6 +1,6 @@ # 0016 · Close the open seams before cutting new ones -- **Status**: in progress · the first schedule after v0.1.0 · A1, A2, A3 done · B1 done (#227) · B2 done (#238 / #243, [0017](0017-a-contradiction-points-upstream.md)) · B3: the signature half done (#190 / #196), cross-pack signatures and range-aware direction done (#233), `disjointWith` into resolution still open · D2's blank-base problem worked around with builtin `metric` / `dimension` classes (#231), the pack itself still planned · the lakehouse engines landed ahead of D4 (#239, [0018](0018-the-lakehouse-is-one-protocol-away.md)) +- **Status**: in progress · the first schedule after v0.1.0 · A1, A2, A3 done · B1 done (#227) · B2 done (#238 / #243, [0017](0017-a-contradiction-points-upstream.md)) · B3 done: the signature half (#190 / #196), cross-pack signatures and range-aware direction (#233), `disjointWith` into resolution · D2's blank-base problem worked around with builtin `metric` / `dimension` classes (#231), the pack itself still planned · the lakehouse engines landed ahead of D4 (#239, [0018](0018-the-lakehouse-is-one-protocol-away.md)) - **Written**: 2026-09-02 · condensed into English 2026-09-03 - **Related**: written after checking [0001](0001-ontology-import-and-governance.md) through [0015](0015-recording-a-sentence-is-not-asserting-a-fact.md) against the code; every item below has its source in those fifteen records, whose 2026-09-02 revision notes are the check's product. This record only orders them and says why this order. @@ -45,9 +45,9 @@ A4 README against code, both languages: promise only what has landed. **B · Finish the reasoning engine**, after A, parallel with C. B1 the R2 proof chain (done, #227; a chain, not a tree — see [0002](0002-reasoning-engine.md)). B2 the derived-vs-asserted contradiction signal, `axiom_violations` kind `derived_contradiction` (done: B2a engine and -queue #238, B2b visibility on graph and panel #243 — 0017). B3 `classify_type_drift` reads `entity_type_disjoint` -instead of the hard-coded list, today's behavior when nothing is declared (class kinship from the -hierarchy landed first, #226). B4 R3 incremental +queue #238, B2b visibility on graph and panel #243 — 0017). B3 resolution reads `entity_type_disjoint` +ahead of the hard-coded list (done: a declared disjointness, inherited, wins over kinship and the +list; nothing declared → today's behavior; class kinship from the hierarchy landed first, #226). B4 R3 incremental maintenance, deferred until a full re-derivation of `ai-timeline-ends` exceeds a threshold (proposal: 10 seconds). diff --git a/docs/decisions/README.md b/docs/decisions/README.md index b2e8b9811..a4c39010d 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -32,14 +32,14 @@ The test for writing one: if someone (including us) looks at a piece of code in | 0006 | [Ontology scale and the extraction prompt](0006-ontology-scale-and-the-prompt.md) | Built · the character budget (24,000) and per-chunk retrieval live, values untested · answer keys still hand-filled | | 0007 | [Counting decides what becomes a relation](0007-who-decides-what-becomes-a-relation.md) | Built · adoption decided by counting (`MIN_DOCS = 2`, `MIN_SIGNALS = 3`), proposals persist (#112) · narrative verbs and `_by` folding still open | | 0008 | [Ontology packs as the cold start](0008-ontology-packs-as-cold-start.md) | Built · five packs embedded, multi-select at creation, schema.org by default · three open questions stay open; Chinese labels got worse | -| 0009 | [An undecided type stays empty](0009-no-type-is-a-type.md) | Implemented · `type_id` nullable, builtin classes gone · kin classes go to Review (#226), `disjointWith` still unread by resolution · `metric` / `dimension` builtin on demand (#231) | +| 0009 | [An undecided type stays empty](0009-no-type-is-a-type.md) | Implemented · `type_id` nullable, builtin classes gone · kin classes go to Review (#226), declared `disjointWith` keeps them apart (0016 B3) · `metric` / `dimension` builtin on demand (#231) | | 0010 | [An unnamed relation stays empty](0010-no-relation-is-no-relation.md) | Implemented · `predicate_id` nullable, `related_to` gone, wording recovered by `fact_surface_predicate` · follow-ups done with 0011 | | 0011 | [A mapping is configuration](0011-a-mapping-is-not-a-fact.md) | Implemented (#126 / #140 / #148) · Review flow and revision history rebuilt · the evidence chain not built | | 0012 | [The ontology is a contract](0012-the-ontology-is-a-contract-not-a-suggestion.md) | Implemented · violation rate 57% → 4%, reversals 39 → 0 · guard extended to adoption and merge (#190 / #196) · reified-shell filter at pack import open | | 0013 | [A source hands over its history](0013-a-source-should-hand-over-its-history.md) | Implemented for GitHub, Jira and Notion (#134 / #135 / #213) · Feishu and Confluence not started · `instant` precision not triggered | | 0014 | [Identity from the person, scope from the token](0014-identity-from-the-person-scope-from-the-token.md) | Implemented (#180) · five read-only MCP tools over Streamable HTTP · tokens page at `/account/tokens` · `can_write` still hard-coded false | | 0015 | [A recorded sentence waits for a nod](0015-recording-a-sentence-is-not-asserting-a-fact.md) | Implemented · memory facts wait in `pending_facts`, nod queue and chat card, `remember` reopened · MCP write is the next cut | -| 0016 | [Close the open seams before cutting new ones](0016-close-the-open-seams-before-cutting-new-ones.md) | In progress · A done · B1 and B2 done, `disjointWith` open · C untouched · D2 worked around (#231); the lakehouse landed ahead of D4 (#239) | +| 0016 | [Close the open seams before cutting new ones](0016-close-the-open-seams-before-cutting-new-ones.md) | In progress · A done · B done (B4 deferred) · C untouched · D2 worked around (#231); the lakehouse landed ahead of D4 (#239) | | 0017 | [A contradiction points at an error upstream](0017-a-contradiction-points-upstream.md) | Implemented · B2a: engine and queue, per-item cap, aggregation by rule pair, cards with clues and repairs (#238) · B2b: contested edges in the alert colour, ghost edges for blocked derivations, the disputed chip and the "did not land" section in the panel (#243) | | 0018 | [The lakehouse is one protocol away](0018-the-lakehouse-is-one-protocol-away.md) | Implemented: Trino (Iceberg / Delta / Hive), Databricks and Snowflake behind the same trait, scheme picks the engine (#239) · replay tests only, real clusters wanted (#240–#242) · MaxCompute waits | From 2120f4147d9fc4aac73a09c6c201d7cb11812e44 Mon Sep 17 00:00:00 2001 From: WaylandYang <145302500+WaylandYang@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:38:50 +0800 Subject: [PATCH 07/10] Source credentials stay on the server (#249) * Source credentials stay on the server Co-Authored-By: Claude Fable 5.1 * A security contact for the repository Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- SECURITY.md | 48 +----- crates/utopia-core/src/models.rs | 28 ++++ .../utopia-server/src/api/sources_routes.rs | 141 ++++++++++++++---- crates/utopia-store/src/sources.rs | 9 +- .../tests/a_viewer_never_sees_a_credential.rs | 139 +++++++++++++++++ ...3-a-source-should-hand-over-its-history.md | 4 + 6 files changed, 293 insertions(+), 76 deletions(-) create mode 100644 crates/utopia-store/tests/a_viewer_never_sees_a_credential.rs diff --git a/SECURITY.md b/SECURITY.md index 30815ea4f..645174465 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,49 +1,5 @@ # Security -*[中文版](SECURITY.zh-CN.md)* +If you find a vulnerability in Utopia, please email **security@deeplethe.com** rather than opening a public issue. Include the affected version or commit, the endpoint or component, and steps to reproduce. -Utopia is at v0.1. Below are the **known, unresolved** limits — not a vulnerability report, -but the places the design has not reached yet. - -## Before you put this on a public network - -**Credentials are stored in the clear.** LLM API keys and Ask-the-Data connection strings -are plain text in Postgres (`llm_settings.chat_api_key`, `data_sources.conn_string`). Anyone -who can read the database can read them. Encryption at rest is a 1.0 item; until then, keep -the system and its database inside a trusted network. - -**The default database password is `utopia`.** By default the port binds to loopback -(`127.0.0.1:1517`), so nothing outside the host can connect. If you change `UTOPIA_DB_BIND` -to expose it, change `UTOPIA_DB_PASSWORD` in `.env` first. - -**A data source is only as safe as its grants.** Registering one is a deployment-level -action, but the connection string reaches every workspace the source is granted to. Grant it -only where that database should be visible, and use a read-only database role in the string -itself — the SQL gate below is defence in depth, not a substitute for least privilege at the -source. - -## What is in place - -- **JWT signing key generated on first start** — 32 bytes from a CSPRNG, stored in the - database. No deployment shares a default key. -- **`Secure` on session cookies behind TLS** — decided from `X-Forwarded-Proto`, so local - HTTP development still works. Force it with `UTOPIA_COOKIE_SECURE=true` if your proxy - omits the header. -- **Database port bound to loopback** — `127.0.0.1:1517`; the app reaches the database over - the compose network. -- **Optional least-privilege runtime role** — set `UTOPIA_APP_DB_PASSWORD` and - `UTOPIA_MIGRATION_URL`, and the app connects as a role that can only read and write - business tables and append to the ledger, while migrations run as the owner. -- **Data sources reach only granted workspaces** — a registered database is mounted into a - knowledge base only where an explicit grant exists. Before this, any base admin could - mount any registered source, which crossed tenants. -- **Read-only gate on Ask-the-Data** — parser allowlist, read-only transaction, enforced row - limit; three layers, so a statement past the parser still cannot write. -- **Accounts are deactivated, not deleted** — `users.deactivated_at` blocks sign-in while the - ledger keeps that person's decisions attributable. -- **Passwords hashed with argon2.** - -## Reporting a vulnerability - -Open an issue. If it involves exploitable detail, start with the minimum needed to reproduce -and we will follow up privately. +You will get an acknowledgement within a few days. Once a fix is released, the advisory names the reporter unless you ask otherwise. diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index 091a5cf84..4085d6ae7 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -137,6 +137,34 @@ pub struct Source { pub created_at: DateTime, } +/// 来源配置里**用来鉴权**的那几个键。凭据只进不出：列表与创建 / 更新的响应都剔掉， +/// 更新时客户端没传或传空串就保留库里的原值，审计里也不落。 +/// +/// **一张表，四处共用。** 此前那条规矩只对 `auth_header` 一个键成立，而对象存储、 +/// WebDAV、Notion 各自的密钥原样发给了每一个 Viewer（#246）。加连接器时**先加这里**， +/// 再写读它的代码。`username` / `account_name` / `access_key_id` 这类是身份标识， +/// 单独拿到鉴不了权，留着让界面显示得出「这是哪个账号」。 +pub const SOURCE_SECRET_KEYS: &[&str] = &[ + "auth_header", + "token", + "password", + "secret_access_key", + "account_key", + "service_account_key", +]; + +impl Source { + /// 剔掉凭据后的这条来源——任何要回给客户端的 `Source` 都从这里过 + pub fn without_secrets(mut self) -> Self { + if let Some(obj) = self.config.as_object_mut() { + for key in SOURCE_SECRET_KEYS { + obj.remove(*key); + } + } + self + } +} + /// 来源同步运行记录（渠道审计历史）。 #[derive(Debug, Clone, Serialize, sqlx::FromRow)] pub struct SyncRun { diff --git a/crates/utopia-server/src/api/sources_routes.rs b/crates/utopia-server/src/api/sources_routes.rs index baaae99fd..3889a69e2 100644 --- a/crates/utopia-server/src/api/sources_routes.rs +++ b/crates/utopia-server/src/api/sources_routes.rs @@ -6,7 +6,7 @@ use axum::Json; use chrono::{DateTime, Utc}; use serde::Deserialize; use serde_json::json; -use utopia_core::models::Role; +use utopia_core::models::{Role, SOURCE_SECRET_KEYS}; use uuid::Uuid; use super::graph_routes::require_kb; @@ -19,6 +19,23 @@ fn new_ingest_token() -> String { format!("utp_{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()) } +/// 取一条来源，并确认它属于路径上的这个库。 +/// +/// `require_kb` 只查人对库的权限；来源 id 是另一个维度——不比对的话，A 库的 +/// Editor 拿着 B 库来源的 id 就能同步、清理、删除它。不属于就当不存在（404）， +/// 与 `get_token` 一直以来的做法一致 +async fn source_in_kb( + state: &AppState, + kb_id: Uuid, + source_id: Uuid, +) -> ApiResult { + let source = utopia_store::sources::get(&state.pool, source_id).await?; + if source.kb_id != kb_id { + return Err(utopia_core::AppError::NotFound.into()); + } + Ok(source) +} + pub async fn list( State(state): State, AuthUser(user): AuthUser, @@ -102,8 +119,8 @@ pub async fn get_token( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; - let source = utopia_store::sources::get(&state.pool, source_id).await?; - if source.kb_id != kb_id || source.kind != "api" { + let source = source_in_kb(&state, kb_id, source_id).await?; + if source.kind != "api" { return Err(utopia_core::AppError::NotFound.into()); } Ok(Json(json!({ "ingest_token": source.ingest_token }))) @@ -116,8 +133,8 @@ pub async fn rotate_token( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; - let source = utopia_store::sources::get(&state.pool, source_id).await?; - if source.kb_id != kb_id || source.kind != "api" { + let source = source_in_kb(&state, kb_id, source_id).await?; + if source.kind != "api" { return Err(utopia_core::AppError::NotFound.into()); } let token = new_ingest_token(); @@ -125,12 +142,34 @@ pub async fn rotate_token( Ok(Json(json!({ "ingest_token": token }))) } -/// 响应前剔除凭据（auth_header 只进不出）。 -fn mask_secrets(mut source: utopia_core::models::Source) -> utopia_core::models::Source { - if let Some(obj) = source.config.as_object_mut() { - obj.remove("auth_header"); +/// 响应前剔除凭据（只进不出；键见 `SOURCE_SECRET_KEYS`）。 +fn mask_secrets(source: utopia_core::models::Source) -> utopia_core::models::Source { + source.without_secrets() +} + +/// 更新时凭据的合并规则，每个 `SOURCE_SECRET_KEYS` 里的键一样：新配置里**没有**这个键 +/// 或值是空串 → 保留库里的原值（表单留空就是「别动」）；显式 `null` → 删掉； +/// 其余照新值。响应从不回显，所以客户端没有办法把旧值原样送回来，规则只能长在这里 +fn keep_secrets(next: &mut serde_json::Value, existing: &serde_json::Value) { + let Some(obj) = next.as_object_mut() else { + return; + }; + for key in SOURCE_SECRET_KEYS { + let keep = match obj.get(*key) { + None => true, + Some(serde_json::Value::Null) => { + obj.remove(*key); + false + } + Some(v) => v.as_str().is_some_and(|s| s.trim().is_empty()), + }; + if keep { + obj.remove(*key); + if let Some(prev) = existing.get(*key) { + obj.insert((*key).to_string(), prev.clone()); + } + } } - source } #[derive(Deserialize)] @@ -158,23 +197,11 @@ pub async fn update( Json(body): Json, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; - // 凭据只进不出：响应从不回显 auth_header，表单留空 = 保留库里原值 + let existing = source_in_kb(&state, kb_id, source_id).await?; + // 凭据只进不出：响应从不回显，表单留空 / 没传 = 保留库里原值 let mut config = body.config; if let Some(cfg) = config.as_mut() { - let blank = cfg - .get("auth_header") - .and_then(|v| v.as_str()) - .map(str::trim) - .is_none_or(|s| s.is_empty()); - if blank { - if let Some(obj) = cfg.as_object_mut() { - obj.remove("auth_header"); - let existing = utopia_store::sources::get(&state.pool, source_id).await?; - if let Some(prev) = existing.config.get("auth_header").and_then(|v| v.as_str()) { - obj.insert("auth_header".into(), json!(prev)); - } - } - } + keep_secrets(cfg, &existing.config); } let source = utopia_store::sources::update( &state.pool, @@ -187,7 +214,7 @@ pub async fn update( ) .await?; state.emit_source(kb_id); - // 审计不落凭据：config 只记除 auth_header 外的键 + // 审计不落凭据：config 只记「改没改」 let _ = utopia_store::audit::record( &state.pool, Some(kb_id), @@ -208,6 +235,7 @@ pub async fn cleanup_missing( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; + source_in_kb(&state, kb_id, source_id).await?; let ids = utopia_store::documents::list_missing(&state.pool, source_id).await?; for id in &ids { utopia_store::documents::delete(&state.pool, *id).await?; @@ -229,7 +257,7 @@ pub async fn delete( ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; // Memory 来源常驻：记忆空间不因来源整理而蒸发（记忆文档本身可在 Library 删除） - let source = utopia_store::sources::get(&state.pool, source_id).await?; + let source = source_in_kb(&state, kb_id, source_id).await?; if source.kind == utopia_store::memory::MEMORY_SOURCE_KIND { return Err(utopia_core::AppError::invalid( "memory_source_permanent", @@ -259,6 +287,7 @@ pub async fn runs( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Viewer).await?; + source_in_kb(&state, kb_id, source_id).await?; let runs = utopia_store::sources::list_runs(&state.pool, source_id, 20).await?; Ok(Json(json!({ "runs": runs }))) } @@ -269,6 +298,7 @@ pub async fn sync_now( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; + source_in_kb(&state, kb_id, source_id).await?; let queued = utopia_store::sources::mark_queued(&state.pool, source_id).await?; if queued { utopia_store::jobs::enqueue( @@ -505,3 +535,60 @@ pub async fn re_extract( .await; Ok(Json(json!({ "queued": ids.len() }))) } + +#[cfg(test)] +mod tests { + use super::keep_secrets; + use serde_json::json; + + #[test] + fn a_blank_or_missing_secret_keeps_the_stored_one() { + let existing = json!({ "bucket": "old", "secret_access_key": "s", "password": "p" }); + // 没传 → 留；空串 → 留；有值 → 换；null → 删 + let mut next = json!({ "bucket": "new", "password": " ", "token": null }); + keep_secrets(&mut next, &existing); + assert_eq!(next["bucket"], "new"); + assert_eq!( + next["secret_access_key"], "s", + "missing keeps the stored value" + ); + assert_eq!(next["password"], "p", "blank keeps the stored value"); + assert!(next.get("token").is_none(), "an explicit null removes it"); + let mut next = json!({ "secret_access_key": "fresh" }); + keep_secrets(&mut next, &existing); + assert_eq!(next["secret_access_key"], "fresh"); + assert_eq!(next["password"], "p"); + } + + #[test] + fn no_secret_reaches_a_response() { + let source = utopia_core::models::Source { + id: uuid::Uuid::nil(), + kb_id: uuid::Uuid::nil(), + kind: "s3".into(), + name: "s".into(), + config: json!({ "bucket": "b", "access_key_id": "AKIA", "secret_access_key": "x", + "account_key": "y", "service_account_key": "z", "password": "w", + "token": "t", "auth_header": "h" }), + icon: None, + sync_interval_minutes: None, + sync_cron: None, + last_sync_at: None, + last_sync_status: "never".into(), + last_sync_error: None, + last_sync_added: 0, + ingest_token: Some("utp_x".into()), + created_at: chrono::Utc::now(), + }; + let masked = super::mask_secrets(source); + let obj = masked.config.as_object().unwrap(); + for key in utopia_core::models::SOURCE_SECRET_KEYS { + assert!(!obj.contains_key(*key), "{key} leaked"); + } + assert_eq!(obj["bucket"], "b"); + assert_eq!( + obj["access_key_id"], "AKIA", + "an identifier is not a secret" + ); + } +} diff --git a/crates/utopia-store/src/sources.rs b/crates/utopia-store/src/sources.rs index f0ca22f79..e4d6f3819 100644 --- a/crates/utopia-store/src/sources.rs +++ b/crates/utopia-store/src/sources.rs @@ -2,7 +2,7 @@ use chrono::{DateTime, Utc}; use sqlx::PgPool; -use utopia_core::models::{Role, Source, SourceView, SyncRun}; +use utopia_core::models::{Role, Source, SourceView, SyncRun, SOURCE_SECRET_KEYS}; use utopia_core::{AppError, AppResult}; use uuid::Uuid; @@ -55,9 +55,11 @@ fn cron_next_after(expr: &str, after: DateTime) -> Option> { } pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult> { - // config 剔除 auth_header：自定义拉取器的凭据不下发给任何客户端 + // config 剔掉凭据：列表给 Viewer 看，哪一种连接器的密钥都不下发。 + // 键在 `SOURCE_SECRET_KEYS` 一张表上——从前这里只减 `auth_header`，五种连接器 + // 的密钥就这么漏出去的（#246） let rows: Vec = sqlx::query_as( - "SELECT s.id, s.kind, s.name, s.config - 'auth_header' AS config, s.icon, + "SELECT s.id, s.kind, s.name, s.config - $2::text[] AS config, s.icon, s.sync_interval_minutes, s.sync_cron, s.last_sync_at, s.last_sync_status, s.last_sync_error, s.last_sync_added, (SELECT count(*) FROM documents d WHERE d.source_id = s.id) AS doc_count, @@ -66,6 +68,7 @@ pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult> { FROM sources s WHERE s.kb_id = $1 ORDER BY s.created_at", ) .bind(kb_id) + .bind(SOURCE_SECRET_KEYS) .fetch_all(pool) .await?; Ok(rows) diff --git a/crates/utopia-store/tests/a_viewer_never_sees_a_credential.rs b/crates/utopia-store/tests/a_viewer_never_sees_a_credential.rs new file mode 100644 index 000000000..5efcab447 --- /dev/null +++ b/crates/utopia-store/tests/a_viewer_never_sees_a_credential.rs @@ -0,0 +1,139 @@ +//! #246：来源列表不带任何凭据。 +//! +//! 列表接口给 Viewer 看，此前只剔了 `auth_header`；对象存储、WebDAV、Notion 各自的 +//! 密钥原样下发。现在凭据键列在 `SOURCE_SECRET_KEYS` 一张表上，列表 SQL 按表剔。 +//! 这里守两件事： +//! +//! 1. **列表里一个凭据键都没有**，每一种连接器都试一遍。 +//! 2. **身份标识留着**（bucket、username、account_name），界面要显示得出「这是哪个账号」； +//! 而同步那条路（`sources::get`）拿到的仍是完整配置——凭据只是不出去，不是没了。 +//! +//! 直接插表而不走 `sources::create`：`KINDS` 少了五种（#247），那是另一个修复。 +//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆，绝不碰已有的库。 + +use sqlx::PgPool; +use utopia_core::models::SOURCE_SECRET_KEYS; +use utopia_store::sources; +use uuid::Uuid; + +async fn seed(pool: &PgPool) -> anyhow::Result<(Uuid, Uuid)> { + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'secret-test')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'secret-test')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'secret-test')", + ) + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + Ok((org, kb)) +} + +#[tokio::test] +async fn a_viewer_never_sees_a_credential() -> anyhow::Result<()> { + let Ok(url) = std::env::var("UTOPIA_DATABASE_URL") else { + eprintln!("跳过：未设 UTOPIA_DATABASE_URL"); + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let (org, kb) = seed(&pool).await?; + + let run = async { + // 每种连接器一条，配置按各自界面真正会写的键 + let fixtures: Vec<(&str, serde_json::Value)> = vec![ + ( + "custom", + serde_json::json!({ "endpoint": "https://x.test/items", "auth_header": "Bearer c" }), + ), + ( + "github_issues", + serde_json::json!({ "repo": "o/r", "auth_header": "Bearer g" }), + ), + ( + "jira_issues", + serde_json::json!({ "base_url": "https://j.test", "project": "P", "auth_header": "Basic j" }), + ), + ( + "s3", + serde_json::json!({ "bucket": "b", "region": "r", "access_key_id": "AKIA", + "secret_access_key": "s3-secret" }), + ), + ( + "azure_blob", + serde_json::json!({ "bucket": "c", "account_name": "acct", "account_key": "az-key" }), + ), + ( + "gcs", + serde_json::json!({ "bucket": "g", "service_account_key": "{\"private_key\":\"x\"}" }), + ), + ( + "webdav", + serde_json::json!({ "base_url": "https://d.test", "path": "/", "username": "u", + "password": "dav-pass" }), + ), + ("notion", serde_json::json!({ "token": "secret_n", "query": "q" })), + ]; + for (kind, config) in &fixtures { + sqlx::query( + "INSERT INTO sources (id, kb_id, kind, name, config) VALUES ($1, $2, $3, $3, $4)", + ) + .bind(Uuid::now_v7()) + .bind(kb) + .bind(kind) + .bind(config) + .execute(&pool) + .await?; + } + + let listed = sources::list(&pool, kb).await?; + assert_eq!(listed.len(), fixtures.len()); + for s in &listed { + let obj = s.config.as_object().expect("config is an object"); + for key in SOURCE_SECRET_KEYS { + assert!( + !obj.contains_key(*key), + "{}: `{key}` must not reach a viewer, got {:?}", + s.kind, + obj + ); + } + } + // 身份标识留着 + let by_kind = |k: &str| { + listed + .iter() + .find(|s| s.kind == k) + .map(|s| s.config.clone()) + .expect("listed") + }; + assert_eq!(by_kind("s3")["bucket"], "b"); + assert_eq!(by_kind("s3")["access_key_id"], "AKIA"); + assert_eq!(by_kind("azure_blob")["account_name"], "acct"); + assert_eq!(by_kind("webdav")["username"], "u"); + assert_eq!(by_kind("custom")["endpoint"], "https://x.test/items"); + assert_eq!(by_kind("notion")["query"], "q"); + + // 同步那条路仍拿完整配置：凭据只是不出去，不是没了 + for s in &listed { + let full = sources::get(&pool, s.id).await?; + let want = &fixtures.iter().find(|(k, _)| *k == s.kind).unwrap().1; + assert_eq!(&full.config, want, "{}: sync still sees the credentials", s.kind); + } + Ok::<_, anyhow::Error>(()) + } + .await; + + let _ = sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await; + run +} diff --git a/docs/decisions/0013-a-source-should-hand-over-its-history.md b/docs/decisions/0013-a-source-should-hand-over-its-history.md index 1e6ccc1f9..c84d1f663 100644 --- a/docs/decisions/0013-a-source-should-hand-over-its-history.md +++ b/docs/decisions/0013-a-source-should-hand-over-its-history.md @@ -107,3 +107,7 @@ shape — a class with an IRI drawn as a circle is a picture that lies. `sync_custom`. Acceptance: unit tests on `render()`, fixture tests where a real response is obtainable, create → sync → versions present → second sync adds 0, and clean `cargo clippy --workspace --all-targets` and `npm run typecheck`. + +## Revisions + +- 2026-09-03: every connector's credentials stay on the server (#246). Until now only `auth_header` was stripped from responses; the object-storage, WebDAV and Notion keys went out to every viewer. The keys now live in one list, `SOURCE_SECRET_KEYS`, shared by the listing, the create / update responses and the update merge (blank or missing keeps the stored value, an explicit `null` removes it). Adding a connector means adding its keys there first. From 5c7a41dfd5bf966b420d00c1707d3d5b8f3bf539 Mon Sep 17 00:00:00 2001 From: WaylandYang <145302500+WaylandYang@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:44:53 +0800 Subject: [PATCH 08/10] Compose points at the third release candidate (#251) Co-authored-by: Claude Fable 5.1 --- .github/workflows/release.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40a04bf4d..08cdcfade 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ on: workflow_dispatch: inputs: tag: - description: "Image tag to publish (e.g. 0.1.0-rc2)" + description: "Image tag to publish (e.g. 0.1.0-rc3)" required: true permissions: diff --git a/docker-compose.yml b/docker-compose.yml index b97180b30..32f74ca59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: # 写一个不存在的标签，clone 下来第一步就是 manifest unknown。 # 所以撤掉一个版本时必须连它一起改——0.1.0 撤了，现在指向 rc。 # 转正之后改回 0.1.0（或改成 latest 让它自动跟随）。 - image: ${UTOPIA_IMAGE:-ghcr.io/deeplethe/utopia:0.1.0-rc2} + image: ${UTOPIA_IMAGE:-ghcr.io/deeplethe/utopia:0.1.0-rc3} profiles: ["app"] environment: # 默认以 owner 身份运行。要启用受限角色（业务表随便读写、台账只增不改）， From 761302d8f641758aadc07c0afaab16423a134c7f Mon Sep 17 00:00:00 2001 From: WaylandYang <145302500+WaylandYang@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:53:32 +0800 Subject: [PATCH 09/10] The security notes are back, with a private address for reports (#252) Co-authored-by: Claude Fable 5.1 --- SECURITY.md | 50 +++++++++++++++++++++++++++++++++++++++++++++-- SECURITY.zh-CN.md | 2 +- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 645174465..f242746f4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,5 +1,51 @@ # Security -If you find a vulnerability in Utopia, please email **security@deeplethe.com** rather than opening a public issue. Include the affected version or commit, the endpoint or component, and steps to reproduce. +*[中文版](SECURITY.zh-CN.md)* -You will get an acknowledgement within a few days. Once a fix is released, the advisory names the reporter unless you ask otherwise. +Utopia is at v0.1. Below are the **known, unresolved** limits — not a vulnerability report, +but the places the design has not reached yet. + +## Before you put this on a public network + +**Credentials are stored in the clear.** LLM API keys and Ask-the-Data connection strings +are plain text in Postgres (`llm_settings.chat_api_key`, `data_sources.conn_string`). Anyone +who can read the database can read them. Encryption at rest is a 1.0 item; until then, keep +the system and its database inside a trusted network. + +**The default database password is `utopia`.** By default the port binds to loopback +(`127.0.0.1:1517`), so nothing outside the host can connect. If you change `UTOPIA_DB_BIND` +to expose it, change `UTOPIA_DB_PASSWORD` in `.env` first. + +**A data source is only as safe as its grants.** Registering one is a deployment-level +action, but the connection string reaches every workspace the source is granted to. Grant it +only where that database should be visible, and use a read-only database role in the string +itself — the SQL gate below is defence in depth, not a substitute for least privilege at the +source. + +## What is in place + +- **JWT signing key generated on first start** — 32 bytes from a CSPRNG, stored in the + database. No deployment shares a default key. +- **`Secure` on session cookies behind TLS** — decided from `X-Forwarded-Proto`, so local + HTTP development still works. Force it with `UTOPIA_COOKIE_SECURE=true` if your proxy + omits the header. +- **Database port bound to loopback** — `127.0.0.1:1517`; the app reaches the database over + the compose network. +- **Optional least-privilege runtime role** — set `UTOPIA_APP_DB_PASSWORD` and + `UTOPIA_MIGRATION_URL`, and the app connects as a role that can only read and write + business tables and append to the ledger, while migrations run as the owner. +- **Data sources reach only granted workspaces** — a registered database is mounted into a + knowledge base only where an explicit grant exists. Before this, any base admin could + mount any registered source, which crossed tenants. +- **Read-only gate on Ask-the-Data** — parser allowlist, read-only transaction, enforced row + limit; three layers, so a statement past the parser still cannot write. +- **Accounts are deactivated, not deleted** — `users.deactivated_at` blocks sign-in while the + ledger keeps that person's decisions attributable. +- **Passwords hashed with argon2.** + +## Reporting a vulnerability + +Email **security@deeplethe.com** rather than opening a public issue. Include the affected +version or commit, the endpoint or component, and steps to reproduce. You will get an +acknowledgement within a few days, and the release that carries the fix names you unless you +ask otherwise. diff --git a/SECURITY.zh-CN.md b/SECURITY.zh-CN.md index 274957a85..3eedc25fa 100644 --- a/SECURITY.zh-CN.md +++ b/SECURITY.zh-CN.md @@ -25,4 +25,4 @@ Utopia 目前是 v0.1。下面是**已知的、尚未解决的**限制 —— ## 报告漏洞 -请开一个 issue。如果涉及可被利用的细节，先只写复现的最小信息，我们再私下沟通完整内容。 +请发邮件到 **security@deeplethe.com**，不要开公开 issue。写明受影响的版本或提交、端点或组件、复现步骤。几天内会有回复；带修复的那个版本会在说明里致谢，除非你不希望。 From 83099a7707c4b6734faef3141ed2e9c136f5c6dc Mon Sep 17 00:00:00 2001 From: WaylandYang <145302500+WaylandYang@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:16:33 +0800 Subject: [PATCH 10/10] Every source kind the UI offers can be created (#253) Co-authored-by: Claude Fable 5.1 --- Cargo.lock | 26 +++++++- Cargo.toml | 2 + crates/utopia-core/Cargo.toml | 1 + crates/utopia-core/src/models.rs | 61 +++++++++++++++++++ crates/utopia-server/src/ingest_sources.rs | 29 +++++---- crates/utopia-store/src/sources.rs | 26 +++----- .../tests/a_source_kind_is_listed_once.rs | 53 ++++++++++++++++ ...3-a-source-should-hand-over-its-history.md | 1 + web/src/api.ts | 17 +----- web/src/pages/Library.tsx | 36 ++--------- web/src/pages/SourcesRail.tsx | 6 +- web/src/sourceKinds.ts | 29 +++++++++ 12 files changed, 211 insertions(+), 76 deletions(-) create mode 100644 crates/utopia-store/tests/a_source_kind_is_listed_once.rs create mode 100644 web/src/sourceKinds.ts diff --git a/Cargo.lock b/Cargo.lock index 83ce8b63b..d17a6e9c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4180,13 +4180,34 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] + [[package]] name = "strum" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "strum_macros", + "strum_macros 0.28.0", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -4464,7 +4485,7 @@ dependencies = [ "icu_segmenter", "itertools 0.14.0", "memchr", - "strum", + "strum 0.28.0", "thiserror 2.0.20", ] @@ -4908,6 +4929,7 @@ dependencies = [ "serde", "serde_json", "sqlx", + "strum 0.27.2", "thiserror 2.0.20", "uuid", ] diff --git a/Cargo.toml b/Cargo.toml index 36502a068..890444caf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,8 @@ dotenvy = "0.15" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } flate2 = "1" +# 枚举的字符串化与遍历（来源种类 `SourceKind`）——手写两份清单的漂移就是 #247 +strum = { version = "0.27", features = ["derive"] } # Phase 1: 摄入/检索/LLM utopia-ingest = { path = "crates/utopia-ingest" } diff --git a/crates/utopia-core/Cargo.toml b/crates/utopia-core/Cargo.toml index 7075cfd84..2d2e8e10f 100644 --- a/crates/utopia-core/Cargo.toml +++ b/crates/utopia-core/Cargo.toml @@ -14,3 +14,4 @@ chrono.workspace = true sqlx.workspace = true figment.workspace = true pgvector.workspace = true +strum.workspace = true diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index 4085d6ae7..63b195c2a 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -165,6 +165,67 @@ impl Source { } } +/// 来源的种类。**一处定义，三处消费**：创建时的白名单、同步时的分派（按枚举穷举匹配， +/// 加一种就得决定它怎么同步）、前端的下拉框（`web/src/sourceKinds.ts`，由 +/// `utopia-store` 的测试对表）。 +/// +/// 此前后端两张手写清单各自演进：五种连接器加了同步分支、进了界面，却没进创建的 +/// 白名单，界面上选得到、建的时候报「kind must be one of…」（#247）。变体顺序就是 +/// 对话框里的顺序；字符串形式由 strum 按 snake_case 生成，不再手写 +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + strum::EnumIter, + strum::IntoStaticStr, + strum::EnumString, +)] +#[strum(serialize_all = "snake_case")] +pub enum SourceKind { + Folder, + Url, + Rss, + GithubIssues, + JiraIssues, + S3, + AzureBlob, + Gcs, + Webdav, + Notion, + Api, + Custom, + /// 每个库自带的记忆来源，不可建不可删（0015） + Memory, + /// 老数据里 `sources.kind` 的默认值，没有对应的界面 + Upload, +} + +impl SourceKind { + pub fn as_str(self) -> &'static str { + self.into() + } + + pub fn parse(s: &str) -> Option { + s.parse().ok() + } + + pub fn all() -> impl Iterator { + ::iter() + } + + /// 人能从界面建的：`memory` 与 `upload` 之外的全部 + pub fn creatable_by_hand(self) -> bool { + !matches!(self, Self::Memory | Self::Upload) + } + + pub fn creatable() -> impl Iterator { + Self::all().filter(|k| k.creatable_by_hand()) + } +} + /// 来源同步运行记录（渠道审计历史）。 #[derive(Debug, Clone, Serialize, sqlx::FromRow)] pub struct SyncRun { diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs index 56afd33c7..a1803d482 100644 --- a/crates/utopia-server/src/ingest_sources.rs +++ b/crates/utopia-server/src/ingest_sources.rs @@ -8,6 +8,7 @@ use crate::state::AppState; use chrono::{DateTime, Utc}; use sha2::{Digest, Sha256}; use utopia_core::models::Source; +use utopia_core::models::SourceKind; use uuid::Uuid; /// 单次同步的新文档上限（防超长 feed/URL 列表拖垮任务） @@ -48,17 +49,23 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<() let run_id = utopia_store::sources::start_run(&state.pool, source_id).await?; state.emit_source(source.kb_id); - let outcome = match source.kind.as_str() { - "url" => sync_urls(state, &source).await, - "rss" => sync_rss(state, &source).await, - "custom" => sync_custom(state, &source).await, - "github_issues" => sync_github_issues(state, &source).await, - "jira_issues" => sync_jira_issues(state, &source).await, - "s3" | "azure_blob" | "gcs" => sync_object_storage(state, &source).await, - "webdav" => sync_webdav(state, &source).await, - "notion" => sync_notion(state, &source).await, - // folder / api 无拉取语义 - _ => Ok(SyncStats::default()), + // 按枚举穷举：加一种来源就得在这里决定它怎么同步，编译器不放过漏掉的那一支 + let outcome = match SourceKind::parse(&source.kind) { + Some(SourceKind::Url) => sync_urls(state, &source).await, + Some(SourceKind::Rss) => sync_rss(state, &source).await, + Some(SourceKind::Custom) => sync_custom(state, &source).await, + Some(SourceKind::GithubIssues) => sync_github_issues(state, &source).await, + Some(SourceKind::JiraIssues) => sync_jira_issues(state, &source).await, + Some(SourceKind::S3 | SourceKind::AzureBlob | SourceKind::Gcs) => { + sync_object_storage(state, &source).await + } + Some(SourceKind::Webdav) => sync_webdav(state, &source).await, + Some(SourceKind::Notion) => sync_notion(state, &source).await, + // 被动容器：folder / api / memory / upload 没有拉取语义 + Some(SourceKind::Folder | SourceKind::Api | SourceKind::Memory | SourceKind::Upload) => { + Ok(SyncStats::default()) + } + None => Err(anyhow::anyhow!("unknown source kind `{}`", source.kind)), }; match outcome { diff --git a/crates/utopia-store/src/sources.rs b/crates/utopia-store/src/sources.rs index e4d6f3819..ab7add384 100644 --- a/crates/utopia-store/src/sources.rs +++ b/crates/utopia-store/src/sources.rs @@ -2,28 +2,22 @@ use chrono::{DateTime, Utc}; use sqlx::PgPool; -use utopia_core::models::{Role, Source, SourceView, SyncRun, SOURCE_SECRET_KEYS}; +use utopia_core::models::{Role, Source, SourceKind, SourceView, SyncRun, SOURCE_SECRET_KEYS}; use utopia_core::{AppError, AppResult}; use uuid::Uuid; /// folder = 纯容器（上传/拖拽入内，无同步语义）；url/rss = 拉取型；api = 推送型。 /// 本机目录监听（watch_folder）已否决——自部署用户看不到服务器磁盘； -/// 未来的 watch 形态是对象存储/网盘（P5 连接器，与 BlobStore 接缝配套）。 +/// 对象存储 / WebDAV / Notion 是它的替代形态（0013）。 /// custom = 自定义拉取器：任何实现 Utopia ingest 接口的 URL（返回 items JSON）即可定时摄取。 /// github_issues / jira_issues = 工单：一张工单连同它的状态变更史成为一篇文档。 /// -/// **改这里就得改前端那份清单**（`Library.tsx` 的建来源对话框与 `api.ts` 的 -/// `SourceView["kind"]`）。两处对不上时的症状是：界面上选得到、建的时候报 -/// 「kind must be one of…」——单元测试与 tsc 都看不见，只有端到端会撞上。 -pub const KINDS: &[&str] = &[ - "folder", - "url", - "rss", - "api", - "custom", - "github_issues", - "jira_issues", -]; +/// 种类的清单**不在这里写**：`SourceKind`（utopia-core）一个枚举出全部——创建的白名单、 +/// 同步的分派、前端的下拉框（有测试对表）。从前这里有一张手写的 `KINDS`，五种连接器 +/// 加了同步却没进这张表，界面上选得到、建不出来（#247） +pub fn creatable_kinds() -> Vec<&'static str> { + SourceKind::creatable().map(|k| k.as_str()).collect() +} /// 校验并规范化标准 5 段 cron 表达式（内部用 cron crate 的 6 段：补秒位）。 pub fn validate_cron(expr: &str) -> AppResult { @@ -93,10 +87,10 @@ pub async fn create( sync_interval_minutes: Option, sync_cron: Option<&str>, ) -> AppResult { - if !KINDS.contains(&kind) { + if !SourceKind::parse(kind).is_some_and(|k| k.creatable_by_hand()) { return Err(AppError::Validation(format!( "kind must be one of: {}", - KINDS.join(", ") + creatable_kinds().join(", ") ))); } if name.trim().is_empty() { diff --git a/crates/utopia-store/tests/a_source_kind_is_listed_once.rs b/crates/utopia-store/tests/a_source_kind_is_listed_once.rs new file mode 100644 index 000000000..4e6164115 --- /dev/null +++ b/crates/utopia-store/tests/a_source_kind_is_listed_once.rs @@ -0,0 +1,53 @@ +//! #247：来源的种类只在一处定义，前后端对表。 +//! +//! 后端 `SourceKind`（utopia-core）一个枚举出两份清单：创建时的白名单、同步时的分派 +//! （后者按枚举穷举匹配，编译器保证加了种类就得决定它怎么同步）。前端那一份在 +//! `web/src/sourceKinds.ts`，这个测试把它读出来跟枚举比——此前两边各自手写，五种 +//! 连接器进了界面、进了同步，却没进创建白名单，界面上选得到、建的时候报 +//! 「kind must be one of…」。单元测试与 tsc 都看不见的那种漂移，这里看得见。 +//! +//! 不需要数据库。 + +use std::path::Path; +use utopia_core::models::SourceKind; + +/// 从 `CREATABLE_SOURCE_KINDS = [ "…", … ] as const` 里把引号里的字面量按顺序读出来 +fn frontend_kinds(src: &str) -> Vec { + let start = src + .find("CREATABLE_SOURCE_KINDS = [") + .expect("web/src/sourceKinds.ts declares CREATABLE_SOURCE_KINDS"); + let body = &src[start..]; + let end = body.find(']').expect("the array closes"); + body[..end] + .split('"') + .skip(1) + .step_by(2) + .map(str::to_string) + .collect() +} + +#[test] +fn the_frontend_list_matches_the_backend_enum() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../web/src/sourceKinds.ts"); + let src = + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())); + let frontend = frontend_kinds(&src); + let backend: Vec = SourceKind::creatable() + .map(|k| k.as_str().to_string()) + .collect(); + assert_eq!( + frontend, backend, + "web/src/sourceKinds.ts and utopia_core::models::SourceKind list different kinds (order matters: it is the dialog's order)" + ); +} + +#[test] +fn every_kind_round_trips_through_its_string() { + for k in SourceKind::all() { + assert_eq!(SourceKind::parse(k.as_str()), Some(k), "{k:?}"); + } + assert_eq!(SourceKind::parse("watch_folder"), None); + assert!(!SourceKind::Memory.creatable_by_hand()); + assert!(!SourceKind::Upload.creatable_by_hand()); + assert!(SourceKind::S3.creatable_by_hand()); +} diff --git a/docs/decisions/0013-a-source-should-hand-over-its-history.md b/docs/decisions/0013-a-source-should-hand-over-its-history.md index c84d1f663..ef7f42ced 100644 --- a/docs/decisions/0013-a-source-should-hand-over-its-history.md +++ b/docs/decisions/0013-a-source-should-hand-over-its-history.md @@ -111,3 +111,4 @@ shape — a class with an IRI drawn as a circle is a picture that lies. ## Revisions - 2026-09-03: every connector's credentials stay on the server (#246). Until now only `auth_header` was stripped from responses; the object-storage, WebDAV and Notion keys went out to every viewer. The keys now live in one list, `SOURCE_SECRET_KEYS`, shared by the listing, the create / update responses and the update merge (blank or missing keeps the stored value, an explicit `null` removes it). Adding a connector means adding its keys there first. +- 2026-09-03: the five connectors added under this record could not be created (#247): the store's hand-written `KINDS` allowlist stopped at seven kinds while the sync dispatcher and the UI knew twelve. The kinds now come from one enum, `SourceKind` in `utopia-core`; the allowlist is derived from it, the dispatcher matches it exhaustively, and a test compares the frontend's `web/src/sourceKinds.ts` against it, so the three can no longer drift apart. diff --git a/web/src/api.ts b/web/src/api.ts index c2775ed09..1ed586093 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -1,3 +1,4 @@ +import type { SourceKind } from "./sourceKinds"; import { S, lang } from "./i18n"; export class ApiError extends Error { @@ -179,21 +180,7 @@ export interface ExtractionDrop { export interface SourceView { id: string; - kind: - | "folder" - | "url" - | "rss" - | "api" - | "custom" - | "github_issues" - | "jira_issues" - | "s3" - | "azure_blob" - | "gcs" - | "webdav" - | "notion" - | "memory" - | "upload"; + kind: SourceKind; name: string; config: { urls?: string[]; diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsx index e5764ca83..5a3797375 100644 --- a/web/src/pages/Library.tsx +++ b/web/src/pages/Library.tsx @@ -14,6 +14,10 @@ import { } from "lucide-react"; import { api, type Doc, type ExtractionDrop, type SourceView } from "../api"; import { S } from "../i18n"; +import { + CREATABLE_SOURCE_KINDS, + type CreatableSourceKind, +} from "../sourceKinds"; import { useKb, useKbId } from "../kb"; import { toast } from "../toast"; import { Chip, type ChipTone, DangerConfirm, Loading, Pager } from "../ui"; @@ -1267,20 +1271,7 @@ function SourceModal({ /** isApi=true 时父级紧接着打开密钥弹窗 */ onDone: (id?: string, isApi?: boolean) => void; }) { - const [kind, setKind] = useState< - | "folder" - | "url" - | "rss" - | "custom" - | "api" - | "github_issues" - | "jira_issues" - | "s3" - | "azure_blob" - | "gcs" - | "webdav" - | "notion" - >("folder"); + const [kind, setKind] = useState("folder"); const [name, setName] = useState(""); const [icon, setIcon] = useState(null); const [urls, setUrls] = useState(""); @@ -1447,22 +1438,7 @@ function SourceModal({
{/* 类型 */}
- {( - [ - "folder", - "url", - "rss", - "github_issues", - "jira_issues", - "s3", - "azure_blob", - "gcs", - "webdav", - "notion", - "api", - "custom", - ] as const - ).map((k) => { + {CREATABLE_SOURCE_KINDS.map((k) => { const Icon = KIND_ICON[k]; return (