diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40a04bf4..08cdcfad 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ on: workflow_dispatch: inputs: tag: - description: "Image tag to publish (e.g. 0.1.0-rc2)" + description: "Image tag to publish (e.g. 0.1.0-rc3)" required: true permissions: diff --git a/Cargo.lock b/Cargo.lock index 4fd49555..d17a6e9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,6 +116,16 @@ dependencies = [ "password-hash", ] +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "async-compression" version = "0.4.43" @@ -324,6 +334,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -934,6 +950,24 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + [[package]] name = "debug_unsafe" version = "0.1.4" @@ -1354,6 +1388,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.34" @@ -1427,6 +1476,7 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -1569,6 +1619,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + [[package]] name = "hex" version = "0.4.3" @@ -1727,7 +1783,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -2128,7 +2184,7 @@ version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ - "base64", + "base64 0.22.1", "ed25519-dalek", "getrandom 0.2.17", "hmac", @@ -2555,6 +2611,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "object" version = "0.39.1" @@ -2572,7 +2638,7 @@ checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" dependencies = [ "async-trait", "aws-lc-rs", - "base64", + "base64 0.22.1", "bytes", "chrono", "crc-fast", @@ -2806,7 +2872,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -3305,7 +3371,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -3875,7 +3941,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "crc", @@ -3952,7 +4018,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "bytes", @@ -3996,7 +4062,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "chrono", @@ -4114,13 +4180,34 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] + [[package]] name = "strum" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "strum_macros", + "strum_macros 0.28.0", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -4212,7 +4299,7 @@ checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07" dependencies = [ "aho-corasick", "arc-swap", - "base64", + "base64 0.22.1", "bitpacking", "bon", "byteorder", @@ -4398,7 +4485,7 @@ dependencies = [ "icu_segmenter", "itertools 0.14.0", "memchr", - "strum", + "strum 0.28.0", "thiserror 2.0.20", ] @@ -4842,6 +4929,7 @@ dependencies = [ "serde", "serde_json", "sqlx", + "strum 0.27.2", "thiserror 2.0.20", "uuid", ] @@ -4922,6 +5010,7 @@ dependencies = [ "async-trait", "axum", "axum-extra", + "base64 0.23.1", "chrono", "dotenvy", "feed-rs", @@ -4929,6 +5018,7 @@ dependencies = [ "futures-util", "jsonwebtoken", "object_store", + "percent-encoding", "quick-xml 0.42.0", "reqwest", "serde", @@ -4940,6 +5030,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "url", "utopia-core", "utopia-extract", "utopia-ingest", @@ -4947,6 +5038,7 @@ dependencies = [ "utopia-search", "utopia-store", "uuid", + "wiremock", ] [[package]] @@ -5506,6 +5598,29 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64 0.22.1", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/Cargo.toml b/Cargo.toml index 36502a06..890444ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,8 @@ dotenvy = "0.15" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } flate2 = "1" +# 枚举的字符串化与遍历（来源种类 `SourceKind`）——手写两份清单的漂移就是 #247 +strum = { version = "0.27", features = ["derive"] } # Phase 1: 摄入/检索/LLM utopia-ingest = { path = "crates/utopia-ingest" } diff --git a/README.md b/README.md index a46c5af1..a71c26b2 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ One Rust binary and one Postgres. Full-text search is embedded in the binary, ve | **Entity resolution and review** | Duplicates are resolved in three stages: exact name or alias, embedding similarity, then a model's call on the doubtful pairs. Every merge can be undone. Uncertain cases go to a review queue: low-confidence extractions, suspected duplicates and cardinality conflicts. | | **Reasoning and derivation** | Ontology axioms compile into rules: transitivity, symmetry, inverses and relation hierarchy derive new facts by forward chaining. Derivation is off by default, since a wrong axiom derives wrong facts. A derived fact is marked as such on the graph, carries validity and confidence like any other, and shows what it was derived from. When it contradicts an asserted fact, the asserted one stands. | | **Conflict detection** | Three kinds of conflict, three sets of choices. A new fact that clashes with an older one: close the old, keep both, or reject the new. Data that breaks an axiom (self-loop, asymmetry, transitive cycle, cardinality): retract the fact, relax the axiom, or accept both. The ontology itself is checked first, because violations of a self-contradictory ontology are noise. | -| **Ontology-driven querying** | Mount a Postgres database on a base and chat can query it alongside the documents. The agent proposes how its tables map onto the ontology, and you confirm. The method behind it, [Ontology2SQL](https://github.com/deeplethe/ontology2sql), is state of the art on BIRD Mini-Dev for SQLite and PostgreSQL ([submission](https://github.com/bird-bench/bird-bench.github.io/pull/218)). | +| **Ontology-driven querying** | Mount a database on a base (Postgres, Trino for Iceberg / Delta Lake / Hive, Databricks, Snowflake) and chat can query it alongside the documents. The agent proposes how its tables map onto the ontology, and you confirm. The method behind it, [Ontology2SQL](https://github.com/deeplethe/ontology2sql), is state of the art on BIRD Mini-Dev for SQLite and PostgreSQL ([submission](https://github.com/bird-bench/bird-bench.github.io/pull/218)). | | **Multi-user and permissions** | Each knowledge base has its own members and roles: owner, admin, editor and viewer. Open bases are readable by everyone in the deployment, restricted ones only by invitation. The first account registered becomes the system administrator. | | **Decision ledger** | Confirming or rejecting a fact, merging or reverting an entity, rebuilding the graph: each leaves a record of who, when, and what the object looked like at the time. The ledger is append-only, and a record outlives its object, even the base it belonged to. | | **[Decision intelligence (in development)](#roadmap)** | Record a decision, replay both what was understood and the course it took, and reason over overlaid scenarios. | @@ -103,7 +103,7 @@ cd web && pnpm install && pnpm dev - [ ] **Decision reasoning**: constraint computation, and replaying a decision after the fact - [ ] **Execution gate**: checking an agent's calls against ontology rules and symbolic logic -- [ ] **Lakehouse for mapping and querying**: mapping exploration and Ontology2SQL over Iceberg / Delta Lake, Databricks, Snowflake and MaxCompute +- [ ] **MaxCompute**: mapping exploration and Ontology2SQL over Alibaba Cloud MaxCompute (Iceberg / Delta Lake via Trino, Databricks and Snowflake are in, awaiting a run against a real cluster) - [ ] **More sources**: MySQL, ClickHouse and Doris drivers; S3, WebDAV, Notion and Feishu connectors - [ ] **Time to the moment**: an `instant` precision beside year / month / day, for sources that carry a real timestamp. Today a connector rounds it to a UTC day, which can shift an event across midnight by one day - [ ] **Agent memory over MCP**: episode writes, the retrieve endpoint, and the MCP server diff --git a/SECURITY.md b/SECURITY.md index 30815ea4..f242746f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -45,5 +45,7 @@ source. ## Reporting a vulnerability -Open an issue. If it involves exploitable detail, start with the minimum needed to reproduce -and we will follow up privately. +Email **security@deeplethe.com** rather than opening a public issue. Include the affected +version or commit, the endpoint or component, and steps to reproduce. You will get an +acknowledgement within a few days, and the release that carries the fix names you unless you +ask otherwise. diff --git a/SECURITY.zh-CN.md b/SECURITY.zh-CN.md index 274957a8..3eedc25f 100644 --- a/SECURITY.zh-CN.md +++ b/SECURITY.zh-CN.md @@ -25,4 +25,4 @@ Utopia 目前是 v0.1。下面是**已知的、尚未解决的**限制 —— ## 报告漏洞 -请开一个 issue。如果涉及可被利用的细节，先只写复现的最小信息，我们再私下沟通完整内容。 +请发邮件到 **security@deeplethe.com**，不要开公开 issue。写明受影响的版本或提交、端点或组件、复现步骤。几天内会有回复；带修复的那个版本会在说明里致谢，除非你不希望。 diff --git a/crates/utopia-core/Cargo.toml b/crates/utopia-core/Cargo.toml index 7075cfd8..2d2e8e10 100644 --- a/crates/utopia-core/Cargo.toml +++ b/crates/utopia-core/Cargo.toml @@ -14,3 +14,4 @@ chrono.workspace = true sqlx.workspace = true figment.workspace = true pgvector.workspace = true +strum.workspace = true diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index f4fb8879..63b195c2 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -137,6 +137,95 @@ pub struct Source { pub created_at: DateTime, } +/// 来源配置里**用来鉴权**的那几个键。凭据只进不出：列表与创建 / 更新的响应都剔掉， +/// 更新时客户端没传或传空串就保留库里的原值，审计里也不落。 +/// +/// **一张表，四处共用。** 此前那条规矩只对 `auth_header` 一个键成立，而对象存储、 +/// WebDAV、Notion 各自的密钥原样发给了每一个 Viewer（#246）。加连接器时**先加这里**， +/// 再写读它的代码。`username` / `account_name` / `access_key_id` 这类是身份标识， +/// 单独拿到鉴不了权，留着让界面显示得出「这是哪个账号」。 +pub const SOURCE_SECRET_KEYS: &[&str] = &[ + "auth_header", + "token", + "password", + "secret_access_key", + "account_key", + "service_account_key", +]; + +impl Source { + /// 剔掉凭据后的这条来源——任何要回给客户端的 `Source` 都从这里过 + pub fn without_secrets(mut self) -> Self { + if let Some(obj) = self.config.as_object_mut() { + for key in SOURCE_SECRET_KEYS { + obj.remove(*key); + } + } + self + } +} + +/// 来源的种类。**一处定义，三处消费**：创建时的白名单、同步时的分派（按枚举穷举匹配， +/// 加一种就得决定它怎么同步）、前端的下拉框（`web/src/sourceKinds.ts`，由 +/// `utopia-store` 的测试对表）。 +/// +/// 此前后端两张手写清单各自演进：五种连接器加了同步分支、进了界面，却没进创建的 +/// 白名单，界面上选得到、建的时候报「kind must be one of…」（#247）。变体顺序就是 +/// 对话框里的顺序；字符串形式由 strum 按 snake_case 生成，不再手写 +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + strum::EnumIter, + strum::IntoStaticStr, + strum::EnumString, +)] +#[strum(serialize_all = "snake_case")] +pub enum SourceKind { + Folder, + Url, + Rss, + GithubIssues, + JiraIssues, + S3, + AzureBlob, + Gcs, + Webdav, + Notion, + Api, + Custom, + /// 每个库自带的记忆来源，不可建不可删（0015） + Memory, + /// 老数据里 `sources.kind` 的默认值，没有对应的界面 + Upload, +} + +impl SourceKind { + pub fn as_str(self) -> &'static str { + self.into() + } + + pub fn parse(s: &str) -> Option { + s.parse().ok() + } + + pub fn all() -> impl Iterator { + ::iter() + } + + /// 人能从界面建的：`memory` 与 `upload` 之外的全部 + pub fn creatable_by_hand(self) -> bool { + !matches!(self, Self::Memory | Self::Upload) + } + + pub fn creatable() -> impl Iterator { + Self::all().filter(|k| k.creatable_by_hand()) + } +} + /// 来源同步运行记录（渠道审计历史）。 #[derive(Debug, Clone, Serialize, sqlx::FromRow)] pub struct SyncRun { @@ -513,6 +602,13 @@ pub struct GraphEdge { pub valid_from: Option>, pub valid_to: Option>, pub confidence: f32, + /// 有争议（0017 §3）：有一条 open 的公理违规或时态冲突指着它。整条边画成 + /// 警戒色——环在节点上、边还是灰的，余光分不出来 + pub contested: bool, + /// 幽灵边（0017 §3）：一条**没有落地**的派生——推出来了却撞上断言。`id` 是那条 + /// `derived_contradiction` 违规的 id，不是任何事实；`derived` 同时为 true， + /// 所以它跟着派生开关走 + pub blocked: bool, } /// 实体详情页的事实行（时间线）。 @@ -549,6 +645,10 @@ pub struct EntityFact { pub corrected: bool, /// 证据集合里最新的文档时间——开放事实的"最后确认时间"（时效性透明化） pub last_evidence_time: Option>, + /// 有争议（0017 §3）：`{ kind, ref_id, derived? }`——哪一种（违规的 kind，或 + /// `temporal_conflict`）、Review 里那一项的 id、派生撞断言时推出来的那句话。 + /// 一条只报最新的一处；行**不压暗**，断言仍然活着 + pub contested: Option, } /// 实体的一次认知变更（记录时间轴上的事件，与 EntityFact 的有效时间轴正交）。 @@ -867,10 +967,10 @@ pub struct ConceptMapping { /// /// **两条事实都展开成 主-谓-宾 文本**：Review 页要让人一眼看出矛盾在哪， /// 而两个 UUID 看不出任何东西。自反那一类两条相同——它就是一条事实。 -#[derive(Debug, Clone, Serialize, sqlx::FromRow)] +#[derive(Debug, Clone, Serialize)] pub struct AxiomViolation { pub id: Uuid, - /// self_loop | asymmetry | cycle | functional + /// self_loop | asymmetry | cycle | functional | signature | derived_contradiction pub kind: String, /// 判据来自哪条关系。人若判「公理写错了」，从这里进本体去改 pub predicate: Option, @@ -881,6 +981,12 @@ pub struct AxiomViolation { /// 环的长度（含首尾）。其余三类为 0——前端据此决定要不要显示「查看路径」 pub path_len: i32, pub detected_at: chrono::DateTime, + /// `derived_contradiction` 独有（0017）：推出来的那条三元组——它没有落库， + /// 只能在这里写出来。字段见 `reasoning::run`。其余种类是 `{}` + pub detail: serde_json::Value, + /// 审核线索（0017 §2）：`stale`（旧断言没写结束日期）、`duplicate`（有同名 + /// 实体）、`unsure`（抽取置信度低）。只给一条，没有就空 + pub hint: Option, } /// 本体自己的一处自相矛盾（见 `ontology_defects`）。 @@ -891,8 +997,11 @@ pub struct AxiomViolation { pub struct OntologyDefect { pub id: Uuid, /// symmetric_and_asymmetric | transitive_and_functional | subclass_cycle - /// | disjoint_with_ancestor | inherits_disjoint + /// | disjoint_with_ancestor | inherits_disjoint | inverse_of_itself + /// | inverse_not_mutual | sub_property_cycle | rules_disagree pub kind: String, + /// `rules_disagree` 独有（0017）：哪两条规则、撞在哪条公理上、几对、几个例子 + pub detail: serde_json::Value, /// 出问题那个对象的标签（类或谓词）。查不到就是它已经被删了 pub subject_label: Option, /// 另一方：互斥的那个类 @@ -924,6 +1033,61 @@ pub struct DerivedFactView { pub premises: Vec, } +/// 一条**没有落地**的派生（0017 §3）：推出来了，撞上一条断言，拦在图外。 +/// +/// 它没有 id——落库的才有。这里用那条 `derived_contradiction` 违规的 id 指它， +/// 面板上的「没落地的」一档与图上的幽灵边都靠这个 id 对上 Review 里的卡片。 +#[derive(Debug, Clone, Serialize, sqlx::FromRow)] +pub struct BlockedDerivation { + pub violation_id: Uuid, + pub subject_id: Uuid, + pub subject: String, + pub object_id: Uuid, + pub object: String, + pub predicate: String, + pub rule: String, + /// 声明所在的谓词 + pub via_label: String, + pub valid_from: Option>, + pub valid_to: Option>, + /// 挡住它的那条断言，与它的三元组文本 + pub against_fact: Uuid, + pub against_text: String, + /// 前提事实 id，按推导顺序——证明链从这里展开 + pub premises: Vec, +} + +/// 证明的一步：一条断言前提，连同它的证据（0002 R2）。 +/// +/// 前提一律是断言（`fact_derivations` 不记派生），所以证明是一条链而不是一棵树： +/// 派生 → 按 `seq` 排好的断言 → 每条断言的原句。叶子就是 chunk。 +#[derive(Debug, Clone, Serialize)] +pub struct ProofStep { + pub seq: i32, + pub fact_id: Uuid, + pub subject_id: Uuid, + pub subject: String, + pub predicate_id: Option, + /// 本体里的关系名；空谓词事实（0010）不参与推导，这里理论上恒有值， + /// 留 Option 是不在读路径上撒谎 + pub predicate: Option, + pub object_id: Option, + pub object: Option, + pub valid_from: Option>, + pub valid_to: Option>, + pub confidence: f32, + /// 这条前提后来被撤了。派生随之失效，但证明还要读得出「当时靠的是什么」 + pub retracted: bool, + pub evidence: Vec, +} + +/// 一条派生事实的完整证明：它本身，加上按顺序展开到原句的前提。 +#[derive(Debug, Clone, Serialize)] +pub struct Proof { + pub derived: DerivedFactView, + pub steps: Vec, +} + /// 审核队列各档的**真实条数**。 /// /// 与列表分开取是有意的：列表有上限（一页十条），数数没有。从前左栏读的是 diff --git a/crates/utopia-reason/src/derive.rs b/crates/utopia-reason/src/derive.rs index 73c8ab2e..aa6c473d 100644 --- a/crates/utopia-reason/src/derive.rs +++ b/crates/utopia-reason/src/derive.rs @@ -20,7 +20,7 @@ //! 前提 A `[2020,2023)`、前提 B `[2022,∞)` → 派生 `[2022,2023)`。交集为空 //! 就不推——两段没有重叠的时候,链本身在任何时刻都不成立。 -use crate::{Axioms, Edge, MAX_DEPTH}; +use crate::{Axioms, Edge, Kind, MAX_DEPTH}; use std::collections::{HashMap, HashSet}; use uuid::Uuid; @@ -31,7 +31,7 @@ use uuid::Uuid; /// ——悄悄截断会让「推完了」和「推了一部分」长得一模一样。 pub const MAX_DERIVED_PER_PREDICATE: usize = 20_000; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Rule { /// `A p B` ∧ `B p C` ⟹ `A p C` Transitive, @@ -379,6 +379,248 @@ pub fn validity( Some(acc) } +// ===================== 矛盾：派生撞上了什么（0017） ===================== + +/// 一条派生撞上了一条断言。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Clash { + /// `Derivation::facts` 里的下标 + pub derived: usize, + /// 撞在哪条公理上：`Functional`（含 inverse_functional）、`Asymmetry`、`SelfLoop` + pub axiom: Kind, + /// 被撞的断言。自环没有对方，取派生的最后一条前提 + pub against: Uuid, +} + +/// 两条规则加在一起产出了互相矛盾的派生。 +/// +/// **按规则对聚合，不逐对报**：`ceo_of ⊑ works_at` 加 `works_at` functional，每个有 +/// 两个 ceo 的组织就撞一对——根子是那两条声明，逐对进队列只会淹掉 Review。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuleClash { + /// (声明所在的谓词, 规则种类)，两条按 (谓词, 种类) 排过序，a ≤ b + pub a: (Uuid, Rule), + pub b: (Uuid, Rule), + pub axiom: Kind, + /// 互撞的派生对，按 `Derivation::facts` 的下标 + pub pairs: Vec<(usize, usize)>, +} + +/// 半开区间 `[from, to)`，两端可空 +type Span = (Option, Option); +/// (谓词, 一端) → 另一端的边：(另一端, 事实, 区间)。functional 两个方向各一份 +type ByEnd = HashMap<(Uuid, Uuid), Vec<(Uuid, Uuid, Span)>>; +/// 一条规则的身份：声明所在的谓词 + 规则种类 +type RuleSide = (Uuid, Rule); +/// 互撞的派生对，按 (规则 a, 规则 b, 撞在哪条公理上) 分组 +type Grouped = HashMap<(RuleSide, RuleSide, Kind), Vec<(usize, usize)>>; + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Contradictions { + pub with_assertions: Vec, + pub between_derivations: Vec, +} + +impl Contradictions { + /// 不该落地的派生下标：撞过断言的，和撞过别的派生的。**写图宁少勿错**（0002） + pub fn blocked(&self) -> HashSet { + let mut out: HashSet = self.with_assertions.iter().map(|c| c.derived).collect(); + for rc in &self.between_derivations { + for (i, j) in &rc.pairs { + out.insert(*i); + out.insert(*j); + } + } + out + } +} + +/// 拿公理量一遍派生：与断言撞的逐条列出，派生之间撞的按规则对聚合。 +/// +/// 只查四类——`functional`（含 inverse）、`asymmetric`、`irreflexive`——因为只有它们 +/// 能由**两条边**判出矛盾；传递环那类要走闭包，派生本身就是闭包的一部分，R0 对断言 +/// 查过就够了。functional 与 asymmetric 都要求**有效区间重叠**：Mira 走了 Devin +/// 接任，两条 `ceo_of` 区间不交，那是接任，不是矛盾。 +/// +/// 撞上断言的派生一律**不落地**（asserted > derived，硬性）；这一步把「让路」这件事 +/// 从静默变成可见——0002 那张表里写了没做的那一行。 +pub fn contradictions( + derivation: &Derivation, + edges: &[TimedEdge], + axioms: &HashMap, + spans: &HashMap, Option)>, +) -> Contradictions { + // 断言的三份索引：(谓词, 主) → 宾；(谓词, 宾) → 主；(谓词, 主, 宾) → 边 + let mut by_ps: ByEnd = HashMap::new(); + let mut by_po: ByEnd = HashMap::new(); + let mut by_spo: HashMap<(Uuid, Uuid, Uuid), Vec<(Uuid, Span)>> = HashMap::new(); + for e in edges { + let span = (e.from, e.to); + let x = e.edge; + by_ps + .entry((x.predicate, x.subject)) + .or_default() + .push((x.object, x.fact, span)); + by_po + .entry((x.predicate, x.object)) + .or_default() + .push((x.subject, x.fact, span)); + by_spo + .entry((x.predicate, x.subject, x.object)) + .or_default() + .push((x.fact, span)); + } + + let mut out = Contradictions::default(); + // 派生的区间：与落库那一侧同一个函数算，算不出的（前提区间不交）本来就不会落 + let derived_spans: Vec> = derivation + .facts + .iter() + .map(|d| validity(&d.premises, spans)) + .collect(); + + for (i, d) in derivation.facts.iter().enumerate() { + let Some(span) = derived_spans[i] else { + continue; + }; + let Some(ax) = axioms.get(&d.predicate) else { + continue; + }; + let Some(&last) = d.premises.last() else { + continue; + }; + if ax.irreflexive && d.subject == d.object { + out.with_assertions.push(Clash { + derived: i, + axiom: Kind::SelfLoop, + against: last, + }); + } + if ax.asymmetric { + if let Some(v) = by_spo.get(&(d.predicate, d.object, d.subject)) { + for (fact, sp) in v { + if overlap(span, *sp).is_some() { + out.with_assertions.push(Clash { + derived: i, + axiom: Kind::Asymmetry, + against: *fact, + }); + } + } + } + } + if ax.functional { + if let Some(v) = by_ps.get(&(d.predicate, d.subject)) { + for (obj, fact, sp) in v { + if *obj != d.object && overlap(span, *sp).is_some() { + out.with_assertions.push(Clash { + derived: i, + axiom: Kind::Functional, + against: *fact, + }); + } + } + } + } + if ax.inverse_functional { + if let Some(v) = by_po.get(&(d.predicate, d.object)) { + for (subj, fact, sp) in v { + if *subj != d.subject && overlap(span, *sp).is_some() { + out.with_assertions.push(Clash { + derived: i, + axiom: Kind::Functional, + against: *fact, + }); + } + } + } + } + } + + // 派生之间：同样三份索引，只不过键的是下标 + let mut d_ps: HashMap<(Uuid, Uuid), Vec> = HashMap::new(); + let mut d_po: HashMap<(Uuid, Uuid), Vec> = HashMap::new(); + let mut d_spo: HashMap<(Uuid, Uuid, Uuid), Vec> = HashMap::new(); + for (i, d) in derivation.facts.iter().enumerate() { + if derived_spans[i].is_none() { + continue; + } + d_ps.entry((d.predicate, d.subject)).or_default().push(i); + d_po.entry((d.predicate, d.object)).or_default().push(i); + d_spo + .entry((d.predicate, d.subject, d.object)) + .or_default() + .push(i); + } + let mut grouped: Grouped = HashMap::new(); + let mut note = |i: usize, j: usize, axiom: Kind| { + let (i, j) = if i < j { (i, j) } else { (j, i) }; + let ri = (derivation.facts[i].via, derivation.facts[i].rule); + let rj = (derivation.facts[j].via, derivation.facts[j].rule); + let (a, b) = if (ri.0, ri.1.as_str()) <= (rj.0, rj.1.as_str()) { + (ri, rj) + } else { + (rj, ri) + }; + grouped.entry((a, b, axiom)).or_default().push((i, j)); + }; + for (i, d) in derivation.facts.iter().enumerate() { + let Some(span) = derived_spans[i] else { + continue; + }; + let Some(ax) = axioms.get(&d.predicate) else { + continue; + }; + let overlapping = |j: usize| derived_spans[j].is_some_and(|s| overlap(span, s).is_some()); + if ax.asymmetric { + if let Some(v) = d_spo.get(&(d.predicate, d.object, d.subject)) { + for &j in v { + if j > i && overlapping(j) { + note(i, j, Kind::Asymmetry); + } + } + } + } + if ax.functional { + if let Some(v) = d_ps.get(&(d.predicate, d.subject)) { + for &j in v { + if j > i && derivation.facts[j].object != d.object && overlapping(j) { + note(i, j, Kind::Functional); + } + } + } + } + if ax.inverse_functional { + if let Some(v) = d_po.get(&(d.predicate, d.object)) { + for &j in v { + if j > i && derivation.facts[j].subject != d.subject && overlapping(j) { + note(i, j, Kind::Functional); + } + } + } + } + } + let mut rule_clashes: Vec = grouped + .into_iter() + .map(|((a, b, axiom), mut pairs)| { + pairs.sort_unstable(); + pairs.dedup(); + RuleClash { a, b, axiom, pairs } + }) + .collect(); + // 输出排过序——这条路的价值有一半在确定性 + rule_clashes.sort_by(|x, y| { + (x.a.0, x.a.1.as_str(), x.b.0, x.b.1.as_str()).cmp(&( + y.a.0, + y.a.1.as_str(), + y.b.0, + y.b.1.as_str(), + )) + }); + out.between_derivations = rule_clashes; + out +} + #[cfg(test)] mod tests { use super::*; @@ -796,4 +1038,163 @@ mod tests { let d = derive(&[ep(P, 1, 1, 1)], &ax); assert!(d.facts.is_empty(), "`A p A` 的逆还是 `A p A`——自环不推"); } + + // ---------- 矛盾（0017） ---------- + + /// 指定谓词的一条带区间的边 + fn et(pred: Uuid, fact: u8, s: u8, o: u8, from: Option, to: Option) -> TimedEdge { + TimedEdge { + edge: Edge { + fact: f(fact), + predicate: pred, + subject: n(s), + object: n(o), + }, + from, + to, + } + } + + fn spans_of(edges: &[TimedEdge]) -> HashMap, Option)> { + edges + .iter() + .map(|e| (e.edge.fact, (e.from, e.to))) + .collect() + } + + /// `ceo_of ⊑ works_at`，works_at functional：Mira 的 ceo_of 推出 works_at Acme， + /// 而账本里说她 works_at Globex——派生撞上断言，指名道姓 + #[test] + fn a_derivation_that_breaks_functional_names_the_assertion_it_hit() { + let ax = HashMap::from([ + ( + P, + Axioms { + sub_property_of: Some(Q), + ..Default::default() + }, + ), + ( + Q, + Axioms { + functional: true, + ..Default::default() + }, + ), + ]); + let edges = [ep(P, 1, 1, 2), ep(Q, 2, 1, 3)]; + let d = derive(&edges, &ax); + assert_eq!(d.facts.len(), 1); + let c = contradictions(&d, &edges, &ax, &spans_of(&edges)); + assert_eq!( + c.with_assertions, + vec![Clash { + derived: 0, + axiom: Kind::Functional, + against: f(2) + }] + ); + assert!(c.between_derivations.is_empty()); + assert_eq!(c.blocked(), HashSet::from([0])); + } + + /// 区间不交就不是矛盾：前任与继任 + #[test] + fn disjoint_intervals_are_succession_and_stay_silent() { + let ax = HashMap::from([ + ( + P, + Axioms { + sub_property_of: Some(Q), + ..Default::default() + }, + ), + ( + Q, + Axioms { + functional: true, + ..Default::default() + }, + ), + ]); + let edges = [ + et(P, 1, 1, 2, Some(10), Some(20)), + et(Q, 2, 1, 3, Some(30), None), + ]; + let d = derive(&edges, &ax); + let c = contradictions(&d, &edges, &ax, &spans_of(&edges)); + assert!(c.with_assertions.is_empty(), "{c:?}"); + } + + /// 对称与非对称：`A p B` 对称推出 `B p A`，而 p 又声明 asymmetric—— + /// 每条断言都撞上自己的镜像 + #[test] + fn a_symmetric_derivation_hits_the_asymmetric_assertion() { + let ax = HashMap::from([( + P, + Axioms { + symmetric: true, + asymmetric: true, + ..Default::default() + }, + )]); + let edges = [ep(P, 1, 1, 2)]; + let d = derive(&edges, &ax); + let c = contradictions(&d, &edges, &ax, &spans_of(&edges)); + assert_eq!(c.with_assertions.len(), 1); + assert_eq!(c.with_assertions[0].axiom, Kind::Asymmetry); + assert_eq!(c.with_assertions[0].against, f(1)); + } + + /// 两条派生互撞时按规则对聚合，而且都不落地 + #[test] + fn derivations_that_disagree_are_grouped_by_the_rules_that_made_them() { + let ax = HashMap::from([ + ( + P, + Axioms { + sub_property_of: Some(Q), + ..Default::default() + }, + ), + ( + Q, + Axioms { + functional: true, + ..Default::default() + }, + ), + ]); + // 1 ceo_of 2 与 1 ceo_of 3：两条 works_at 由同一条规则推出，互相排斥 + let edges = [ep(P, 1, 1, 2), ep(P, 2, 1, 3), ep(P, 3, 4, 5)]; + let d = derive(&edges, &ax); + assert_eq!(d.facts.len(), 3); + let c = contradictions(&d, &edges, &ax, &spans_of(&edges)); + assert!(c.with_assertions.is_empty()); + assert_eq!(c.between_derivations.len(), 1); + let rc = &c.between_derivations[0]; + assert_eq!(rc.a, (P, Rule::SubProperty)); + assert_eq!(rc.b, (P, Rule::SubProperty)); + assert_eq!(rc.axiom, Kind::Functional); + assert_eq!(rc.pairs.len(), 1); + // 第三条（4 works_at 5）没跟谁撞，照常落地 + assert_eq!(c.blocked().len(), 2); + assert!(!c.blocked().contains(&2)); + } + + /// 谓词上没有公理就没有矛盾可言 + #[test] + fn a_predicate_without_axioms_cannot_contradict() { + let ax = HashMap::from([( + P, + Axioms { + sub_property_of: Some(Q), + ..Default::default() + }, + )]); + let edges = [ep(P, 1, 1, 2), ep(Q, 2, 1, 3)]; + let d = derive(&edges, &ax); + let c = contradictions(&d, &edges, &ax, &spans_of(&edges)); + assert_eq!(c, Contradictions::default()); + } } diff --git a/crates/utopia-reason/src/lib.rs b/crates/utopia-reason/src/lib.rs index b12b6528..b6c3ca5c 100644 --- a/crates/utopia-reason/src/lib.rs +++ b/crates/utopia-reason/src/lib.rs @@ -67,7 +67,7 @@ pub struct Violation { pub path: Vec, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Kind { /// `A p A`，而 p 声明了 irreflexive SelfLoop, @@ -85,6 +85,11 @@ pub enum Kind { /// 是为了与其它四类走同一条落库、清陈、裁决的路——left 与 right 同一条事实， /// 与自反那类同款 Signature, + /// 一条派生撞上了一条断言（0017）：推出来的 `A p B` 与账本里的某条断言在 p 的 + /// 公理上不能并存。派生不落地，这一行把它摆到人面前。`left` 是被撞的断言， + /// `right` 是派生的最后一条前提，`path` 是全部前提；推出来的三元组本身在 + /// `axiom_violations.detail` 里——它没有落库，没有 id 可指 + DerivedContradiction, } impl Kind { @@ -95,6 +100,7 @@ impl Kind { Kind::Cycle => "cycle", Kind::Functional => "functional", Kind::Signature => "signature", + Kind::DerivedContradiction => "derived_contradiction", } } } diff --git a/crates/utopia-server/Cargo.toml b/crates/utopia-server/Cargo.toml index ffd2b546..628e03db 100644 --- a/crates/utopia-server/Cargo.toml +++ b/crates/utopia-server/Cargo.toml @@ -41,3 +41,9 @@ async-trait = "0.1.92" sqlparser = "0.62.0" object_store = { version = "0.14.1", features = ["aws", "azure", "gcp"] } quick-xml.workspace = true +url = "2.5.8" +percent-encoding = "2.3.2" +base64 = "0.23.1" + +[dev-dependencies] +wiremock = "0.6.5" diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rs index 2c73d269..f6ef8237 100644 --- a/crates/utopia-server/src/api/chat.rs +++ b/crates/utopia-server/src/api/chat.rs @@ -71,7 +71,7 @@ fn tools_schema(can_write: bool, data_source_names: &[String]) -> serde_json::Va }, "sql": { "type": "string", - "description": "One SELECT/WITH statement (PostgreSQL dialect)." + "description": "One SELECT/WITH statement in the source's own SQL dialect (PostgreSQL, Trino, Databricks or Snowflake; the schema document names the engine)." }, "purpose": { "type": "string", @@ -435,7 +435,7 @@ pub async fn chat( }; if !ds_names.is_empty() { system_prompt.push_str(&format!( - "\nData: query_data runs read-only SQL (PostgreSQL dialect) against: {}. \ + "\nData: query_data runs read-only SQL (in each source's own dialect) against: {}. \ For questions about numbers/metrics, search for the source's schema document \ first, then query. State units and the time range you used in the answer.", ds_names.join(", ") diff --git a/crates/utopia-server/src/api/datasource_routes.rs b/crates/utopia-server/src/api/datasource_routes.rs index 9a3e9e82..8956a4ed 100644 --- a/crates/utopia-server/src/api/datasource_routes.rs +++ b/crates/utopia-server/src/api/datasource_routes.rs @@ -53,10 +53,25 @@ pub async fn create( Json(body): Json, ) -> ApiResult> { require_admin(&user)?; + // 引擎跟着 scheme 走，界面只有一个连接串输入框；body.engine 只为兼容旧调用留着 + let engine = crate::query_engine::engine_from_conn(&body.conn_string).ok_or_else(|| { + utopia_core::AppError::invalid( + "unsupported_conn_scheme", + format!( + "Connection string must start with one of: postgres://, trino://, databricks://, snowflake:// (engines: {})", + crate::query_engine::ENGINES.join(", ") + ), + ) + })?; + let _ = &body.engine; + // 连接串的形状在登记时就校验（缺令牌、缺 warehouse……），错误信息里带写法； + // 否则要等到「测试」才知道，而那一步只回 ok:false + crate::query_engine::engine_for(engine, &body.conn_string) + .map_err(|e| utopia_core::AppError::invalid("bad_conn_string", e.to_string()))?; let id = utopia_store::datasources::create( &state.pool, &body.name, - &body.engine, + engine, &body.conn_string, user.id, ) @@ -228,7 +243,7 @@ async fn sync_schema_doc(state: &AppState, kb_id: Uuid, ds_id: Uuid) -> anyhow:: .await?; let mut md = format!( - "# Data source: {name}\n\nTables and columns available for SQL queries against this source.\n" + "# Data source: {name}\n\nEngine: {engine}. Tables and columns available for SQL queries against this source; write SQL in this engine's dialect.\n" ); let mut current = String::new(); let mut tables = 0usize; diff --git a/crates/utopia-server/src/api/graph_routes.rs b/crates/utopia-server/src/api/graph_routes.rs index 48081132..8962ac27 100644 --- a/crates/utopia-server/src/api/graph_routes.rs +++ b/crates/utopia-server/src/api/graph_routes.rs @@ -145,9 +145,12 @@ pub async fn entity_detail( // 只有先改一次名才够得着——而两个张伟并存是「宁分勿合」的正当产物，不是 // 改名改出来的。合并入口该长在能看见同名的地方。 let same_name = utopia_store::graph::same_name_peers(&state.pool, kb_id, entity_id).await?; + // 没落地的派生（0017 §3）也单独一个键：它们连 `derived_facts` 都不在 + let blocked = + utopia_store::reasoning::blocked_for_entity(&state.pool, kb_id, entity_id).await?; Ok(Json(json!({ "entity": entity, "facts": facts, - "derived": derived, "same_name": same_name, + "derived": derived, "blocked": blocked, "same_name": same_name, }))) } @@ -228,6 +231,30 @@ pub async fn fact_evidence( Ok(Json(json!({ "evidence": evidence }))) } +/// 一条派生事实的证明（0002 R2）：前提按推导顺序，每条带证据，一路到原句。 +/// 派生已失效或不存在时 `proof` 为 null——不是错误，界面据此退回文本前提 +pub async fn derived_proof( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, derived_id)): Path<(Uuid, Uuid)>, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Viewer).await?; + let proof = utopia_store::reasoning::proof(&state.pool, kb_id, derived_id).await?; + Ok(Json(json!({ "proof": proof }))) +} + +/// 没落地的派生的证明链（0017 §3）：前提在那条 `derived_contradiction` 违规的 +/// `path` 里，展开方式与落了地的一样。违规不存在时 `steps` 为 null +pub async fn blocked_proof( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, violation_id)): Path<(Uuid, Uuid)>, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Viewer).await?; + let steps = utopia_store::reasoning::blocked_proof(&state.pool, kb_id, violation_id).await?; + Ok(Json(json!({ "steps": steps }))) +} + /// 手动触发抽取（failed 重试 / 补配模型后补抽）。 pub async fn extract( State(state): State, diff --git a/crates/utopia-server/src/api/mod.rs b/crates/utopia-server/src/api/mod.rs index 8818406b..aa5ff762 100644 --- a/crates/utopia-server/src/api/mod.rs +++ b/crates/utopia-server/src/api/mod.rs @@ -290,6 +290,16 @@ pub fn router(state: AppState, cfg: &AppConfig) -> Router { "/kbs/{id}/facts/{fact_id}/evidence", get(graph_routes::fact_evidence), ) + // 派生事实的证明（0002 R2）：前提按顺序展开到原句 + .route( + "/kbs/{id}/derived/{derived_id}/proof", + get(graph_routes::derived_proof), + ) + // 没落地的派生的证明（0017 §3）：前提在违规的 path 里 + .route( + "/kbs/{id}/violations/{violation_id}/proof", + get(graph_routes::blocked_proof), + ) .route("/kbs/{id}/events", get(events_routes::kb_events)) .route( "/kbs/{id}/sources", diff --git a/crates/utopia-server/src/api/review_routes.rs b/crates/utopia-server/src/api/review_routes.rs index ba1a619e..c6fc951d 100644 --- a/crates/utopia-server/src/api/review_routes.rs +++ b/crates/utopia-server/src/api/review_routes.rs @@ -497,8 +497,11 @@ pub async fn decide_mapping( #[derive(Deserialize)] pub struct DecideViolationReq { - /// fact_retracted | axiom_relaxed | accepted + /// fact_retracted | fact_closed | axiom_relaxed | accepted pub resolution: String, + /// `fact_closed` 必填：旧断言在哪一天结束 + #[serde(default)] + pub close_at: Option>, } /// 人裁决一处公理违规。 @@ -518,16 +521,101 @@ pub async fn decide_violation( require_kb(&state, &user, kb_id, Role::Editor).await?; if !matches!( req.resolution.as_str(), - "fact_retracted" | "axiom_relaxed" | "accepted" + "fact_retracted" | "fact_closed" | "axiom_relaxed" | "accepted" ) { return Err(utopia_core::AppError::invalid( "bad_resolution", - "resolution 只能是 fact_retracted、axiom_relaxed 或 accepted", + "resolution 只能是 fact_retracted、fact_closed、axiom_relaxed 或 accepted", ) .into()); } + let row: Option<(String, Uuid)> = sqlx::query_as( + "SELECT kind, left_fact FROM axiom_violations + WHERE id = $1 AND kb_id = $2 AND status = 'open'", + ) + .bind(violation_id) + .bind(kb_id) + .fetch_optional(&state.pool) + .await + .map_err(utopia_core::AppError::Db)?; + let Some((kind, left)) = row else { + return Err(utopia_core::AppError::NotFound.into()); + }; + // 派生撞断言那一类（0017）的修法就在卡片上，端点替人执行：撤旧断言、或给它一个 + // 结束日期。其它几类仍只记决定——那些卡片上两条都是断言，撤哪条端点判不了 + let repaired = kind == "derived_contradiction"; + match (repaired, req.resolution.as_str()) { + (true, "fact_retracted") => { + let snap = fact_snapshot(&state, kb_id, left).await; + utopia_store::graph::reject_fact(&state.pool, kb_id, left).await?; + if let Some(d) = snap { + let _ = utopia_store::audit::record( + &state.pool, + Some(kb_id), + user.id, + "fact.reject", + "fact", + Some(left), + d, + ) + .await; + } + } + (true, "fact_closed") => { + let Some(at) = req.close_at else { + return Err(utopia_core::AppError::invalid( + "close_at_required", + "fact_closed 要给出结束日期", + ) + .into()); + }; + let open: Option<(Uuid,)> = sqlx::query_as( + "SELECT id FROM facts + WHERE id = $1 AND invalidated_at IS NULL AND valid_to IS NULL", + ) + .bind(left) + .fetch_optional(&state.pool) + .await + .map_err(utopia_core::AppError::Db)?; + if open.is_none() { + return Err(utopia_core::AppError::invalid( + "not_open", + "这条断言已有结束日期，或已被撤", + ) + .into()); + } + let snap = fact_snapshot(&state, kb_id, left).await; + utopia_store::temporal::close_superseded(&state.pool, left, at, "day").await?; + if let Some(mut d) = snap { + d["valid_to"] = json!(at.to_rfc3339()); + let _ = utopia_store::audit::record( + &state.pool, + Some(kb_id), + user.id, + "fact.close", + "fact", + Some(left), + d, + ) + .await; + } + } + (false, "fact_closed") => { + return Err(utopia_core::AppError::invalid( + "bad_resolution", + "fact_closed 只用于 derived_contradiction", + ) + .into()); + } + _ => {} + } utopia_store::reasoning::decide(&state.pool, kb_id, violation_id, &req.resolution, user.id) .await?; + // 路清了就让派生落地，人不必再去点一次「推一遍」。撤与闭合把断言挪开了， + // 认可则在 materialize 里放行 + if repaired { + utopia_store::reasoning::materialize(&state.pool, kb_id).await?; + } let _ = utopia_store::audit::record( &state.pool, Some(kb_id), diff --git a/crates/utopia-server/src/api/sources_routes.rs b/crates/utopia-server/src/api/sources_routes.rs index baaae99f..3889a69e 100644 --- a/crates/utopia-server/src/api/sources_routes.rs +++ b/crates/utopia-server/src/api/sources_routes.rs @@ -6,7 +6,7 @@ use axum::Json; use chrono::{DateTime, Utc}; use serde::Deserialize; use serde_json::json; -use utopia_core::models::Role; +use utopia_core::models::{Role, SOURCE_SECRET_KEYS}; use uuid::Uuid; use super::graph_routes::require_kb; @@ -19,6 +19,23 @@ fn new_ingest_token() -> String { format!("utp_{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()) } +/// 取一条来源，并确认它属于路径上的这个库。 +/// +/// `require_kb` 只查人对库的权限；来源 id 是另一个维度——不比对的话，A 库的 +/// Editor 拿着 B 库来源的 id 就能同步、清理、删除它。不属于就当不存在（404）， +/// 与 `get_token` 一直以来的做法一致 +async fn source_in_kb( + state: &AppState, + kb_id: Uuid, + source_id: Uuid, +) -> ApiResult { + let source = utopia_store::sources::get(&state.pool, source_id).await?; + if source.kb_id != kb_id { + return Err(utopia_core::AppError::NotFound.into()); + } + Ok(source) +} + pub async fn list( State(state): State, AuthUser(user): AuthUser, @@ -102,8 +119,8 @@ pub async fn get_token( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; - let source = utopia_store::sources::get(&state.pool, source_id).await?; - if source.kb_id != kb_id || source.kind != "api" { + let source = source_in_kb(&state, kb_id, source_id).await?; + if source.kind != "api" { return Err(utopia_core::AppError::NotFound.into()); } Ok(Json(json!({ "ingest_token": source.ingest_token }))) @@ -116,8 +133,8 @@ pub async fn rotate_token( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; - let source = utopia_store::sources::get(&state.pool, source_id).await?; - if source.kb_id != kb_id || source.kind != "api" { + let source = source_in_kb(&state, kb_id, source_id).await?; + if source.kind != "api" { return Err(utopia_core::AppError::NotFound.into()); } let token = new_ingest_token(); @@ -125,12 +142,34 @@ pub async fn rotate_token( Ok(Json(json!({ "ingest_token": token }))) } -/// 响应前剔除凭据（auth_header 只进不出）。 -fn mask_secrets(mut source: utopia_core::models::Source) -> utopia_core::models::Source { - if let Some(obj) = source.config.as_object_mut() { - obj.remove("auth_header"); +/// 响应前剔除凭据（只进不出；键见 `SOURCE_SECRET_KEYS`）。 +fn mask_secrets(source: utopia_core::models::Source) -> utopia_core::models::Source { + source.without_secrets() +} + +/// 更新时凭据的合并规则，每个 `SOURCE_SECRET_KEYS` 里的键一样：新配置里**没有**这个键 +/// 或值是空串 → 保留库里的原值（表单留空就是「别动」）；显式 `null` → 删掉； +/// 其余照新值。响应从不回显，所以客户端没有办法把旧值原样送回来，规则只能长在这里 +fn keep_secrets(next: &mut serde_json::Value, existing: &serde_json::Value) { + let Some(obj) = next.as_object_mut() else { + return; + }; + for key in SOURCE_SECRET_KEYS { + let keep = match obj.get(*key) { + None => true, + Some(serde_json::Value::Null) => { + obj.remove(*key); + false + } + Some(v) => v.as_str().is_some_and(|s| s.trim().is_empty()), + }; + if keep { + obj.remove(*key); + if let Some(prev) = existing.get(*key) { + obj.insert((*key).to_string(), prev.clone()); + } + } } - source } #[derive(Deserialize)] @@ -158,23 +197,11 @@ pub async fn update( Json(body): Json, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; - // 凭据只进不出：响应从不回显 auth_header，表单留空 = 保留库里原值 + let existing = source_in_kb(&state, kb_id, source_id).await?; + // 凭据只进不出：响应从不回显，表单留空 / 没传 = 保留库里原值 let mut config = body.config; if let Some(cfg) = config.as_mut() { - let blank = cfg - .get("auth_header") - .and_then(|v| v.as_str()) - .map(str::trim) - .is_none_or(|s| s.is_empty()); - if blank { - if let Some(obj) = cfg.as_object_mut() { - obj.remove("auth_header"); - let existing = utopia_store::sources::get(&state.pool, source_id).await?; - if let Some(prev) = existing.config.get("auth_header").and_then(|v| v.as_str()) { - obj.insert("auth_header".into(), json!(prev)); - } - } - } + keep_secrets(cfg, &existing.config); } let source = utopia_store::sources::update( &state.pool, @@ -187,7 +214,7 @@ pub async fn update( ) .await?; state.emit_source(kb_id); - // 审计不落凭据：config 只记除 auth_header 外的键 + // 审计不落凭据：config 只记「改没改」 let _ = utopia_store::audit::record( &state.pool, Some(kb_id), @@ -208,6 +235,7 @@ pub async fn cleanup_missing( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; + source_in_kb(&state, kb_id, source_id).await?; let ids = utopia_store::documents::list_missing(&state.pool, source_id).await?; for id in &ids { utopia_store::documents::delete(&state.pool, *id).await?; @@ -229,7 +257,7 @@ pub async fn delete( ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; // Memory 来源常驻：记忆空间不因来源整理而蒸发（记忆文档本身可在 Library 删除） - let source = utopia_store::sources::get(&state.pool, source_id).await?; + let source = source_in_kb(&state, kb_id, source_id).await?; if source.kind == utopia_store::memory::MEMORY_SOURCE_KIND { return Err(utopia_core::AppError::invalid( "memory_source_permanent", @@ -259,6 +287,7 @@ pub async fn runs( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Viewer).await?; + source_in_kb(&state, kb_id, source_id).await?; let runs = utopia_store::sources::list_runs(&state.pool, source_id, 20).await?; Ok(Json(json!({ "runs": runs }))) } @@ -269,6 +298,7 @@ pub async fn sync_now( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; + source_in_kb(&state, kb_id, source_id).await?; let queued = utopia_store::sources::mark_queued(&state.pool, source_id).await?; if queued { utopia_store::jobs::enqueue( @@ -505,3 +535,60 @@ pub async fn re_extract( .await; Ok(Json(json!({ "queued": ids.len() }))) } + +#[cfg(test)] +mod tests { + use super::keep_secrets; + use serde_json::json; + + #[test] + fn a_blank_or_missing_secret_keeps_the_stored_one() { + let existing = json!({ "bucket": "old", "secret_access_key": "s", "password": "p" }); + // 没传 → 留；空串 → 留；有值 → 换；null → 删 + let mut next = json!({ "bucket": "new", "password": " ", "token": null }); + keep_secrets(&mut next, &existing); + assert_eq!(next["bucket"], "new"); + assert_eq!( + next["secret_access_key"], "s", + "missing keeps the stored value" + ); + assert_eq!(next["password"], "p", "blank keeps the stored value"); + assert!(next.get("token").is_none(), "an explicit null removes it"); + let mut next = json!({ "secret_access_key": "fresh" }); + keep_secrets(&mut next, &existing); + assert_eq!(next["secret_access_key"], "fresh"); + assert_eq!(next["password"], "p"); + } + + #[test] + fn no_secret_reaches_a_response() { + let source = utopia_core::models::Source { + id: uuid::Uuid::nil(), + kb_id: uuid::Uuid::nil(), + kind: "s3".into(), + name: "s".into(), + config: json!({ "bucket": "b", "access_key_id": "AKIA", "secret_access_key": "x", + "account_key": "y", "service_account_key": "z", "password": "w", + "token": "t", "auth_header": "h" }), + icon: None, + sync_interval_minutes: None, + sync_cron: None, + last_sync_at: None, + last_sync_status: "never".into(), + last_sync_error: None, + last_sync_added: 0, + ingest_token: Some("utp_x".into()), + created_at: chrono::Utc::now(), + }; + let masked = super::mask_secrets(source); + let obj = masked.config.as_object().unwrap(); + for key in utopia_core::models::SOURCE_SECRET_KEYS { + assert!(!obj.contains_key(*key), "{key} leaked"); + } + assert_eq!(obj["bucket"], "b"); + assert_eq!( + obj["access_key_id"], "AKIA", + "an identifier is not a secret" + ); + } +} diff --git a/crates/utopia-server/src/api/tools.rs b/crates/utopia-server/src/api/tools.rs index 22c41d9f..98a52a2c 100644 --- a/crates/utopia-server/src/api/tools.rs +++ b/crates/utopia-server/src/api/tools.rs @@ -410,8 +410,9 @@ pub(super) fn charter_source_json(n: usize, h: &utopia_search::DocsSection) -> s /// 问数执行：安全闸（解析白名单）→ 引擎执行（只读会话 + 强制 LIMIT + 超时）→ JSON 行。 async fn run_query(state: &AppState, ds_id: Uuid, sql: &str) -> anyhow::Result { - let guarded = crate::query_engine::guard_sql(sql)?; let (engine, conn) = utopia_store::datasources::engine_and_conn(&state.pool, ds_id).await?; + // 闸门按引擎选方言：Databricks 的反引号、Snowflake 的 :: 转型都得先过得了解析 + let guarded = crate::query_engine::guard_sql_for(&engine, sql)?; let result = crate::query_engine::engine_for(&engine, &conn)? .execute(&guarded) .await?; diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs index 56afd33c..a1803d48 100644 --- a/crates/utopia-server/src/ingest_sources.rs +++ b/crates/utopia-server/src/ingest_sources.rs @@ -8,6 +8,7 @@ use crate::state::AppState; use chrono::{DateTime, Utc}; use sha2::{Digest, Sha256}; use utopia_core::models::Source; +use utopia_core::models::SourceKind; use uuid::Uuid; /// 单次同步的新文档上限（防超长 feed/URL 列表拖垮任务） @@ -48,17 +49,23 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<() let run_id = utopia_store::sources::start_run(&state.pool, source_id).await?; state.emit_source(source.kb_id); - let outcome = match source.kind.as_str() { - "url" => sync_urls(state, &source).await, - "rss" => sync_rss(state, &source).await, - "custom" => sync_custom(state, &source).await, - "github_issues" => sync_github_issues(state, &source).await, - "jira_issues" => sync_jira_issues(state, &source).await, - "s3" | "azure_blob" | "gcs" => sync_object_storage(state, &source).await, - "webdav" => sync_webdav(state, &source).await, - "notion" => sync_notion(state, &source).await, - // folder / api 无拉取语义 - _ => Ok(SyncStats::default()), + // 按枚举穷举：加一种来源就得在这里决定它怎么同步，编译器不放过漏掉的那一支 + let outcome = match SourceKind::parse(&source.kind) { + Some(SourceKind::Url) => sync_urls(state, &source).await, + Some(SourceKind::Rss) => sync_rss(state, &source).await, + Some(SourceKind::Custom) => sync_custom(state, &source).await, + Some(SourceKind::GithubIssues) => sync_github_issues(state, &source).await, + Some(SourceKind::JiraIssues) => sync_jira_issues(state, &source).await, + Some(SourceKind::S3 | SourceKind::AzureBlob | SourceKind::Gcs) => { + sync_object_storage(state, &source).await + } + Some(SourceKind::Webdav) => sync_webdav(state, &source).await, + Some(SourceKind::Notion) => sync_notion(state, &source).await, + // 被动容器：folder / api / memory / upload 没有拉取语义 + Some(SourceKind::Folder | SourceKind::Api | SourceKind::Memory | SourceKind::Upload) => { + Ok(SyncStats::default()) + } + None => Err(anyhow::anyhow!("unknown source kind `{}`", source.kind)), }; match outcome { diff --git a/crates/utopia-server/src/query_engine.rs b/crates/utopia-server/src/query_engine.rs deleted file mode 100644 index 01a015bc..00000000 --- a/crates/utopia-server/src/query_engine.rs +++ /dev/null @@ -1,205 +0,0 @@ -//! 问数查询引擎：trait 接缝（BlobStore 同手法）+ 引擎无关的安全闸。 -//! -//! 引擎按协议族扩，不按产品名扩：postgres（本文件）→ mysql 线协议族（白捡 -//! TiDB/OceanBase/Doris/StarRocks）→ HTTP 族（ClickHouse、Trino——后者一个顶起 -//! Iceberg/Delta/Hive 整个湖仓生态）。挂载模型与注册表引擎无关，加引擎零迁移。 -//! -//! 安全闸（纵深防御，不信任模型）： -//! 1. sqlparser 解析：仅放行单条 SELECT/WITH（含 CTE），拒绝 DML/DDL/多语句/SELECT INTO -//! 2. 强制外包一层 LIMIT（cap+1 探测截断） -//! 3. 会话级只读 + 语句超时（引擎各自的机制，parser 万一漏网也写不进去） -//! 4. 结果统一为 JSON Lines（各引擎都有原生 JSON 行输出，也是模型最好消化的格式） - -use sqlparser::ast::Statement; -use sqlparser::dialect::PostgreSqlDialect; -use sqlparser::parser::Parser; -use sqlx::postgres::PgPoolOptions; -use sqlx::Row; -use std::time::Duration; - -/// 行数上限（外包 LIMIT cap+1，第 201 行只用来判断截断）。 -pub const ROW_CAP: usize = 200; -const STATEMENT_TIMEOUT_SECS: u32 = 10; - -pub struct QueryResult { - /// 每行一个 JSON 对象文本（键序 = 查询列序） - pub rows: Vec, - pub truncated: bool, -} - -#[derive(Debug)] -pub struct SchemaColumn { - pub schema: String, - pub table: String, - pub column: String, - pub data_type: String, - pub comment: Option, -} - -#[async_trait::async_trait] -pub trait QueryEngine: Send + Sync { - async fn test(&self) -> anyhow::Result<()>; - async fn fetch_schema(&self) -> anyhow::Result>; - /// 执行已过闸的 SELECT。实现自身仍需强制只读会话与超时（纵深防御）。 - async fn execute(&self, sql: &str) -> anyhow::Result; -} - -/// 引擎工厂。conn 凭据只在服务端流转。 -pub fn engine_for(engine: &str, conn: &str) -> anyhow::Result> { - match engine { - "postgres" => Ok(Box::new(PostgresEngine { - conn: conn.to_string(), - })), - other => anyhow::bail!("Unsupported engine: {other}"), - } -} - -/// 安全闸第 1 层：解析并校验，返回规整后的语句文本。 -pub fn guard_sql(sql: &str) -> anyhow::Result { - let cleaned = sql.trim().trim_end_matches(';').trim(); - if cleaned.is_empty() { - anyhow::bail!("Empty SQL"); - } - let statements = Parser::parse_sql(&PostgreSqlDialect {}, cleaned) - .map_err(|e| anyhow::anyhow!("SQL parse error: {e}"))?; - if statements.len() != 1 { - anyhow::bail!("Exactly one statement is allowed"); - } - match &statements[0] { - Statement::Query(_) => Ok(cleaned.to_string()), - other => anyhow::bail!( - "Read-only: only SELECT/WITH queries are allowed (got {})", - statement_kind(other) - ), - } -} - -fn statement_kind(s: &Statement) -> &'static str { - match s { - Statement::Insert { .. } => "INSERT", - Statement::Update { .. } => "UPDATE", - Statement::Delete { .. } => "DELETE", - Statement::Drop { .. } => "DROP", - Statement::CreateTable { .. } | Statement::CreateView { .. } => "CREATE", - Statement::AlterTable { .. } => "ALTER", - Statement::Truncate { .. } => "TRUNCATE", - Statement::Copy { .. } => "COPY", - _ => "a non-SELECT statement", - } -} - -// --------------------------------------------------------------------------- -// Postgres 族（顺带覆盖 Greenplum/Timescale 等 PG 兼容系） -// --------------------------------------------------------------------------- - -pub struct PostgresEngine { - conn: String, -} - -impl PostgresEngine { - async fn pool(&self) -> anyhow::Result { - Ok(PgPoolOptions::new() - .max_connections(1) - .acquire_timeout(Duration::from_secs(5)) - .connect(&self.conn) - .await?) - } -} - -#[async_trait::async_trait] -impl QueryEngine for PostgresEngine { - async fn test(&self) -> anyhow::Result<()> { - let pool = self.pool().await?; - sqlx::query("SELECT 1").execute(&pool).await?; - pool.close().await; - Ok(()) - } - - async fn fetch_schema(&self) -> anyhow::Result> { - let pool = self.pool().await?; - let rows: Vec<(String, String, String, String, Option)> = sqlx::query_as( - "SELECT c.table_schema, c.table_name, c.column_name, - c.data_type, pgd.description - FROM information_schema.columns c - LEFT JOIN pg_catalog.pg_statio_all_tables st - ON st.schemaname = c.table_schema AND st.relname = c.table_name - LEFT JOIN pg_catalog.pg_description pgd - ON pgd.objoid = st.relid AND pgd.objsubid = c.ordinal_position - WHERE c.table_schema NOT IN ('pg_catalog', 'information_schema') - ORDER BY c.table_schema, c.table_name, c.ordinal_position", - ) - .fetch_all(&pool) - .await?; - pool.close().await; - Ok(rows - .into_iter() - .map(|(schema, table, column, data_type, comment)| SchemaColumn { - schema, - table, - column, - data_type, - comment, - }) - .collect()) - } - - async fn execute(&self, sql: &str) -> anyhow::Result { - let pool = self.pool().await?; - // 纵深防御第 3 层：会话级只读 + 超时（parser 漏网也写不进去、跑不死库） - sqlx::query("SET default_transaction_read_only = on") - .execute(&pool) - .await?; - sqlx::query(&format!( - "SET statement_timeout = '{STATEMENT_TIMEOUT_SECS}s'" - )) - .execute(&pool) - .await?; - // 第 2 层：外包 LIMIT；row_to_json 让 PG 全权处理类型→JSON（文本键序保留列序） - let wrapped = format!( - "SELECT row_to_json(_q)::text AS _j FROM ( {sql} ) AS _q LIMIT {}", - ROW_CAP + 1 - ); - let fetched = sqlx::query(&wrapped).fetch_all(&pool).await?; - pool.close().await; - - let truncated = fetched.len() > ROW_CAP; - let rows = fetched - .into_iter() - .take(ROW_CAP) - .map(|r| r.try_get::("_j").unwrap_or_else(|_| "{}".into())) - .collect(); - Ok(QueryResult { rows, truncated }) - } -} - -#[cfg(test)] -mod tests { - use super::guard_sql; - - #[test] - fn allows_select_and_cte() { - assert!(guard_sql("SELECT region, sum(amount) FROM orders GROUP BY 1").is_ok()); - assert!(guard_sql("WITH t AS (SELECT 1 AS x) SELECT * FROM t;").is_ok()); - } - - #[test] - fn rejects_writes_and_ddl() { - for bad in [ - "UPDATE orders SET amount = 0", - "DELETE FROM orders", - "INSERT INTO orders (region) VALUES ('east')", - "DROP TABLE orders", - "TRUNCATE orders", - "CREATE TABLE t (id int)", - "ALTER TABLE orders ADD COLUMN x int", - ] { - assert!(guard_sql(bad).is_err(), "should reject: {bad}"); - } - } - - #[test] - fn rejects_multi_statement() { - assert!(guard_sql("SELECT 1; DROP TABLE orders").is_err()); - assert!(guard_sql("").is_err()); - } -} diff --git a/crates/utopia-server/src/query_engine/conn.rs b/crates/utopia-server/src/query_engine/conn.rs new file mode 100644 index 00000000..1d3cd519 --- /dev/null +++ b/crates/utopia-server/src/query_engine/conn.rs @@ -0,0 +1,253 @@ +//! 连接串解析。一个输入框、四种 scheme；这里把 URL 拆成各引擎要的字段。 +//! +//! 写法沿用 `postgres://user:pass@host/db` 的形状：凭据在 userinfo 里，HTTP 族的 +//! 令牌放 password 位（`databricks://:TOKEN@…`），路径是「目录 / 库 / schema」， +//! 引擎特有的开关走 query。`ssl=false` 让 HTTP 族走明文——给本地代理与测试用， +//! 线上的三家都只认 https。 + +use percent_encoding::percent_decode_str; +use url::Url; + +fn decode(s: &str) -> String { + percent_decode_str(s).decode_utf8_lossy().into_owned() +} + +fn query(u: &Url, key: &str) -> Option { + u.query_pairs() + .find(|(k, _)| k == key) + .map(|(_, v)| v.into_owned()) + .filter(|v| !v.is_empty()) +} + +fn ssl_off(u: &Url) -> bool { + matches!( + query(u, "ssl").as_deref(), + Some("false") | Some("0") | Some("off") | Some("no") + ) +} + +fn segments(u: &Url) -> Vec { + u.path_segments() + .map(|s| s.filter(|x| !x.is_empty()).map(decode).collect()) + .unwrap_or_default() +} + +/// 令牌：password 位优先；没有 password 时 username 位也算（`databricks://TOKEN@host` +/// 少打一个冒号是最常见的手滑）；最后看 `?token=` +fn token_of(u: &Url) -> Option { + u.password() + .map(decode) + .filter(|s| !s.is_empty()) + .or_else(|| Some(decode(u.username())).filter(|s| !s.is_empty())) + .or_else(|| query(u, "token")) +} + +fn base_of(u: &Url, https: bool, default_port: u16) -> anyhow::Result { + let host = u + .host_str() + .ok_or_else(|| anyhow::anyhow!("{}://: a host is required", u.scheme()))?; + let port = u.port().unwrap_or(default_port); + let scheme = if https { "https" } else { "http" }; + // 默认端口不写进 URL：reqwest 照样能连，日志里也干净 + let explicit = match (https, port) { + (true, 443) | (false, 80) => String::new(), + _ => format!(":{port}"), + }; + Ok(format!("{scheme}://{host}{explicit}")) +} + +/// `trino://user[:password]@host[:port]/[catalog[/schema]][?ssl=true|false]` +/// +/// 明文 http 是 Trino 的默认（8080）；带密码、`ssl=true`、或端口 443 / 8443 时走 https—— +/// Trino 自己也拒绝在明文上收密码。`presto://` 是同一个协议的旧名。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrinoConn { + pub base: String, + pub user: String, + pub password: Option, + pub catalog: Option, + pub schema: Option, +} + +impl TrinoConn { + pub fn parse(conn: &str) -> anyhow::Result { + let u = Url::parse(conn.trim())?; + let user = decode(u.username()); + if user.is_empty() { + anyhow::bail!("trino://: a user is required (it becomes X-Trino-User), e.g. trino://alice@host:8080/hive/default"); + } + let password = u.password().map(decode).filter(|s| !s.is_empty()); + let https = !ssl_off(&u) + && (password.is_some() + || query(&u, "ssl").as_deref() == Some("true") + || matches!(u.port(), Some(443) | Some(8443))); + let base = base_of(&u, https, if https { 443 } else { 8080 })?; + let segs = segments(&u); + Ok(Self { + base, + user, + password, + catalog: segs.first().cloned(), + schema: segs.get(1).cloned(), + }) + } +} + +/// `databricks://:TOKEN@workspace-host/sql/1.0/warehouses/WAREHOUSE_ID[?catalog=main&schema=default]` +/// +/// 路径就是 JDBC 里的 httpPath，从控制台复制过来不用改；`?warehouse=ID` 也认。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DatabricksConn { + pub base: String, + pub token: String, + pub warehouse_id: String, + pub catalog: Option, + pub schema: Option, +} + +impl DatabricksConn { + pub fn parse(conn: &str) -> anyhow::Result { + let u = Url::parse(conn.trim())?; + let base = base_of(&u, !ssl_off(&u), if ssl_off(&u) { 80 } else { 443 })?; + let token = token_of(&u).ok_or_else(|| { + anyhow::anyhow!("databricks://: a personal access token is required, e.g. databricks://:TOKEN@host/sql/1.0/warehouses/ID") + })?; + let segs = segments(&u); + let from_path = segs + .iter() + .position(|s| s == "warehouses") + .and_then(|i| segs.get(i + 1).cloned()); + let warehouse_id = query(&u, "warehouse").or(from_path).ok_or_else(|| { + anyhow::anyhow!("databricks://: a SQL warehouse is required — the /sql/1.0/warehouses/ID path or ?warehouse=ID") + })?; + Ok(Self { + base, + token, + warehouse_id, + catalog: query(&u, "catalog"), + schema: query(&u, "schema"), + }) + } +} + +/// `snowflake://:TOKEN@account.snowflakecomputing.com/[DATABASE[/SCHEMA]][?warehouse=WH&role=R&token_type=pat|oauth]` +/// +/// SQL API 不收密码，只收令牌：默认当作 programmatic access token，`token_type=oauth` +/// 换成 OAuth 令牌。密钥对 JWT 要本地签名，这一版不做。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnowflakeConn { + pub base: String, + pub token: String, + /// `X-Snowflake-Authorization-Token-Type` 的值 + pub token_type: &'static str, + pub database: Option, + pub schema: Option, + pub warehouse: Option, + pub role: Option, +} + +impl SnowflakeConn { + pub fn parse(conn: &str) -> anyhow::Result { + let u = Url::parse(conn.trim())?; + let base = base_of(&u, !ssl_off(&u), if ssl_off(&u) { 80 } else { 443 })?; + let token = token_of(&u).ok_or_else(|| { + anyhow::anyhow!("snowflake://: a programmatic access token or OAuth token is required, e.g. snowflake://:TOKEN@account.snowflakecomputing.com/DB/SCHEMA?warehouse=WH") + })?; + let token_type = match query(&u, "token_type") + .as_deref() + .map(str::to_ascii_lowercase) + .as_deref() + { + None | Some("pat") | Some("programmatic_access_token") => "PROGRAMMATIC_ACCESS_TOKEN", + Some("oauth") => "OAUTH", + Some(other) => { + anyhow::bail!("snowflake://: unknown token_type '{other}' (pat or oauth)") + } + }; + let segs = segments(&u); + Ok(Self { + base, + token, + token_type, + database: segs.first().cloned(), + schema: segs.get(1).cloned(), + warehouse: query(&u, "warehouse"), + role: query(&u, "role"), + }) + } +} + +#[cfg(test)] +mod tests { + use super::{DatabricksConn, SnowflakeConn, TrinoConn}; + + #[test] + fn trino_defaults_to_plain_http_and_upgrades_when_it_must() { + let c = TrinoConn::parse("trino://alice@lake.internal:8080/iceberg/sales").unwrap(); + assert_eq!(c.base, "http://lake.internal:8080"); + assert_eq!(c.user, "alice"); + assert_eq!(c.catalog.as_deref(), Some("iceberg")); + assert_eq!(c.schema.as_deref(), Some("sales")); + + let c = TrinoConn::parse("trino://alice:s%40cret@lake.internal/hive").unwrap(); + assert_eq!(c.base, "https://lake.internal"); + assert_eq!(c.password.as_deref(), Some("s@cret")); + + let c = TrinoConn::parse("trino://alice@lake.internal:8443/hive").unwrap(); + assert_eq!(c.base, "https://lake.internal:8443"); + + let c = TrinoConn::parse("presto://bob@127.0.0.1:9000?ssl=false").unwrap(); + assert_eq!(c.base, "http://127.0.0.1:9000"); + assert_eq!(c.catalog, None); + + assert!(TrinoConn::parse("trino://lake.internal/hive").is_err()); + } + + #[test] + fn databricks_reads_the_http_path_from_the_console() { + let c = DatabricksConn::parse( + "databricks://:dapi123@dbc-abc.cloud.databricks.com/sql/1.0/warehouses/9f2a?catalog=main&schema=sales", + ) + .unwrap(); + assert_eq!(c.base, "https://dbc-abc.cloud.databricks.com"); + assert_eq!(c.token, "dapi123"); + assert_eq!(c.warehouse_id, "9f2a"); + assert_eq!(c.catalog.as_deref(), Some("main")); + assert_eq!(c.schema.as_deref(), Some("sales")); + + let c = DatabricksConn::parse("databricks://dapi123@host?warehouse=w1").unwrap(); + assert_eq!(c.token, "dapi123"); + assert_eq!(c.warehouse_id, "w1"); + + assert!(DatabricksConn::parse("databricks://host/sql/1.0/warehouses/w1").is_err()); + assert!(DatabricksConn::parse("databricks://:t@host").is_err()); + } + + #[test] + fn snowflake_takes_a_token_and_the_session_knobs() { + let c = SnowflakeConn::parse( + "snowflake://:tok@xy12345.eu-central-1.snowflakecomputing.com/ANALYTICS/PUBLIC?warehouse=WH&role=ANALYST", + ) + .unwrap(); + assert_eq!( + c.base, + "https://xy12345.eu-central-1.snowflakecomputing.com" + ); + assert_eq!(c.token, "tok"); + assert_eq!(c.token_type, "PROGRAMMATIC_ACCESS_TOKEN"); + assert_eq!(c.database.as_deref(), Some("ANALYTICS")); + assert_eq!(c.schema.as_deref(), Some("PUBLIC")); + assert_eq!(c.warehouse.as_deref(), Some("WH")); + assert_eq!(c.role.as_deref(), Some("ANALYST")); + + let c = + SnowflakeConn::parse("snowflake://:tok@acct.snowflakecomputing.com?token_type=oauth") + .unwrap(); + assert_eq!(c.token_type, "OAUTH"); + assert!(SnowflakeConn::parse( + "snowflake://:tok@acct.snowflakecomputing.com?token_type=jwt" + ) + .is_err()); + assert!(SnowflakeConn::parse("snowflake://acct.snowflakecomputing.com/DB").is_err()); + } +} diff --git a/crates/utopia-server/src/query_engine/databricks.rs b/crates/utopia-server/src/query_engine/databricks.rs new file mode 100644 index 00000000..7f9eb379 --- /dev/null +++ b/crates/utopia-server/src/query_engine/databricks.rs @@ -0,0 +1,271 @@ +//! Databricks SQL Statement Execution API（`/api/2.0/sql/statements`）。 +//! 一个 SQL warehouse 后面是 Unity Catalog 的整个湖仓（Delta 为主）， +//! 令牌是 personal access token。结果要 INLINE + JSON_ARRAY：值全是字符串， +//! 按 manifest 里的列类型还原成数与布尔。 + +use super::conn::DatabricksConn; +use super::{ + coerce, rows_to_json_lines, sql_literal, truncate_rows, wrap_limit, QueryEngine, QueryResult, + SchemaColumn, HTTP_POLL_BUDGET, ROW_CAP, +}; +use serde::Deserialize; +use serde_json::json; +use std::time::{Duration, Instant}; + +pub struct DatabricksEngine { + conn: DatabricksConn, +} + +#[derive(Deserialize)] +struct StatementResponse { + statement_id: Option, + status: Status, + manifest: Option, + result: Option, +} + +#[derive(Deserialize)] +struct Status { + state: String, + error: Option, +} + +#[derive(Deserialize)] +struct StatusError { + message: Option, + error_code: Option, +} + +#[derive(Deserialize)] +struct Manifest { + schema: Option, +} + +#[derive(Deserialize)] +struct Schema { + columns: Vec, +} + +#[derive(Deserialize)] +struct ColumnInfo { + name: String, + type_text: Option, +} + +#[derive(Deserialize)] +struct ResultData { + data_array: Option>>, +} + +impl DatabricksEngine { + pub fn new(conn: DatabricksConn) -> Self { + Self { conn } + } + + async fn run(&self, sql: &str) -> anyhow::Result<(Vec, Vec>)> { + let client = super::http()?; + let mut body = json!({ + "warehouse_id": self.conn.warehouse_id, + "statement": sql, + "wait_timeout": "30s", + "on_wait_timeout": "CONTINUE", + "disposition": "INLINE", + "format": "JSON_ARRAY", + "row_limit": ROW_CAP + 1, + }); + if let Some(c) = &self.conn.catalog { + body["catalog"] = json!(c); + } + if let Some(s) = &self.conn.schema { + body["schema"] = json!(s); + } + let mut resp: StatementResponse = client + .post(format!("{}/api/2.0/sql/statements", self.conn.base)) + .bearer_auth(&self.conn.token) + .json(&body) + .send() + .await? + .error_for_status()? + .json() + .await?; + let started = Instant::now(); + loop { + match resp.status.state.as_str() { + "SUCCEEDED" => break, + "PENDING" | "RUNNING" => { + let id = resp + .statement_id + .clone() + .ok_or_else(|| anyhow::anyhow!("Databricks returned no statement_id"))?; + if started.elapsed() > HTTP_POLL_BUDGET { + anyhow::bail!( + "Databricks statement did not finish within {}s", + HTTP_POLL_BUDGET.as_secs() + ); + } + tokio::time::sleep(Duration::from_secs(1)).await; + resp = client + .get(format!("{}/api/2.0/sql/statements/{id}", self.conn.base)) + .bearer_auth(&self.conn.token) + .send() + .await? + .error_for_status()? + .json() + .await?; + } + other => { + let e = resp.status.error.as_ref(); + let code = e + .and_then(|e| e.error_code.clone()) + .map(|c| format!("{c}: ")) + .unwrap_or_default(); + let msg = e + .and_then(|e| e.message.clone()) + .unwrap_or_else(|| format!("statement ended in state {other}")); + anyhow::bail!("{code}{msg}"); + } + } + } + let columns: Vec = resp + .manifest + .and_then(|m| m.schema) + .map(|s| s.columns) + .unwrap_or_default(); + let raw_rows = resp.result.and_then(|r| r.data_array).unwrap_or_default(); + let rows = raw_rows + .into_iter() + .map(|row| { + row.iter() + .enumerate() + .map(|(i, v)| { + let ty = columns + .get(i) + .and_then(|c| c.type_text.as_deref()) + .unwrap_or(""); + coerce(ty, v) + }) + .collect() + }) + .collect(); + Ok((columns.into_iter().map(|c| c.name).collect(), rows)) + } +} + +#[async_trait::async_trait] +impl QueryEngine for DatabricksEngine { + async fn test(&self) -> anyhow::Result<()> { + self.run("SELECT 1").await.map(|_| ()) + } + + async fn fetch_schema(&self) -> anyhow::Result> { + // 带 catalog 就查那个 catalog 的 information_schema；不带就是会话默认的 + let prefix = self + .conn + .catalog + .as_deref() + .map(|c| format!("`{}`.", c.replace('`', "``"))) + .unwrap_or_default(); + let schema_filter = self + .conn + .schema + .as_deref() + .map(|s| format!(" AND table_schema = {}", sql_literal(s))) + .unwrap_or_default(); + let sql = format!( + "SELECT table_schema, table_name, column_name, data_type, comment \ + FROM {prefix}information_schema.columns \ + WHERE table_schema <> 'information_schema'{schema_filter} \ + ORDER BY table_schema, table_name, ordinal_position" + ); + let (_, rows) = self.run(&sql).await?; + Ok(rows.into_iter().map(super::trino::schema_row).collect()) + } + + async fn execute(&self, sql: &str) -> anyhow::Result { + let (columns, rows) = self.run(&wrap_limit(sql)).await?; + let (rows, truncated) = truncate_rows(rows); + Ok(QueryResult { + rows: rows_to_json_lines(&columns, &rows), + truncated, + }) + } +} + +#[cfg(test)] +mod tests { + use super::super::conn::DatabricksConn; + use super::super::QueryEngine; + use super::DatabricksEngine; + use serde_json::json; + use wiremock::matchers::{header, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn conn(server: &MockServer) -> DatabricksConn { + DatabricksConn::parse(&format!( + "databricks://:dapi-test@{}/sql/1.0/warehouses/wh1?catalog=main&ssl=false", + server.uri().trim_start_matches("http://") + )) + .unwrap() + } + + #[tokio::test] + async fn polls_until_succeeded_and_restores_types() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/2.0/sql/statements")) + .and(header("authorization", "Bearer dapi-test")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "statement_id": "s1", + "status": { "state": "PENDING" } + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/api/2.0/sql/statements/s1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "statement_id": "s1", + "status": { "state": "SUCCEEDED" }, + "manifest": { "schema": { "columns": [ + { "name": "region", "type_text": "STRING", "position": 0 }, + { "name": "total", "type_text": "DECIMAL(12,2)", "position": 1 }, + { "name": "active", "type_text": "BOOLEAN", "position": 2 } + ] } }, + "result": { "data_array": [ ["east", "12.50", "true"], ["west", null, "false"] ] } + }))) + .expect(1) + .mount(&server) + .await; + + let out = DatabricksEngine::new(conn(&server)) + .execute("SELECT region, total, active FROM orders") + .await + .unwrap(); + assert_eq!( + out.rows, + vec![ + r#"{"region":"east","total":12.5,"active":true}"#, + r#"{"region":"west","total":null,"active":false}"# + ] + ); + } + + #[tokio::test] + async fn a_failed_statement_reports_the_message() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/2.0/sql/statements")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "statement_id": "s2", + "status": { "state": "FAILED", "error": { "error_code": "BAD_REQUEST", "message": "TABLE_OR_VIEW_NOT_FOUND: nope" } } + }))) + .mount(&server) + .await; + let err = DatabricksEngine::new(conn(&server)) + .execute("SELECT * FROM nope") + .await + .unwrap_err() + .to_string(); + assert!(err.contains("TABLE_OR_VIEW_NOT_FOUND"), "{err}"); + } +} diff --git a/crates/utopia-server/src/query_engine/mod.rs b/crates/utopia-server/src/query_engine/mod.rs new file mode 100644 index 00000000..2bd17132 --- /dev/null +++ b/crates/utopia-server/src/query_engine/mod.rs @@ -0,0 +1,349 @@ +//! 问数查询引擎：trait 接缝（BlobStore 同手法）+ 引擎无关的安全闸。 +//! +//! 引擎按协议族扩，不按产品名扩：postgres 线协议（`postgres.rs`）→ HTTP 族—— +//! `trino.rs` 一个顶起 Iceberg / Delta / Hive 整个湖仓生态，`databricks.rs`、 +//! `snowflake.rs` 各走自家的 SQL REST API。挂载模型与注册表引擎无关，加引擎只放宽 +//! 一条 CHECK。连接串是唯一的输入：引擎由 scheme 决定（[`engine_from_conn`]）， +//! 剩下的部分各引擎自己拆（`conn.rs`），凭据只在服务端流转。 +//! +//! 安全闸（纵深防御，不信任模型）： +//! 1. sqlparser 解析：仅放行单条 SELECT/WITH（含 CTE），拒绝 DML/DDL/多语句/SELECT INTO。 +//! 按引擎选方言；sqlparser 没有 Trino 方言，Generic 是它的超集 +//! 2. 强制外包一层 LIMIT（cap+1 探测截断） +//! 3. 会话级只读 + 语句超时（引擎各自的机制，parser 万一漏网也写不进去）。 +//! HTTP 族没有会话，只有语句超时——只读靠第 1 层，这是它们比线协议少的那一层 +//! 4. 结果统一为 JSON Lines：PG 让库自己转；HTTP 族拿到列名与值后在这里拼，列序保留 + +mod conn; +mod databricks; +mod postgres; +mod snowflake; +mod trino; + +use sqlparser::ast::Statement; +use sqlparser::dialect::{DatabricksDialect, GenericDialect, PostgreSqlDialect, SnowflakeDialect}; +use sqlparser::parser::Parser; +use std::time::Duration; + +/// 行数上限（外包 LIMIT cap+1，第 201 行只用来判断截断）。 +pub const ROW_CAP: usize = 200; +pub(crate) const STATEMENT_TIMEOUT_SECS: u32 = 10; +/// HTTP 族：单次请求的超时，与整条语句从提交到拿完结果的轮询预算 +pub(crate) const HTTP_REQUEST_TIMEOUT: Duration = Duration::from_secs(20); +pub(crate) const HTTP_POLL_BUDGET: Duration = Duration::from_secs(30); + +/// 注册表里 `engine` 列的取值。迁移里的 CHECK 与这张表要一致 +pub const ENGINES: &[&str] = &["postgres", "trino", "databricks", "snowflake"]; + +#[derive(Debug)] +pub struct QueryResult { + /// 每行一个 JSON 对象文本（键序 = 查询列序） + pub rows: Vec, + pub truncated: bool, +} + +#[derive(Debug)] +pub struct SchemaColumn { + pub schema: String, + pub table: String, + pub column: String, + pub data_type: String, + pub comment: Option, +} + +#[async_trait::async_trait] +pub trait QueryEngine: Send + Sync { + async fn test(&self) -> anyhow::Result<()>; + async fn fetch_schema(&self) -> anyhow::Result>; + /// 执行已过闸的 SELECT。实现自身仍需强制只读会话与超时（纵深防御）。 + async fn execute(&self, sql: &str) -> anyhow::Result; +} + +/// scheme → 引擎名。界面只有一个连接串输入框，这里是它唯一的分派点。 +pub fn engine_from_conn(conn: &str) -> Option<&'static str> { + let scheme = conn.trim().split("://").next()?.to_ascii_lowercase(); + match scheme.as_str() { + "postgres" | "postgresql" => Some("postgres"), + "trino" | "presto" => Some("trino"), + "databricks" => Some("databricks"), + "snowflake" => Some("snowflake"), + _ => None, + } +} + +/// 引擎工厂。conn 凭据只在服务端流转。 +pub fn engine_for(engine: &str, conn: &str) -> anyhow::Result> { + match engine { + "postgres" => Ok(Box::new(postgres::PostgresEngine::new(conn))), + "trino" => Ok(Box::new(trino::TrinoEngine::new(conn::TrinoConn::parse( + conn, + )?))), + "databricks" => Ok(Box::new(databricks::DatabricksEngine::new( + conn::DatabricksConn::parse(conn)?, + ))), + "snowflake" => Ok(Box::new(snowflake::SnowflakeEngine::new( + conn::SnowflakeConn::parse(conn)?, + ))), + other => anyhow::bail!("Unsupported engine: {other}"), + } +} + +/// 安全闸第 1 层：按引擎方言解析并校验，返回规整后的语句文本。 +pub fn guard_sql_for(engine: &str, sql: &str) -> anyhow::Result { + let cleaned = sql.trim().trim_end_matches(';').trim(); + if cleaned.is_empty() { + anyhow::bail!("Empty SQL"); + } + let parsed = match engine { + "databricks" => Parser::parse_sql(&DatabricksDialect {}, cleaned), + "snowflake" => Parser::parse_sql(&SnowflakeDialect {}, cleaned), + "trino" => Parser::parse_sql(&GenericDialect {}, cleaned), + _ => Parser::parse_sql(&PostgreSqlDialect {}, cleaned), + }; + let statements = parsed.map_err(|e| anyhow::anyhow!("SQL parse error: {e}"))?; + if statements.len() != 1 { + anyhow::bail!("Exactly one statement is allowed"); + } + match &statements[0] { + Statement::Query(_) => Ok(cleaned.to_string()), + other => anyhow::bail!( + "Read-only: only SELECT/WITH queries are allowed (got {})", + statement_kind(other) + ), + } +} + +fn statement_kind(s: &Statement) -> &'static str { + match s { + Statement::Insert { .. } => "INSERT", + Statement::Update { .. } => "UPDATE", + Statement::Delete { .. } => "DELETE", + Statement::CreateTable { .. } => "CREATE TABLE", + Statement::Drop { .. } => "DROP", + Statement::AlterTable { .. } => "ALTER TABLE", + Statement::Truncate { .. } => "TRUNCATE", + _ => "a non-SELECT statement", + } +} + +/// 第 2 层：外包一层 LIMIT。三个 HTTP 引擎都认这个写法；PG 有自己的 row_to_json 版本 +pub(crate) fn wrap_limit(sql: &str) -> String { + format!("SELECT * FROM ( {sql} ) AS _q LIMIT {}", ROW_CAP + 1) +} + +/// 第 201 行只用来判断截断，不交给模型 +pub(crate) fn truncate_rows(mut rows: Vec) -> (Vec, bool) { + let truncated = rows.len() > ROW_CAP; + rows.truncate(ROW_CAP); + (rows, truncated) +} + +/// HTTP 族共用：「列名 + 行值」拼成 JSON Lines。手拼而不是 `serde_json::Map`， +/// 后者不开 `preserve_order` 就按键排序，而列序是查询写下的顺序，模型读表靠它 +pub(crate) fn rows_to_json_lines( + columns: &[String], + rows: &[Vec], +) -> Vec { + rows.iter() + .map(|row| { + let mut line = String::from("{"); + for (i, col) in columns.iter().enumerate() { + if i > 0 { + line.push(','); + } + line.push_str(&serde_json::to_string(col).unwrap_or_else(|_| "\"?\"".into())); + line.push(':'); + let value = row.get(i).cloned().unwrap_or(serde_json::Value::Null); + line.push_str(&value.to_string()); + } + line.push('}'); + line + }) + .collect() +} + +/// Databricks 的 JSON_ARRAY 与 Snowflake 的 data 把每个值都给成字符串（或 null）。 +/// 按列类型把数与布尔还原，其余留字符串——模型对 `"42"` 和 `42` 的算术不一样 +pub(crate) fn coerce(type_name: &str, raw: &serde_json::Value) -> serde_json::Value { + let serde_json::Value::String(s) = raw else { + return raw.clone(); + }; + let ty = type_name.to_ascii_uppercase(); + const NUMERIC: &[&str] = &[ + "INT", "LONG", "SHORT", "BYTE", "FLOAT", "DOUBLE", "DECIMAL", "NUMBER", "FIXED", "REAL", + "NUMERIC", + ]; + // INTERVAL 也含 "INT"：解析不成数就原样留下，不会误伤 + if NUMERIC.iter().any(|k| ty.contains(k)) { + if let Ok(n) = s.parse::() { + return n.into(); + } + if let Ok(f) = s.parse::() { + if let Some(n) = serde_json::Number::from_f64(f) { + return serde_json::Value::Number(n); + } + } + } + if ty.starts_with("BOOL") { + match s.as_str() { + "true" | "TRUE" => return true.into(), + "false" | "FALSE" => return false.into(), + _ => {} + } + } + raw.clone() +} + +/// 单引号字面量的转义：schema 名进 information_schema 的 WHERE 子句 +pub(crate) fn sql_literal(s: &str) -> String { + format!("'{}'", s.replace('\'', "''")) +} + +/// HTTP 族共用的客户端。 +/// +/// **代理策略是显式的**：回环地址与 `NO_PROXY` 里的主机直连，其余按 `HTTPS_PROXY` / +/// `HTTP_PROXY` / `ALL_PROXY` 走。不用 reqwest 的系统代理探测——Windows 上它读注册表， +/// 而注册表里 `127.*` 这种绕过写法它认不全，本机的替身服务会被送进代理拿回 502。 +/// 服务进程该看环境变量，这条规矩与 docker-compose 里的写法一致 +pub(crate) fn http() -> anyhow::Result { + Ok(reqwest::Client::builder() + .timeout(HTTP_REQUEST_TIMEOUT) + .user_agent("utopia") + .proxy(reqwest::Proxy::custom(|url: &reqwest::Url| proxy_for(url))) + .build()?) +} + +fn proxy_for(url: &reqwest::Url) -> Option { + let host = url.host_str()?; + let loopback = host.eq_ignore_ascii_case("localhost") + || host + .trim_matches(|c| c == '[' || c == ']') + .parse::() + .map(|ip| ip.is_loopback()) + .unwrap_or(false); + if loopback || no_proxy_matches(host) { + return None; + } + let keys: &[&str] = if url.scheme() == "https" { + &["HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"] + } else { + &["HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy"] + }; + keys.iter() + .find_map(|k| std::env::var(k).ok()) + .filter(|v| !v.trim().is_empty()) + .and_then(|v| reqwest::Url::parse(v.trim()).ok()) +} + +/// `NO_PROXY=localhost,127.0.0.1,.internal,corp.example` 的常见写法：整名相等， +/// 或者以点开头的后缀匹配 +fn no_proxy_matches(host: &str) -> bool { + let raw = std::env::var("NO_PROXY") + .or_else(|_| std::env::var("no_proxy")) + .unwrap_or_default(); + raw.split(',') + .map(str::trim) + .filter(|p| !p.is_empty() && *p != "*") + .any(|p| { + let p = p.trim_start_matches('.'); + host.eq_ignore_ascii_case(p) + || host + .to_ascii_lowercase() + .ends_with(&format!(".{}", p.to_ascii_lowercase())) + }) + || raw.split(',').any(|p| p.trim() == "*") +} + +#[cfg(test)] +mod tests { + use super::{coerce, engine_from_conn, guard_sql_for, rows_to_json_lines}; + use serde_json::json; + + fn guard_sql(sql: &str) -> anyhow::Result { + guard_sql_for("postgres", sql) + } + + #[test] + fn allows_select_and_cte() { + assert!(guard_sql("SELECT region, sum(amount) FROM orders GROUP BY 1").is_ok()); + assert!(guard_sql("WITH t AS (SELECT 1 AS x) SELECT * FROM t;").is_ok()); + } + + #[test] + fn rejects_writes_and_ddl() { + for bad in [ + "UPDATE orders SET amount = 0", + "DELETE FROM orders", + "INSERT INTO orders (region) VALUES ('east')", + "DROP TABLE orders", + "TRUNCATE orders", + "CREATE TABLE t (id int)", + "ALTER TABLE orders ADD COLUMN x int", + ] { + assert!(guard_sql(bad).is_err(), "should reject: {bad}"); + } + } + + #[test] + fn rejects_multi_statement() { + assert!(guard_sql("SELECT 1; DROP TABLE orders").is_err()); + assert!(guard_sql("").is_err()); + } + + #[test] + fn every_dialect_keeps_the_same_gate() { + for engine in ["postgres", "trino", "databricks", "snowflake"] { + assert!( + guard_sql_for(engine, "SELECT a FROM t WHERE b > 1").is_ok(), + "{engine}" + ); + assert!(guard_sql_for(engine, "DELETE FROM t").is_err(), "{engine}"); + assert!( + guard_sql_for(engine, "SELECT 1; SELECT 2").is_err(), + "{engine}" + ); + } + // 各家的方言细节：反引号、双冒号转型，都要过得去 + assert!(guard_sql_for("databricks", "SELECT `region` FROM main.sales.orders").is_ok()); + assert!(guard_sql_for("snowflake", "SELECT amount::number FROM db.public.orders").is_ok()); + assert!(guard_sql_for("trino", "SELECT count(*) FROM hive.default.orders").is_ok()); + } + + #[test] + fn engine_follows_the_scheme() { + assert_eq!(engine_from_conn("postgres://u:p@h/db"), Some("postgres")); + assert_eq!(engine_from_conn("postgresql://u:p@h/db"), Some("postgres")); + assert_eq!(engine_from_conn("trino://u@h:8443/hive"), Some("trino")); + assert_eq!(engine_from_conn("presto://u@h/hive"), Some("trino")); + assert_eq!( + engine_from_conn("databricks://:t@h/sql/1.0/warehouses/x"), + Some("databricks") + ); + assert_eq!( + engine_from_conn("snowflake://:t@a.snowflakecomputing.com/db"), + Some("snowflake") + ); + assert_eq!(engine_from_conn("mysql://u@h/db"), None); + assert_eq!(engine_from_conn("garbage"), None); + } + + #[test] + fn json_lines_keep_column_order() { + let cols = vec!["zeta".to_string(), "alpha".to_string()]; + let rows = vec![vec![json!(1), json!("x")], vec![json!(null)]]; + assert_eq!( + rows_to_json_lines(&cols, &rows), + vec![r#"{"zeta":1,"alpha":"x"}"#, r#"{"zeta":null,"alpha":null}"#] + ); + } + + #[test] + fn strings_come_back_as_numbers_when_the_column_says_so() { + assert_eq!(coerce("DOUBLE", &json!("12.5")), json!(12.5)); + assert_eq!(coerce("fixed", &json!("42")), json!(42)); + assert_eq!(coerce("BOOLEAN", &json!("true")), json!(true)); + assert_eq!(coerce("STRING", &json!("42")), json!("42")); + assert_eq!(coerce("INTERVAL", &json!("1 day")), json!("1 day")); + assert_eq!(coerce("DOUBLE", &json!(null)), json!(null)); + } +} diff --git a/crates/utopia-server/src/query_engine/postgres.rs b/crates/utopia-server/src/query_engine/postgres.rs new file mode 100644 index 00000000..5bba3a4e --- /dev/null +++ b/crates/utopia-server/src/query_engine/postgres.rs @@ -0,0 +1,93 @@ +//! Postgres 族（顺带覆盖 Greenplum / Timescale 等 PG 兼容系）。线协议直连， +//! 是四个引擎里唯一有会话可设只读的那个。 + +use super::{QueryEngine, QueryResult, SchemaColumn, ROW_CAP, STATEMENT_TIMEOUT_SECS}; +use sqlx::postgres::PgPoolOptions; +use sqlx::Row; +use std::time::Duration; + +pub struct PostgresEngine { + conn: String, +} + +impl PostgresEngine { + pub fn new(conn: &str) -> Self { + Self { + conn: conn.to_string(), + } + } + + async fn pool(&self) -> anyhow::Result { + Ok(PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(Duration::from_secs(5)) + .connect(&self.conn) + .await?) + } +} + +#[async_trait::async_trait] +impl QueryEngine for PostgresEngine { + async fn test(&self) -> anyhow::Result<()> { + let pool = self.pool().await?; + sqlx::query("SELECT 1").execute(&pool).await?; + pool.close().await; + Ok(()) + } + + async fn fetch_schema(&self) -> anyhow::Result> { + let pool = self.pool().await?; + let rows: Vec<(String, String, String, String, Option)> = sqlx::query_as( + "SELECT c.table_schema, c.table_name, c.column_name, + c.data_type, pgd.description + FROM information_schema.columns c + LEFT JOIN pg_catalog.pg_statio_all_tables st + ON st.schemaname = c.table_schema AND st.relname = c.table_name + LEFT JOIN pg_catalog.pg_description pgd + ON pgd.objoid = st.relid AND pgd.objsubid = c.ordinal_position + WHERE c.table_schema NOT IN ('pg_catalog', 'information_schema') + ORDER BY c.table_schema, c.table_name, c.ordinal_position", + ) + .fetch_all(&pool) + .await?; + pool.close().await; + Ok(rows + .into_iter() + .map(|(schema, table, column, data_type, comment)| SchemaColumn { + schema, + table, + column, + data_type, + comment, + }) + .collect()) + } + + async fn execute(&self, sql: &str) -> anyhow::Result { + let pool = self.pool().await?; + // 纵深防御第 3 层：会话级只读 + 超时（parser 漏网也写不进去、跑不死库） + sqlx::query("SET default_transaction_read_only = on") + .execute(&pool) + .await?; + sqlx::query(&format!( + "SET statement_timeout = '{STATEMENT_TIMEOUT_SECS}s'" + )) + .execute(&pool) + .await?; + // 第 2 层：外包 LIMIT；row_to_json 让 PG 全权处理类型→JSON（文本键序保留列序） + let wrapped = format!( + "SELECT row_to_json(_q)::text AS _j FROM ( {sql} ) AS _q LIMIT {}", + ROW_CAP + 1 + ); + let fetched = sqlx::query(&wrapped).fetch_all(&pool).await?; + pool.close().await; + + let truncated = fetched.len() > ROW_CAP; + let rows = fetched + .into_iter() + .take(ROW_CAP) + .map(|r| r.try_get::("_j").unwrap_or_else(|_| "{}".into())) + .collect(); + Ok(QueryResult { rows, truncated }) + } +} diff --git a/crates/utopia-server/src/query_engine/snowflake.rs b/crates/utopia-server/src/query_engine/snowflake.rs new file mode 100644 index 00000000..ff810611 --- /dev/null +++ b/crates/utopia-server/src/query_engine/snowflake.rs @@ -0,0 +1,257 @@ +//! Snowflake SQL API v2（`/api/v2/statements`）。同步提交（`async=false`）拿不完的 +//! 语句回 202，沿 statementHandle 轮询。值全是字符串，按 rowType 还原数与布尔。 +//! +//! 只收令牌，不收密码：programmatic access token 或 OAuth。密钥对 JWT 要本地签名， +//! 这一版不做——见 `conn.rs`。 + +use super::conn::SnowflakeConn; +use super::{ + coerce, rows_to_json_lines, sql_literal, truncate_rows, wrap_limit, QueryEngine, QueryResult, + SchemaColumn, HTTP_POLL_BUDGET, STATEMENT_TIMEOUT_SECS, +}; +use reqwest::StatusCode; +use serde::Deserialize; +use serde_json::json; +use std::time::{Duration, Instant}; + +pub struct SnowflakeEngine { + conn: SnowflakeConn, +} + +#[derive(Deserialize)] +struct StatementResponse { + #[serde(rename = "resultSetMetaData")] + meta: Option, + data: Option>>, + message: Option, + code: Option, + #[serde(rename = "statementHandle")] + handle: Option, +} + +#[derive(Deserialize)] +struct Meta { + #[serde(rename = "rowType")] + row_type: Vec, +} + +#[derive(Deserialize)] +struct RowType { + name: String, + #[serde(rename = "type")] + ty: String, +} + +impl SnowflakeEngine { + pub fn new(conn: SnowflakeConn) -> Self { + Self { conn } + } + + fn request(&self, r: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + r.bearer_auth(&self.conn.token) + .header("X-Snowflake-Authorization-Token-Type", self.conn.token_type) + .header("Accept", "application/json") + } + + async fn run(&self, sql: &str) -> anyhow::Result<(Vec, Vec>)> { + let client = super::http()?; + let mut body = json!({ + "statement": sql, + "timeout": STATEMENT_TIMEOUT_SECS, + "parameters": { "MULTI_STATEMENT_COUNT": "1" }, + }); + for (key, value) in [ + ("database", &self.conn.database), + ("schema", &self.conn.schema), + ("warehouse", &self.conn.warehouse), + ("role", &self.conn.role), + ] { + if let Some(v) = value { + body[key] = json!(v); + } + } + let mut http = self + .request(client.post(format!("{}/api/v2/statements?async=false", self.conn.base))) + .json(&body) + .send() + .await?; + let started = Instant::now(); + // 202 = 还在跑；其余非 2xx 的 body 里带 message + while http.status() == StatusCode::ACCEPTED { + let partial: StatementResponse = http.json().await?; + let handle = partial.handle.ok_or_else(|| { + anyhow::anyhow!("Snowflake returned 202 without a statementHandle") + })?; + if started.elapsed() > HTTP_POLL_BUDGET { + anyhow::bail!( + "Snowflake statement did not finish within {}s", + HTTP_POLL_BUDGET.as_secs() + ); + } + tokio::time::sleep(Duration::from_secs(1)).await; + http = self + .request(client.get(format!("{}/api/v2/statements/{handle}", self.conn.base))) + .send() + .await?; + } + if !http.status().is_success() { + let status = http.status(); + let text = http.text().await.unwrap_or_default(); + let msg = serde_json::from_str::(&text) + .ok() + .and_then(|r| r.message) + .unwrap_or(text); + anyhow::bail!("Snowflake {status}: {msg}"); + } + let resp: StatementResponse = http.json().await?; + if let (Some(code), Some(message)) = (&resp.code, &resp.message) { + // 2xx 里也可能带业务错误码；090001 是 "statement executed successfully" + if code != "090001" && resp.meta.is_none() { + anyhow::bail!("Snowflake {code}: {message}"); + } + } + let types: Vec = resp.meta.map(|m| m.row_type).unwrap_or_default(); + let rows = resp + .data + .unwrap_or_default() + .into_iter() + .map(|row| { + row.iter() + .enumerate() + .map(|(i, v)| coerce(types.get(i).map(|t| t.ty.as_str()).unwrap_or(""), v)) + .collect() + }) + .collect(); + Ok((types.into_iter().map(|t| t.name).collect(), rows)) + } +} + +#[async_trait::async_trait] +impl QueryEngine for SnowflakeEngine { + async fn test(&self) -> anyhow::Result<()> { + self.run("SELECT 1").await.map(|_| ()) + } + + async fn fetch_schema(&self) -> anyhow::Result> { + let database = self.conn.database.as_deref().ok_or_else(|| { + anyhow::anyhow!("snowflake://: put the database in the connection string (snowflake://:TOKEN@account/DATABASE) so the schema can be read") + })?; + let schema_filter = self + .conn + .schema + .as_deref() + .map(|s| format!(" AND table_schema = {}", sql_literal(s))) + .unwrap_or_default(); + let sql = format!( + "SELECT table_schema, table_name, column_name, data_type, comment \ + FROM \"{}\".information_schema.columns \ + WHERE table_schema <> 'INFORMATION_SCHEMA'{schema_filter} \ + ORDER BY table_schema, table_name, ordinal_position", + database.replace('"', "\"\"") + ); + let (_, rows) = self.run(&sql).await?; + Ok(rows.into_iter().map(super::trino::schema_row).collect()) + } + + async fn execute(&self, sql: &str) -> anyhow::Result { + let (columns, rows) = self.run(&wrap_limit(sql)).await?; + let (rows, truncated) = truncate_rows(rows); + Ok(QueryResult { + rows: rows_to_json_lines(&columns, &rows), + truncated, + }) + } +} + +#[cfg(test)] +mod tests { + use super::super::conn::SnowflakeConn; + use super::super::QueryEngine; + use super::SnowflakeEngine; + use serde_json::json; + use wiremock::matchers::{header, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn conn(server: &MockServer) -> SnowflakeConn { + SnowflakeConn::parse(&format!( + "snowflake://:pat-test@{}/ANALYTICS/PUBLIC?warehouse=WH&ssl=false", + server.uri().trim_start_matches("http://") + )) + .unwrap() + } + + #[tokio::test] + async fn a_synchronous_answer_is_typed_by_row_type() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/v2/statements")) + .and(header("authorization", "Bearer pat-test")) + .and(header( + "X-Snowflake-Authorization-Token-Type", + "PROGRAMMATIC_ACCESS_TOKEN", + )) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "resultSetMetaData": { "numRows": 1, "rowType": [ + { "name": "REGION", "type": "text" }, + { "name": "TOTAL", "type": "fixed", "scale": 2 } + ] }, + "data": [ ["east", "42.10"] ], + "code": "090001", + "statementHandle": "h1", + "message": "Statement executed successfully." + }))) + .expect(1) + .mount(&server) + .await; + let out = SnowflakeEngine::new(conn(&server)) + .execute("SELECT region, total FROM orders") + .await + .unwrap(); + assert_eq!(out.rows, vec![r#"{"REGION":"east","TOTAL":42.1}"#]); + } + + #[tokio::test] + async fn a_202_is_polled_until_the_answer_arrives() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/v2/statements")) + .respond_with(ResponseTemplate::new(202).set_body_json(json!({ + "code": "333334", "statementHandle": "h2", "message": "Asynchronous execution in progress." + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/api/v2/statements/h2")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "resultSetMetaData": { "rowType": [ { "name": "N", "type": "fixed" } ] }, + "data": [ ["1"] ], "code": "090001", "statementHandle": "h2" + }))) + .expect(1) + .mount(&server) + .await; + let out = SnowflakeEngine::new(conn(&server)) + .execute("SELECT 1 AS n") + .await + .unwrap(); + assert_eq!(out.rows, vec![r#"{"N":1}"#]); + } + + #[tokio::test] + async fn an_error_body_is_surfaced() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/v2/statements")) + .respond_with(ResponseTemplate::new(422).set_body_json(json!({ + "code": "002003", "message": "SQL compilation error: Object 'NOPE' does not exist" + }))) + .mount(&server) + .await; + let err = SnowflakeEngine::new(conn(&server)) + .execute("SELECT * FROM nope") + .await + .unwrap_err() + .to_string(); + assert!(err.contains("does not exist"), "{err}"); + } +} diff --git a/crates/utopia-server/src/query_engine/trino.rs b/crates/utopia-server/src/query_engine/trino.rs new file mode 100644 index 00000000..4b2c40dc --- /dev/null +++ b/crates/utopia-server/src/query_engine/trino.rs @@ -0,0 +1,266 @@ +//! Trino（旧名 Presto）：REST 协议 `POST /v1/statement`，然后沿 `nextUri` 一页页取。 +//! 一个引擎顶起整个湖仓——Iceberg / Delta / Hive / Hudi 都是它的 catalog， +//! 换格式不换协议。Starburst 同协议。 +//! +//! 没有会话可设只读：超时靠 `X-Trino-Session: query_max_execution_time`， +//! 只读靠 `guard_sql_for`。 + +use super::conn::TrinoConn; +use super::{ + rows_to_json_lines, sql_literal, truncate_rows, wrap_limit, QueryEngine, QueryResult, + SchemaColumn, HTTP_POLL_BUDGET, STATEMENT_TIMEOUT_SECS, +}; +use base64::Engine as _; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; +use serde::Deserialize; +use std::time::Instant; + +pub struct TrinoEngine { + conn: TrinoConn, +} + +#[derive(Deserialize)] +struct Column { + name: String, +} + +#[derive(Deserialize)] +struct Page { + #[serde(rename = "nextUri")] + next_uri: Option, + columns: Option>, + data: Option>>, + error: Option, +} + +#[derive(Deserialize)] +struct TrinoError { + message: String, + #[serde(rename = "errorName")] + error_name: Option, +} + +impl TrinoEngine { + pub fn new(conn: TrinoConn) -> Self { + Self { conn } + } + + fn headers(&self) -> anyhow::Result { + let mut h = HeaderMap::new(); + h.insert("X-Trino-User", HeaderValue::from_str(&self.conn.user)?); + h.insert("X-Trino-Source", HeaderValue::from_static("utopia")); + h.insert( + "X-Trino-Session", + HeaderValue::from_str(&format!( + "query_max_execution_time={STATEMENT_TIMEOUT_SECS}s" + ))?, + ); + if let Some(c) = &self.conn.catalog { + h.insert("X-Trino-Catalog", HeaderValue::from_str(c)?); + } + if let Some(s) = &self.conn.schema { + h.insert("X-Trino-Schema", HeaderValue::from_str(s)?); + } + if let Some(p) = &self.conn.password { + let raw = format!("{}:{p}", self.conn.user); + let token = base64::engine::general_purpose::STANDARD.encode(raw); + h.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Basic {token}"))?, + ); + } + Ok(h) + } + + /// 提交并沿 nextUri 收完：列在第一个带 columns 的页上，数据分页累积 + async fn run(&self, sql: &str) -> anyhow::Result<(Vec, Vec>)> { + let client = super::http()?; + let headers = self.headers()?; + let mut page: Page = client + .post(format!("{}/v1/statement", self.conn.base)) + .headers(headers.clone()) + .body(sql.to_string()) + .send() + .await? + .error_for_status()? + .json() + .await?; + let started = Instant::now(); + let mut columns: Option> = None; + let mut rows = Vec::new(); + loop { + if let Some(e) = page.error { + let name = e.error_name.map(|n| format!("{n}: ")).unwrap_or_default(); + anyhow::bail!("{name}{}", e.message); + } + if columns.is_none() { + columns = page + .columns + .take() + .map(|cs| cs.into_iter().map(|c| c.name).collect()); + } + if let Some(d) = page.data.take() { + rows.extend(d); + } + let Some(next) = page.next_uri.take() else { + break; + }; + if started.elapsed() > HTTP_POLL_BUDGET { + anyhow::bail!( + "Trino query did not finish within {}s", + HTTP_POLL_BUDGET.as_secs() + ); + } + page = client + .get(&next) + .headers(headers.clone()) + .send() + .await? + .error_for_status()? + .json() + .await?; + } + Ok((columns.unwrap_or_default(), rows)) + } +} + +#[async_trait::async_trait] +impl QueryEngine for TrinoEngine { + async fn test(&self) -> anyhow::Result<()> { + self.run("SELECT 1").await.map(|_| ()) + } + + async fn fetch_schema(&self) -> anyhow::Result> { + let catalog = self.conn.catalog.as_deref().ok_or_else(|| { + anyhow::anyhow!("trino://: put the catalog in the connection string (trino://user@host/CATALOG) so the schema can be read") + })?; + let schema_filter = self + .conn + .schema + .as_deref() + .map(|s| format!(" AND table_schema = {}", sql_literal(s))) + .unwrap_or_default(); + let sql = format!( + "SELECT table_schema, table_name, column_name, data_type, comment \ + FROM \"{}\".information_schema.columns \ + WHERE table_schema <> 'information_schema'{schema_filter} \ + ORDER BY table_schema, table_name, ordinal_position", + catalog.replace('"', "\"\"") + ); + let (_, rows) = self.run(&sql).await?; + Ok(rows.into_iter().map(schema_row).collect()) + } + + async fn execute(&self, sql: &str) -> anyhow::Result { + let (columns, rows) = self.run(&wrap_limit(sql)).await?; + let (rows, truncated) = truncate_rows(rows); + Ok(QueryResult { + rows: rows_to_json_lines(&columns, &rows), + truncated, + }) + } +} + +/// information_schema 的一行 → SchemaColumn（值可能是 null，comment 常是） +pub(crate) fn schema_row(row: Vec) -> SchemaColumn { + let text = |i: usize| -> String { + row.get(i) + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string() + }; + SchemaColumn { + schema: text(0), + table: text(1), + column: text(2), + data_type: text(3), + comment: row + .get(4) + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .map(str::to_string), + } +} + +#[cfg(test)] +mod tests { + use super::super::conn::TrinoConn; + use super::super::QueryEngine; + use super::TrinoEngine; + use serde_json::json; + use wiremock::matchers::{body_string_contains, header, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + #[tokio::test] + async fn follows_next_uri_and_keeps_column_order() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(header("X-Trino-User", "alice")) + .and(header("X-Trino-Catalog", "hive")) + .and(body_string_contains("LIMIT 201")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "q1", + "nextUri": format!("{}/v1/statement/q1/1", server.uri()), + "stats": { "state": "QUEUED" } + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/statement/q1/1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "q1", + "columns": [ { "name": "region", "type": "varchar" }, { "name": "total", "type": "double" } ], + "data": [ ["east", 12.5], ["west", 3] ], + "stats": { "state": "FINISHED" } + }))) + .expect(1) + .mount(&server) + .await; + + let uri = server.uri(); + let conn = TrinoConn::parse(&format!( + "trino://alice@{}/hive/default?ssl=false", + uri.trim_start_matches("http://") + )) + .unwrap(); + let out = TrinoEngine::new(conn) + .execute("SELECT region, total FROM orders") + .await + .unwrap(); + assert_eq!( + out.rows, + vec![ + r#"{"region":"east","total":12.5}"#, + r#"{"region":"west","total":3}"# + ] + ); + assert!(!out.truncated); + } + + #[tokio::test] + async fn a_trino_error_page_becomes_an_error() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "q2", + "error": { "message": "line 1:8: Table 'hive.default.nope' does not exist", "errorName": "TABLE_NOT_FOUND" }, + "stats": { "state": "FAILED" } + }))) + .mount(&server) + .await; + let conn = TrinoConn::parse(&format!( + "trino://alice@{}/hive?ssl=false", + server.uri().trim_start_matches("http://") + )) + .unwrap(); + let err = TrinoEngine::new(conn) + .execute("SELECT * FROM nope") + .await + .unwrap_err() + .to_string(); + assert!(err.contains("TABLE_NOT_FOUND"), "{err}"); + } +} diff --git a/crates/utopia-store/src/datasources.rs b/crates/utopia-store/src/datasources.rs index 5fad6389..6e3ec3a9 100644 --- a/crates/utopia-store/src/datasources.rs +++ b/crates/utopia-store/src/datasources.rs @@ -18,17 +18,16 @@ type DataSourceRow = ( Option, ); -/// 连接串 → 无凭据摘要（host:port/db）。解析失败给占位符，绝不回显原串。 +/// 连接串 → 无凭据摘要（host[:port]/path）。解析失败给占位符，绝不回显原串。 +/// 端口没写就不补：四种 scheme 的默认端口各不相同，补错比不补更误导 pub fn conn_summary(conn: &str) -> String { url::Url::parse(conn) .ok() .map(|u| { format!( - "{}:{}{}", + "{}{}{}", u.host_str().unwrap_or("?"), - u.port() - .map(|p| p.to_string()) - .unwrap_or_else(|| "5432".into()), + u.port().map(|p| format!(":{p}")).unwrap_or_default(), u.path() ) }) @@ -77,16 +76,12 @@ pub async fn create( "Data source name is required", )); } - if engine != "postgres" { - return Err(AppError::invalid( - "only_postgres", - "Only the postgres engine is supported for now", - )); - } - if !conn_string.starts_with("postgres://") && !conn_string.starts_with("postgresql://") { + // 引擎由调用方按连接串的 scheme 定（`query_engine::engine_from_conn`）； + // 允许的取值在迁移 0020 的 CHECK 里，这里不再复制一份 + if engine.is_empty() || conn_string.trim().is_empty() { return Err(AppError::invalid( "bad_conn_string", - "Connection string must start with postgres://", + "A connection string is required", )); } let id = Uuid::now_v7(); diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rs index 741e93f9..fe7689ad 100644 --- a/crates/utopia-store/src/graph.rs +++ b/crates/utopia-store/src/graph.rs @@ -418,13 +418,29 @@ async fn edges_among( // // 图要它们，因为「这条边是推出来的」正是用户该看见的信息之一；`derived` // 那一位让界面画得出区别，也让人整体过滤掉。 + // + // 第三段是**幽灵边**（0017 §3）：推出来却没落地的派生，住在 `axiom_violations` + // 的 `detail` 里。它的 id 是违规的 id；`derived` 与 `blocked` 同时为 true， + // 界面据此让它跟着派生开关走、画成争议色往背景混的那一档。 + // + // 断言那一段多算一位 `contested`：有 open 的违规或时态冲突指着它。派生撞断言 + // 时被撞的是 left；right 只是最后一条前提，它本身没有争议 let edges: Vec = sqlx::query_as( "SELECT f.id, f.subject_id AS source, f.object_id AS target, COALESCE(r.key, fact_surface_predicate(f.id)) AS predicate, COALESCE(r.label, fact_surface_predicate(f.id)) AS label, r.id IS NULL AS inferred, FALSE AS derived, NULL::text AS rule, ARRAY[]::uuid[] AS premises, - f.valid_from, f.valid_to, f.confidence + f.valid_from, f.valid_to, f.confidence, + (EXISTS (SELECT 1 FROM axiom_violations v + WHERE v.status = 'open' + AND (v.left_fact = f.id + OR (v.right_fact = f.id AND v.kind <> 'derived_contradiction'))) + OR EXISTS (SELECT 1 FROM fact_conflicts c + WHERE c.status = 'open' + AND (c.old_fact_id = f.id OR c.new_fact_id = f.id)) + ) AS contested, + FALSE AS blocked FROM facts f LEFT JOIN relation_types r ON r.id = f.predicate_id WHERE f.kb_id = $1 AND f.invalidated_at IS NULL AND f.object_id IS NOT NULL AND f.subject_id = ANY($2) AND f.object_id = ANY($2) @@ -437,14 +453,35 @@ async fn edges_among( FALSE AS inferred, TRUE AS derived, ru.kind AS rule, ARRAY(SELECT fd.premise_fact_id FROM fact_derivations fd WHERE fd.derived_fact_id = d.id ORDER BY fd.seq) AS premises, - d.valid_from, d.valid_to, d.confidence + d.valid_from, d.valid_to, d.confidence, + FALSE AS contested, FALSE AS blocked FROM derived_facts d JOIN relation_types r ON r.id = d.predicate_id JOIN rules ru ON ru.id = d.rule_id WHERE d.kb_id = $1 AND d.invalidated_at IS NULL AND d.subject_id = ANY($2) AND d.object_id = ANY($2) AND ($3::timestamptz IS NULL OR ((d.valid_from IS NULL OR d.valid_from <= $3) - AND (d.valid_to IS NULL OR d.valid_to > $3)))", + AND (d.valid_to IS NULL OR d.valid_to > $3))) + UNION ALL + SELECT v.id, + (v.detail->>'subject_id')::uuid AS source, + (v.detail->>'object_id')::uuid AS target, + v.detail->>'predicate' AS predicate, v.detail->>'predicate' AS label, + FALSE AS inferred, TRUE AS derived, v.detail->>'rule' AS rule, + v.path AS premises, + (v.detail->>'valid_from')::timestamptz AS valid_from, + (v.detail->>'valid_to')::timestamptz AS valid_to, + 0::real AS confidence, + TRUE AS contested, TRUE AS blocked + FROM axiom_violations v + WHERE v.kb_id = $1 AND v.kind = 'derived_contradiction' AND v.status = 'open' + AND (v.detail->>'subject_id')::uuid = ANY($2) + AND (v.detail->>'object_id')::uuid = ANY($2) + AND ($3::timestamptz IS NULL + OR (((v.detail->>'valid_from')::timestamptz IS NULL + OR (v.detail->>'valid_from')::timestamptz <= $3) + AND ((v.detail->>'valid_to')::timestamptz IS NULL + OR (v.detail->>'valid_to')::timestamptz > $3)))", ) .bind(kb_id) .bind(ids) @@ -568,7 +605,25 @@ pub async fn entity_detail( (f.supersedes IS NOT NULL) AS corrected, (SELECT MAX(COALESCE(d.doc_time, d.created_at)) FROM fact_evidence fe JOIN documents d ON d.id = fe.document_id - WHERE fe.fact_id = f.id) AS last_evidence_time + WHERE fe.fact_id = f.id) AS last_evidence_time, + COALESCE( + (SELECT jsonb_build_object( + 'kind', v.kind, 'ref_id', v.id, + 'derived', CASE WHEN v.kind = 'derived_contradiction' + THEN (v.detail->>'subject') || ' · ' + || (v.detail->>'predicate') || ' · ' + || (v.detail->>'object') END) + FROM axiom_violations v + WHERE v.status = 'open' + AND (v.left_fact = f.id + OR (v.right_fact = f.id AND v.kind <> 'derived_contradiction')) + ORDER BY v.detected_at DESC LIMIT 1), + (SELECT jsonb_build_object('kind', 'temporal_conflict', 'ref_id', c.id) + FROM fact_conflicts c + WHERE c.status = 'open' + AND (c.old_fact_id = f.id OR c.new_fact_id = f.id) + ORDER BY c.created_at DESC LIMIT 1) + ) AS contested FROM facts f LEFT JOIN relation_types r ON r.id = f.predicate_id LEFT JOIN entities o diff --git a/crates/utopia-store/src/reasoning.rs b/crates/utopia-store/src/reasoning.rs index a0562078..1e7b799c 100644 --- a/crates/utopia-store/src/reasoning.rs +++ b/crates/utopia-store/src/reasoning.rs @@ -12,13 +12,14 @@ //! 提案刷回待看，等于每跑一次就把人的否决抹掉一次。所以这里 `ON CONFLICT` //! 什么都不做——已经在库里的那一行，无论 open 还是 resolved，都按原样留着。 +use serde_json::json; use sqlx::PgPool; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use utopia_core::models::{AxiomViolation, DerivedFactView, OntologyDefect}; /// 规则种类的字面量。用 &'static str 而不是枚举:它直接进 SQL 也直接做键 type RuleKind = &'static str; use utopia_core::AppResult; -use utopia_reason::derive::TimedEdge; +use utopia_reason::derive::{Contradictions, Derivation, TimedEdge}; use utopia_reason::{check, Axioms, Edge, Kind, Violation}; use uuid::Uuid; @@ -35,37 +36,17 @@ pub struct Report { pub inserted: usize, /// 清掉的陈旧 open 行 pub cleared: usize, + /// 派生撞上断言的条数（0017），已含在 `found` 里 + pub contradictions: usize, + /// 撞上单谓词上限、没进队列的矛盾条数。**不为零时说明根子在规则**： + /// 一条谓词上上百条派生都撞了，逐条看是没有意义的 + pub contradictions_capped: usize, + /// 互撞的规则对数——进 `ontology_defects`，不进这张表 + pub rules_disagree: usize, } -/// 取这个库里所有能参与检查的边。 -/// -/// 三个过滤条件都是必要的： -/// -/// - `invalidated_at IS NULL`——被推翻的事实不该再报矛盾，它已经不是我们的断言了 -/// - `predicate_id IS NOT NULL`——没有谓词就没有公理可依（见 `facts.predicate_id`） -/// - `object_id IS NOT NULL`——属性事实的宾语是字面值，公理谈的是实体之间的关系 -async fn edges(pool: &PgPool, kb_id: Uuid) -> AppResult> { - let rows: Vec<(Uuid, Uuid, Uuid, Uuid)> = sqlx::query_as( - "SELECT id, predicate_id, subject_id, object_id - FROM facts - WHERE kb_id = $1 - AND invalidated_at IS NULL - AND predicate_id IS NOT NULL - AND object_id IS NOT NULL", - ) - .bind(kb_id) - .fetch_all(pool) - .await?; - Ok(rows - .into_iter() - .map(|(fact, predicate, subject, object)| Edge { - fact, - predicate, - subject, - object, - }) - .collect()) -} +/// 单个谓词上进队列的矛盾上限（0017 §1）。超出的部分只计数。 +const MAX_CLASHES_PER_PREDICATE: usize = 50; /// 取这个库的谓词公理。 /// @@ -229,7 +210,8 @@ pub async fn record_signature_breaks( /// 跑一遍检查，把结果落库。 pub async fn run(pool: &PgPool, kb_id: Uuid) -> AppResult { - let edges = edges(pool, kb_id).await?; + let (timed, spans, _) = timed_edges(pool, kb_id).await?; + let edges: Vec = timed.iter().map(|t| t.edge).collect(); let axioms = axioms(pool, kb_id).await?; let mut violations = check(&edges, &axioms); // 第五类不在纯逻辑引擎里：它要看实体的类型与谓词的 domain / range，那是库里的 @@ -243,10 +225,63 @@ pub async fn run(pool: &PgPool, kb_id: Uuid) -> AppResult { }); } + // 第六类（0017）：推出来却落不了地的派生。与 `materialize` 用同一个函数算， + // 所以这里报的正是那边拦下的——两边各算一套的话，队列会跟图对不上 + let derivation = utopia_reason::derive::derive(&timed, &axioms); + let clashes = utopia_reason::derive::contradictions(&derivation, &timed, &axioms, &spans); + let names = names_for(pool, &derivation, &clashes).await?; + let mut details: HashMap<(Uuid, Uuid), serde_json::Value> = HashMap::new(); + let mut per_pred: HashMap = HashMap::new(); + let mut contradictions_capped = 0usize; + for c in &clashes.with_assertions { + let d = &derivation.facts[c.derived]; + let Some(&last) = d.premises.last() else { + continue; + }; + let key = (c.against, last); + if details.contains_key(&key) { + continue; + } + let n = per_pred.entry(d.predicate).or_default(); + if *n >= MAX_CLASHES_PER_PREDICATE { + contradictions_capped += 1; + continue; + } + *n += 1; + let span = utopia_reason::derive::validity(&d.premises, &spans); + details.insert( + key, + json!({ + "axiom": c.axiom.as_str(), + "rule": d.rule.as_str(), + "via": d.via, + "via_label": names.predicate(d.via), + "subject_id": d.subject, + "subject": names.entity(d.subject), + "predicate_id": d.predicate, + "predicate": names.predicate(d.predicate), + "object_id": d.object, + "object": names.entity(d.object), + "valid_from": span.and_then(|s| s.0).map(|t| stamp(t).to_rfc3339()), + "valid_to": span.and_then(|s| s.1).map(|t| stamp(t).to_rfc3339()), + "premises": d.premises, + }), + ); + violations.push(Violation { + kind: Kind::DerivedContradiction, + left: c.against, + right: last, + path: d.premises.clone(), + }); + } + let mut report = Report { edges: edges.len(), predicates_with_axioms: axioms.len(), found: violations.len(), + contradictions: details.len(), + contradictions_capped, + rules_disagree: clashes.between_derivations.len(), ..Default::default() }; @@ -261,9 +296,13 @@ pub async fn run(pool: &PgPool, kb_id: Uuid) -> AppResult { right, path, } = v; + let detail = details + .get(&(*left, *right)) + .cloned() + .unwrap_or_else(|| json!({})); let id: Option<(Uuid,)> = sqlx::query_as( - "INSERT INTO axiom_violations (id, kb_id, kind, left_fact, right_fact, path) - VALUES ($1, $2, $3, $4, $5, $6) + "INSERT INTO axiom_violations (id, kb_id, kind, left_fact, right_fact, path, detail) + VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (kb_id, kind, left_fact, right_fact) DO NOTHING RETURNING id", ) @@ -273,6 +312,7 @@ pub async fn run(pool: &PgPool, kb_id: Uuid) -> AppResult { .bind(left) .bind(right) .bind(path) + .bind(&detail) .fetch_optional(&mut *tx) .await?; if id.is_some() { @@ -303,10 +343,144 @@ pub async fn run(pool: &PgPool, kb_id: Uuid) -> AppResult { .execute(&mut *tx) .await?; report.cleared = cleared.rows_affected() as usize; + + // 派生之间互撞的按规则对进 `ontology_defects`——根子是那两条声明，不是哪条事实。 + // 同一对谓词上可能有几种撞法（functional 与 asymmetric 各撞各的），唯一键只到 + // 谓词对，所以合成一行，几种撞法都写进 detail + let mut by_pair: HashMap<(Uuid, Uuid), Vec> = HashMap::new(); + let mut order: Vec<(Uuid, Uuid)> = Vec::new(); + for rc in &clashes.between_derivations { + let triple = |i: usize| { + let d = &derivation.facts[i]; + format!( + "{} · {} · {}", + names.entity(d.subject), + names.predicate(d.predicate), + names.entity(d.object) + ) + }; + let examples: Vec = rc + .pairs + .iter() + .take(3) + .map(|(i, j)| json!([triple(*i), triple(*j)])) + .collect(); + let key = (rc.a.0, rc.b.0); + if !by_pair.contains_key(&key) { + order.push(key); + } + by_pair.entry(key).or_default().push(json!({ + "rule_a": rc.a.1.as_str(), + "via_a": names.predicate(rc.a.0), + "rule_b": rc.b.1.as_str(), + "via_b": names.predicate(rc.b.0), + "axiom": rc.axiom.as_str(), + "count": rc.pairs.len(), + "examples": examples, + })); + } + let mut fresh_defects: Vec = Vec::with_capacity(order.len()); + for key in order { + let rules = by_pair.remove(&key).unwrap_or_default(); + let count: usize = rules + .iter() + .map(|r| r["count"].as_u64().unwrap_or(0) as usize) + .sum(); + // 已经有人认可过的那一行保持 resolved，只刷 detail：0017 说认可之后不再报 + let (id,): (Uuid,) = sqlx::query_as( + "INSERT INTO ontology_defects (id, kb_id, kind, subject, other, path, detail) + VALUES ($1, $2, 'rules_disagree', $3, $4, '{}', $5) + ON CONFLICT (kb_id, kind, subject, other) DO UPDATE SET detail = EXCLUDED.detail + RETURNING id", + ) + .bind(Uuid::now_v7()) + .bind(kb_id) + .bind(key.0) + .bind(key.1) + .bind(json!({ "count": count, "rules": rules })) + .fetch_one(&mut *tx) + .await?; + fresh_defects.push(id); + } + sqlx::query( + "DELETE FROM ontology_defects + WHERE kb_id = $1 AND kind = 'rules_disagree' AND status = 'open' + AND NOT (id = ANY($2))", + ) + .bind(kb_id) + .bind(&fresh_defects) + .execute(&mut *tx) + .await?; tx.commit().await?; Ok(report) } +/// 矛盾要写成人能读的话，而派生没有落库、没有文本可查——名字在这里补。 +struct Names { + entities: HashMap, + predicates: HashMap, +} + +impl Names { + fn entity(&self, id: Uuid) -> String { + self.entities + .get(&id) + .cloned() + .unwrap_or_else(|| "?".into()) + } + fn predicate(&self, id: Uuid) -> String { + self.predicates + .get(&id) + .cloned() + .unwrap_or_else(|| "?".into()) + } +} + +async fn names_for( + pool: &PgPool, + derivation: &Derivation, + clashes: &Contradictions, +) -> AppResult { + let mut ents: HashSet = HashSet::new(); + let mut preds: HashSet = HashSet::new(); + let mut want = |i: usize| { + let d = &derivation.facts[i]; + ents.insert(d.subject); + ents.insert(d.object); + preds.insert(d.predicate); + preds.insert(d.via); + }; + for c in &clashes.with_assertions { + want(c.derived); + } + for rc in &clashes.between_derivations { + for (i, j) in rc.pairs.iter().take(3) { + want(*i); + want(*j); + } + } + for rc in &clashes.between_derivations { + preds.insert(rc.a.0); + preds.insert(rc.b.0); + } + let ents: Vec = ents.into_iter().collect(); + let preds: Vec = preds.into_iter().collect(); + let entities: Vec<(Uuid, String)> = + sqlx::query_as("SELECT id, canonical_name FROM entities WHERE id = ANY($1)") + .bind(&ents) + .fetch_all(pool) + .await?; + let predicates: Vec<(Uuid, String)> = + sqlx::query_as("SELECT id, label FROM relation_types WHERE id = ANY($1)") + .bind(&preds) + .fetch_all(pool) + .await?; + Ok(Names { + entities: entities.into_iter().collect(), + predicates: predicates.into_iter().collect(), + }) +} + /// Review 页要看的:还没人表态的违规,连同两条事实的三元组文本。 /// /// 展开成文本在 SQL 里做而不是回来再查一遍:一页几十条,每条两个三元组, @@ -336,10 +510,20 @@ pub async fn open_violations( v.left_fact, l.text AS left_text, v.right_fact, rt.text AS right_text, coalesce(array_length(v.path, 1), 0) AS path_len, - v.detected_at + v.detected_at, v.detail, + lf.valid_to IS NULL AS left_open, + lf.confidence AS left_confidence, + EXISTS ( + SELECT 1 FROM entities e + JOIN entities x ON x.kb_id = e.kb_id AND x.id <> e.id + AND x.merged_into IS NULL + AND lower(x.canonical_name) = lower(e.canonical_name) + WHERE e.id IN (lf.subject_id, lf.object_id) + ) AS same_name_peers FROM axiom_violations v JOIN triple l ON l.id = v.left_fact JOIN triple rt ON rt.id = v.right_fact + JOIN facts lf ON lf.id = v.left_fact WHERE v.kb_id = $1 AND v.status = 'open' ORDER BY v.detected_at DESC LIMIT $2 OFFSET $3", @@ -348,7 +532,60 @@ pub async fn open_violations( .bind(limit) .bind(offset) .fetch_all(pool) - .await?) + .await? + .into_iter() + .map(|r: ViolationRow| { + let hint = if r.kind == "derived_contradiction" { + hint_for(&r).map(String::from) + } else { + None + }; + AxiomViolation { + id: r.id, + kind: r.kind, + predicate: r.predicate, + left_fact: r.left_fact, + left_text: r.left_text, + right_fact: r.right_fact, + right_text: r.right_text, + path_len: r.path_len, + detected_at: r.detected_at, + detail: r.detail, + hint, + } + }) + .collect()) +} + +#[derive(sqlx::FromRow)] +struct ViolationRow { + id: Uuid, + kind: String, + predicate: Option, + left_fact: Uuid, + left_text: String, + right_fact: Uuid, + right_text: String, + path_len: i32, + detected_at: chrono::DateTime, + detail: serde_json::Value, + left_open: bool, + left_confidence: f32, + same_name_peers: bool, +} + +/// 线索按最常见的错法排（0017 §2）：旧断言没写结束日期、两个同名实体、抽取本来就 +/// 没把握。一次只给一条——三条并列等于没给 +fn hint_for(r: &ViolationRow) -> Option<&'static str> { + if r.left_open && r.detail.get("valid_from").is_some_and(|v| !v.is_null()) { + Some("stale") + } else if r.same_name_peers { + Some("duplicate") + } else if r.left_confidence < 0.75 { + Some("unsure") + } else { + None + } } /// 人裁决一处违规。 @@ -463,7 +700,8 @@ pub async fn check_ontology(pool: &PgPool, kb_id: Uuid) -> AppResult 'rules_disagree' + AND NOT (id = ANY($2))", ) .bind(kb_id) .bind(&fresh) @@ -497,6 +735,84 @@ pub struct DeriveReport { /// 而下游靠它推出的 `employs` 反倒进了库——一条派生的前提凭空消失。 /// 数出来，别再让它静默一次 pub unruled: usize, + /// 推出来了却撞上断言或别的派生、这一轮拦下没落的（0017）。**拦下的每一条 + /// 都在 Review 里有对应的一行**——`run` 与这里用同一个函数算 + pub blocked: usize, +} + +/// 一次取数，三样东西：带区间的边、每条事实的区间、精度与置信度。 +/// `run` 与 `materialize` 共用——两边看到的边必须是同一批 +type TimedEdges = ( + Vec, + HashMap, Option)>, + HashMap, Option, f32)>, +); + +async fn timed_edges(pool: &PgPool, kb_id: Uuid) -> AppResult { + // 输入**只有断言**。派生住在另一张表，所以这里连过滤都不必写——那正是 + // 分表买到的东西：忘了排除的后果是推不出东西，不是把自己的输出喂回自己 + let rows: Vec = sqlx::query_as( + "SELECT id, predicate_id, subject_id, object_id, + valid_from, valid_to, valid_from_precision, valid_to_precision, confidence + FROM facts + WHERE kb_id = $1 + AND invalidated_at IS NULL + AND predicate_id IS NOT NULL + AND object_id IS NOT NULL", + ) + .bind(kb_id) + .fetch_all(pool) + .await?; + + let mut edges = Vec::with_capacity(rows.len()); + let mut meta: HashMap, Option, f32)> = HashMap::new(); + let mut spans: HashMap, Option)> = HashMap::new(); + for (id, pred, subj, obj, from, to, fp, tp, conf) in rows { + let (f, t) = (from.map(|x| x.timestamp()), to.map(|x| x.timestamp())); + edges.push(TimedEdge { + edge: Edge { + fact: id, + predicate: pred, + subject: subj, + object: obj, + }, + from: f, + to: t, + }); + spans.insert(id, (f, t)); + meta.insert(id, (fp, tp, conf)); + } + Ok((edges, spans, meta)) +} + +/// 人认可过并存的（派生三元组, 断言）对：这些派生下一轮照常落地（0017 §2）。 +async fn accepted_clashes( + pool: &PgPool, + kb_id: Uuid, +) -> AppResult> { + let rows: Vec<(Uuid, serde_json::Value)> = sqlx::query_as( + "SELECT left_fact, detail FROM axiom_violations + WHERE kb_id = $1 AND kind = 'derived_contradiction' AND resolution = 'accepted'", + ) + .bind(kb_id) + .fetch_all(pool) + .await?; + let id = |v: &serde_json::Value, k: &str| { + v.get(k) + .and_then(|x| x.as_str()) + .and_then(|s| s.parse::().ok()) + }; + Ok(rows + .into_iter() + .filter_map(|(against, d)| { + Some(( + id(&d, "subject_id")?, + id(&d, "predicate_id")?, + id(&d, "object_id")?, + against, + )) + }) + .collect()) } /// 派生事实的身份：三元组 + 区间。 @@ -610,52 +926,40 @@ type LiveRow = ( pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult { let ax = axioms(pool, kb_id).await?; let rules = compile_rules(pool, kb_id, &ax).await?; - - // 输入**只有断言**。派生住在另一张表，所以这里连过滤都不必写——那正是 - // 分表买到的东西：忘了排除的后果是推不出东西，不是把自己的输出喂回自己 - let rows: Vec = sqlx::query_as( - "SELECT id, predicate_id, subject_id, object_id, - valid_from, valid_to, valid_from_precision, valid_to_precision, confidence - FROM facts - WHERE kb_id = $1 - AND invalidated_at IS NULL - AND predicate_id IS NOT NULL - AND object_id IS NOT NULL", - ) - .bind(kb_id) - .fetch_all(pool) - .await?; - - let mut edges = Vec::with_capacity(rows.len()); - let mut meta: HashMap, Option, f32)> = HashMap::new(); - let mut spans: HashMap, Option)> = HashMap::new(); - for (id, pred, subj, obj, from, to, fp, tp, conf) in rows { - let (f, t) = (from.map(|x| x.timestamp()), to.map(|x| x.timestamp())); - edges.push(TimedEdge { - edge: Edge { - fact: id, - predicate: pred, - subject: subj, - object: obj, - }, - from: f, - to: t, - }); - spans.insert(id, (f, t)); - meta.insert(id, (fp, tp, conf)); - } + let (edges, spans, meta) = timed_edges(pool, kb_id).await?; let derivation = utopia_reason::derive::derive(&edges, &ax); + // asserted > derived 是硬性的（0002）：撞上断言的派生不落地。人认可过并存的 + // 除外；派生之间互撞的两边都不落，认可与否只影响报不报（0017） + let clashes = utopia_reason::derive::contradictions(&derivation, &edges, &ax, &spans); + let accepted = accepted_clashes(pool, kb_id).await?; + let mut blocked: HashSet = HashSet::new(); + for c in &clashes.with_assertions { + let d = &derivation.facts[c.derived]; + if !accepted.contains(&(d.subject, d.predicate, d.object, c.against)) { + blocked.insert(c.derived); + } + } + for rc in &clashes.between_derivations { + for (i, j) in &rc.pairs { + blocked.insert(*i); + blocked.insert(*j); + } + } let mut report = DeriveReport { rules: rules.len(), edges: edges.len(), derived: derivation.facts.len(), capped: derivation.capped.len(), + blocked: blocked.len(), ..Default::default() }; let mut wanted: HashMap = HashMap::new(); - for d in &derivation.facts { + for (i, d) in derivation.facts.iter().enumerate() { + if blocked.contains(&i) { + continue; + } let Some((from, to)) = utopia_reason::derive::validity(&d.premises, &spans) else { continue; }; @@ -776,9 +1080,9 @@ pub async fn open_defects( offset: i64, ) -> AppResult> { Ok(sqlx::query_as( - "SELECT d.id, d.kind, + "SELECT d.id, d.kind, d.detail, COALESCE(st.label, sr.label) AS subject_label, - ot.label AS other_label, + COALESCE(ot.label, orr.label) AS other_label, COALESCE( (SELECT array_agg(t.label ORDER BY x.ord) FROM unnest(d.path) WITH ORDINALITY AS x(id, ord) @@ -790,6 +1094,7 @@ pub async fn open_defects( LEFT JOIN entity_types st ON st.id = d.subject LEFT JOIN relation_types sr ON sr.id = d.subject LEFT JOIN entity_types ot ON ot.id = d.other + LEFT JOIN relation_types orr ON orr.id = d.other WHERE d.kb_id = $1 AND d.status = 'open' ORDER BY d.detected_at DESC LIMIT $2 OFFSET $3", @@ -857,6 +1162,204 @@ pub async fn mark_inference_ran(pool: &PgPool, kb_id: Uuid) -> AppResult<()> { Ok(()) } +/// 一条派生事实的证明，展开到原句（0002 R2）。 +/// +/// `fact_derivations` 只记直接前提，而前提一律是断言，所以「递归展开」在这里 +/// 退化成一条链：派生 → 按 `seq` 的断言 → 每条断言的证据。叶子是 chunk， +/// 界面上一路点到文档。**撤了的前提照样列出并打上标记**：派生随前提失效， +/// 但「当时靠的是什么」要读得出来，那正是记录轴存在的理由。 +/// +/// 派生已失效或不存在时回 None——不是错误，界面据此收起。 +pub async fn proof( + pool: &PgPool, + kb_id: Uuid, + derived_id: Uuid, +) -> AppResult> { + let Some(derived) = derived_one(pool, kb_id, derived_id).await? else { + return Ok(None); + }; + let premises: Vec = sqlx::query_scalar( + "SELECT premise_fact_id FROM fact_derivations WHERE derived_fact_id = $1 ORDER BY seq", + ) + .bind(derived_id) + .fetch_all(pool) + .await?; + let steps = steps_for(pool, &premises).await?; + Ok(Some(utopia_core::models::Proof { derived, steps })) +} + +/// 一串前提展开成证明的步：三元组、区间、撤没撤、证据。 +/// +/// 落了地的派生（`fact_derivations`）与没落地的（`axiom_violations.path`）都从这里 +/// 走——前提是同一种东西，证明链没有理由长两个样 +async fn steps_for( + pool: &PgPool, + premises: &[Uuid], +) -> AppResult> { + #[allow(clippy::type_complexity)] + let rows: Vec<( + i64, + Uuid, + Uuid, + String, + Option, + Option, + Option, + Option, + Option>, + Option>, + f32, + bool, + )> = sqlx::query_as( + "SELECT x.ord - 1, f.id, f.subject_id, s.canonical_name, + f.predicate_id, r.label, f.object_id, o.canonical_name, + f.valid_from, f.valid_to, f.confidence, + f.invalidated_at IS NOT NULL + FROM unnest($1::uuid[]) WITH ORDINALITY AS x(id, ord) + JOIN facts f ON f.id = x.id + JOIN entities s ON s.id = f.subject_id + LEFT JOIN relation_types r ON r.id = f.predicate_id + LEFT JOIN entities o ON o.id = f.object_id + ORDER BY x.ord", + ) + .bind(premises) + .fetch_all(pool) + .await?; + let mut steps = Vec::with_capacity(rows.len()); + for ( + seq, + fact_id, + subject_id, + subject, + predicate_id, + predicate, + object_id, + object, + valid_from, + valid_to, + confidence, + retracted, + ) in rows + { + // 一条链最多 MAX_DEPTH 步，逐条取证据是可数的几次往返 + let evidence = crate::graph::fact_evidence(pool, fact_id).await?; + steps.push(utopia_core::models::ProofStep { + seq: seq as i32, + fact_id, + subject_id, + subject, + predicate_id, + predicate, + object_id, + object, + valid_from, + valid_to, + confidence, + retracted, + evidence, + }); + } + Ok(steps) +} + +/// 没落地的派生里，与这个实体有关的那些（0017 §3）——面板「推出来的」一档的 +/// 「没落地的」小节。 +pub async fn blocked_for_entity( + pool: &PgPool, + kb_id: Uuid, + entity_id: Uuid, +) -> AppResult> { + Ok(sqlx::query_as( + "SELECT v.id AS violation_id, + (v.detail->>'subject_id')::uuid AS subject_id, + COALESCE(v.detail->>'subject', '?') AS subject, + (v.detail->>'object_id')::uuid AS object_id, + COALESCE(v.detail->>'object', '?') AS object, + COALESCE(v.detail->>'predicate', '?') AS predicate, + COALESCE(v.detail->>'rule', '?') AS rule, + COALESCE(v.detail->>'via_label', '?') AS via_label, + (v.detail->>'valid_from')::timestamptz AS valid_from, + (v.detail->>'valid_to')::timestamptz AS valid_to, + v.left_fact AS against_fact, + s.canonical_name || ' · ' + || COALESCE(r.label, fact_surface_predicate(f.id), '?') || ' · ' + || COALESCE(o.canonical_name, '?') AS against_text, + v.path AS premises + FROM axiom_violations v + JOIN facts f ON f.id = v.left_fact + JOIN entities s ON s.id = f.subject_id + LEFT JOIN relation_types r ON r.id = f.predicate_id + LEFT JOIN entities o ON o.id = f.object_id + WHERE v.kb_id = $1 AND v.kind = 'derived_contradiction' AND v.status = 'open' + AND (v.detail->>'subject_id' = $2::text OR v.detail->>'object_id' = $2::text) + ORDER BY v.detected_at DESC", + ) + .bind(kb_id) + .bind(entity_id) + .fetch_all(pool) + .await?) +} + +/// 没落地的派生的证明链：它的前提就在违规的 `path` 里。找不到那条违规时 `None` +pub async fn blocked_proof( + pool: &PgPool, + kb_id: Uuid, + violation_id: Uuid, +) -> AppResult>> { + let path: Option<(Vec,)> = sqlx::query_as( + "SELECT path FROM axiom_violations + WHERE id = $1 AND kb_id = $2 AND kind = 'derived_contradiction'", + ) + .bind(violation_id) + .bind(kb_id) + .fetch_optional(pool) + .await?; + match path { + None => Ok(None), + Some((p,)) => Ok(Some(steps_for(pool, &p).await?)), + } +} + +/// 按 id 取一条派生（失效的也取：证明要能回看）。 +async fn derived_one( + pool: &PgPool, + kb_id: Uuid, + derived_id: Uuid, +) -> AppResult> { + Ok(sqlx::query_as( + "SELECT d.id, + d.subject_id, s.canonical_name AS subject, + d.object_id, o.canonical_name AS object, + r.label AS predicate, + ru.kind AS rule, + d.valid_from, d.valid_to, d.confidence, d.derived_at, + COALESCE( + (SELECT array_agg( + ps.canonical_name || ' · ' + || COALESCE(pr.label, '?') || ' · ' + || COALESCE(po.canonical_name, '?') + ORDER BY fd.seq) + FROM fact_derivations fd + JOIN facts pf ON pf.id = fd.premise_fact_id + JOIN entities ps ON ps.id = pf.subject_id + LEFT JOIN relation_types pr ON pr.id = pf.predicate_id + LEFT JOIN entities po ON po.id = pf.object_id + WHERE fd.derived_fact_id = d.id), + ARRAY[]::text[] + ) AS premises + FROM derived_facts d + JOIN entities s ON s.id = d.subject_id + JOIN entities o ON o.id = d.object_id + JOIN relation_types r ON r.id = d.predicate_id + JOIN rules ru ON ru.id = d.rule_id + WHERE d.kb_id = $1 AND d.id = $2", + ) + .bind(kb_id) + .bind(derived_id) + .fetch_optional(pool) + .await?) +} + /// 一条派生事实，配好展示与证明所需的文本（实体面板的「推出来的」那一档）。 /// /// **证明一起取回来**：这一档存在的理由就是「这条边不是谁说的，是这么推出来的」， diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs index f82377d9..9d886a66 100644 --- a/crates/utopia-store/src/resolution.rs +++ b/crates/utopia-store/src/resolution.rs @@ -107,10 +107,12 @@ pub fn recall_keys(name: &str) -> Vec { /// 易混具体类型：抽取常在这几类间摇摆（一个团队算组织还是项目？平台算项目还是产品？）。 /// 同名跨这组类型 → 照建实体（宁分勿合），但入队审核对交 LLM/人工裁决。 /// -/// **这张表该从本体读，今天还没有。** `owl:disjointWith` 已经落库（`entity_type_disjoint`， -/// 有导入有编辑），但消费者只有推理机的本体自检；消解这边仍按这三个硬编码的 key 判。 -/// 没装包的库里这三个 key 不存在，于是这一档永不命中，所有跨类型同名判 `Disjoint`—— -/// 变严不变松，不会错合。改从本体读是 0016 的 B3。 +/// **本体说了算，这张表只是没声明时的退路。** 判两个类能不能指同一个东西，先看 +/// `owl:disjointWith`（`entity_type_disjoint`，含继承：Person ⟂ Organization 就让 +/// Corporation ⟂ Person）——声明了互斥的一律分开，哪怕它们在类层级上是一家； +/// 没声明的再看类层级（同一支系当易混，#226），最后才是这三个硬 key。 +/// 没装包也没声明的库里这三个 key 不存在，于是所有跨类型同名判 `Disjoint`—— +/// 变严不变松，不会错合（0016 B3） pub const CONFUSABLE_TYPE_KEYS: &[&str] = &["organization", "project", "product"]; /// 单次消解最多入队的漂移审核对（防同名大组刷爆审核队列）。 @@ -352,6 +354,9 @@ const CONTAIN_SCAN_LIMIT: i64 = 16; /// 所以这里靠 `kb_id` 收窄行集并设上限，且只在**新建实体时**跑一次， /// 不是每条 mention。大库上如果不够，正解是建一张「后缀键」表走等值查， /// 而不是加模糊索引。 +/// 包含扫描的一行：(id, 本名, 类型 key, 类型 id, 画像)。类型 id 用来比本体声明的互斥 +type ContainRow = (Uuid, String, Option, Option, Option); + async fn containment_reviews( pool: &PgPool, kb_id: Uuid, @@ -379,9 +384,11 @@ async fn containment_reviews( // 启明 X7 加速卡→concept vs 启明 X7 推理加速卡→product）， // 按类型相等去查，这两对一个都捞不到。相容性交给下面的 classify_type_drift。 // 多取一些行，因为硬互斥的会在 Rust 侧被筛掉 + // 本体声明了跟这个类互斥的那些类（含继承），一次取出，逐行比 id + let disjoint = declared_disjoint_from(pool, kb_id, type_id).await?; // 第三列可空：未分类实体也要参与包含关系扫描（0009） - let rows: Vec<(Uuid, String, Option, Option)> = sqlx::query_as( - "SELECT e.id, e.canonical_name, t.key, e.profile_embedding + let rows: Vec = sqlx::query_as( + "SELECT e.id, e.canonical_name, t.key, e.type_id, e.profile_embedding FROM entities e LEFT JOIN entity_types t ON t.id = e.type_id WHERE e.kb_id = $1 AND e.merged_into IS NULL AND e.id <> $2 @@ -419,12 +426,15 @@ async fn containment_reviews( Ok(rows .into_iter() // 哪些类型对可能指同一个东西，既有规则已经想清楚了，别另发明一套： - // person vs organization 永不合并，concept 兜底与谁都可能是一个 - .filter(|(_, _, type_key, _)| { - classify_type_drift(mention_key.as_deref(), type_key.as_deref()) != TypeDrift::Disjoint + // 本体声明互斥的永不合并，person vs organization 永不合并， + // concept 兜底与谁都可能是一个 + .filter(|(_, _, type_key, other_type, _)| { + !other_type.is_some_and(|t| disjoint.contains(&t)) + && classify_type_drift(mention_key.as_deref(), type_key.as_deref()) + != TypeDrift::Disjoint }) .take(MAX_CONTAIN_REVIEWS) - .map(|(id, other_name, _, emb)| { + .map(|(id, other_name, _, _, emb)| { // 分数只是给队列排序用的参考，**不参与是否合并的判断**—— // 那个判断本来就不在这条路上 let score = ctx @@ -466,6 +476,43 @@ fn drift_reason(mention_key: Option<&str>, other_key: Option<&str>, sim: Option< } } +/// 本体声明了跟这个类互斥的全部类（0016 B3）。 +/// +/// **互斥是继承的**：Person ⟂ Organization 一条声明，就让 Person 的每个子类跟 +/// Organization 的每个子类都互斥。所以先沿父链往上收集这个类的祖先，取它们声明的 +/// 互斥对象，再沿子链往下展开。表里两个方向各存一行，问一个方向就够。 +/// +/// 没判出类型（`None`）时没有类可问，回空集：那一侧本来就走召回候选那一档 +async fn declared_disjoint_from( + pool: &PgPool, + kb_id: Uuid, + type_id: Option, +) -> AppResult> { + let Some(type_id) = type_id else { + return Ok(HashSet::new()); + }; + let rows: Vec<(Uuid,)> = sqlx::query_as( + "WITH RECURSIVE up(id) AS ( + SELECT $2::uuid + UNION + SELECT p.parent_id FROM entity_type_parents p JOIN up ON p.child_id = up.id + ), hit(id) AS ( + SELECT d.b_id FROM entity_type_disjoint d JOIN up ON d.a_id = up.id + WHERE d.kb_id = $1 + ), down(id) AS ( + SELECT id FROM hit + UNION + SELECT p.child_id FROM entity_type_parents p JOIN down ON p.parent_id = down.id + ) + SELECT id FROM down", + ) + .bind(kb_id) + .bind(type_id) + .fetch_all(pool) + .await?; + Ok(rows.into_iter().map(|(id,)| id).collect()) +} + /// 两个类是不是一家的：一方是另一方的祖先，或者两者共有一个**不是根**的祖先。 /// /// `CONFUSABLE_TYPE_KEYS` 那张三 key 的硬表是给没装包的库准备的；装了 schema.org @@ -561,12 +608,18 @@ async fn resolve_type_drift( .fetch_all(pool) .await?; + // 本体声明了互斥的类，一次取出（含继承）。声明优先于下面所有启发式 + let disjoint = declared_disjoint_from(pool, kb_id, type_id).await?; let mut recall_cands: Vec<&CrossCandidate> = Vec::new(); let mut review_cands: Vec<&CrossCandidate> = Vec::new(); for c in &cross { let mut drift = classify_type_drift(mention_key.as_deref(), c.type_key.as_deref()); - // 硬表判不上的，再看类层级：同一支系下的同名当易混，进审阅队列 - if drift == TypeDrift::Disjoint { + if c.type_id.is_some_and(|t| disjoint.contains(&t)) { + // 本体说这两类互斥：哪怕硬表说易混、类层级说一家，也分开。 + // 声明是人写下的判断，启发式只是没声明时的猜测 + drift = TypeDrift::Disjoint; + } else if drift == TypeDrift::Disjoint { + // 硬表判不上的，再看类层级：同一支系下的同名当易混，进审阅队列 if let (Some(a), Some(b)) = (type_id, c.type_id) { if types_are_kin(pool, a, b).await? { drift = TypeDrift::Review; diff --git a/crates/utopia-store/src/sources.rs b/crates/utopia-store/src/sources.rs index f0ca22f7..ab7add38 100644 --- a/crates/utopia-store/src/sources.rs +++ b/crates/utopia-store/src/sources.rs @@ -2,28 +2,22 @@ use chrono::{DateTime, Utc}; use sqlx::PgPool; -use utopia_core::models::{Role, Source, SourceView, SyncRun}; +use utopia_core::models::{Role, Source, SourceKind, SourceView, SyncRun, SOURCE_SECRET_KEYS}; use utopia_core::{AppError, AppResult}; use uuid::Uuid; /// folder = 纯容器（上传/拖拽入内，无同步语义）；url/rss = 拉取型；api = 推送型。 /// 本机目录监听（watch_folder）已否决——自部署用户看不到服务器磁盘； -/// 未来的 watch 形态是对象存储/网盘（P5 连接器，与 BlobStore 接缝配套）。 +/// 对象存储 / WebDAV / Notion 是它的替代形态（0013）。 /// custom = 自定义拉取器：任何实现 Utopia ingest 接口的 URL（返回 items JSON）即可定时摄取。 /// github_issues / jira_issues = 工单：一张工单连同它的状态变更史成为一篇文档。 /// -/// **改这里就得改前端那份清单**（`Library.tsx` 的建来源对话框与 `api.ts` 的 -/// `SourceView["kind"]`）。两处对不上时的症状是：界面上选得到、建的时候报 -/// 「kind must be one of…」——单元测试与 tsc 都看不见，只有端到端会撞上。 -pub const KINDS: &[&str] = &[ - "folder", - "url", - "rss", - "api", - "custom", - "github_issues", - "jira_issues", -]; +/// 种类的清单**不在这里写**：`SourceKind`（utopia-core）一个枚举出全部——创建的白名单、 +/// 同步的分派、前端的下拉框（有测试对表）。从前这里有一张手写的 `KINDS`，五种连接器 +/// 加了同步却没进这张表，界面上选得到、建不出来（#247） +pub fn creatable_kinds() -> Vec<&'static str> { + SourceKind::creatable().map(|k| k.as_str()).collect() +} /// 校验并规范化标准 5 段 cron 表达式（内部用 cron crate 的 6 段：补秒位）。 pub fn validate_cron(expr: &str) -> AppResult { @@ -55,9 +49,11 @@ fn cron_next_after(expr: &str, after: DateTime) -> Option> { } pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult> { - // config 剔除 auth_header：自定义拉取器的凭据不下发给任何客户端 + // config 剔掉凭据：列表给 Viewer 看，哪一种连接器的密钥都不下发。 + // 键在 `SOURCE_SECRET_KEYS` 一张表上——从前这里只减 `auth_header`，五种连接器 + // 的密钥就这么漏出去的（#246） let rows: Vec = sqlx::query_as( - "SELECT s.id, s.kind, s.name, s.config - 'auth_header' AS config, s.icon, + "SELECT s.id, s.kind, s.name, s.config - $2::text[] AS config, s.icon, s.sync_interval_minutes, s.sync_cron, s.last_sync_at, s.last_sync_status, s.last_sync_error, s.last_sync_added, (SELECT count(*) FROM documents d WHERE d.source_id = s.id) AS doc_count, @@ -66,6 +62,7 @@ pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult> { FROM sources s WHERE s.kb_id = $1 ORDER BY s.created_at", ) .bind(kb_id) + .bind(SOURCE_SECRET_KEYS) .fetch_all(pool) .await?; Ok(rows) @@ -90,10 +87,10 @@ pub async fn create( sync_interval_minutes: Option, sync_cron: Option<&str>, ) -> AppResult { - if !KINDS.contains(&kind) { + if !SourceKind::parse(kind).is_some_and(|k| k.creatable_by_hand()) { return Err(AppError::Validation(format!( "kind must be one of: {}", - KINDS.join(", ") + creatable_kinds().join(", ") ))); } if name.trim().is_empty() { diff --git a/crates/utopia-store/tests/a_contradiction_points_upstream.rs b/crates/utopia-store/tests/a_contradiction_points_upstream.rs new file mode 100644 index 00000000..04c1020c --- /dev/null +++ b/crates/utopia-store/tests/a_contradiction_points_upstream.rs @@ -0,0 +1,384 @@ +//! 0017：派生撞上断言时，让路这件事从静默变成可见。 +//! +//! `ceo_of ⊑ works_at`，`works_at` functional。Mira `ceo_of` Acme 推出 Mira `works_at` +//! Acme，而账本里说她 `works_at` Globex。这里守四件事： +//! +//! 1. **派生不落地，而队列里有一行。** `run` 记一条 `derived_contradiction`，left 是被撞 +//! 的断言，right 是最后一条前提，detail 写着推出来的三元组；`materialize` 拦下它。 +//! 2. **修了就落。** 给旧断言一个结束日期，派生的区间与它不再重叠，下一轮落地， +//! 队列里那一行随之清掉。 +//! 3. **认可就落。** 人说两边都对，`accepted` 之后派生照常落地，那一行留着不再报。 +//! 4. **派生之间互撞按规则对聚合。** 两个 ceo 推出两条互斥的 works_at，进 +//! `ontology_defects` 一行 `rules_disagree`，两条派生都不落。 +//! +//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆，绝不碰已有的库。 + +use sqlx::PgPool; +use utopia_store::reasoning; +use uuid::Uuid; + +struct Fixture { + org: Uuid, + user: Uuid, + kb: Uuid, + ceo_of: Uuid, + works_at: Uuid, + mira: Uuid, + acme: Uuid, + globex: Uuid, + initech: Uuid, +} + +async fn seed(pool: &PgPool) -> anyhow::Result { + let (org, ws, kb, user) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + let etype = Uuid::now_v7(); + let (ceo_of, works_at) = (Uuid::now_v7(), Uuid::now_v7()); + let (mira, acme, globex, initech) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'contradiction-test')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'contradiction-test')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO users (id, org_id, email, display_name, password_hash) + VALUES ($1, $2, $1 || '@contradiction.test', 'c', 'x')", + ) + .bind(user) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases (id, workspace_id, name, materialize_inferences) + VALUES ($1, $2, 'contradiction-test', TRUE)", + ) + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, 'thing', 'Thing')", + ) + .bind(etype) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO relation_types (id, kb_id, key, label, functional) + VALUES ($1, $2, 'works_at', 'works at', TRUE)", + ) + .bind(works_at) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO relation_types (id, kb_id, key, label, sub_property_of) + VALUES ($1, $2, 'ceo_of', 'CEO of', $3)", + ) + .bind(ceo_of) + .bind(kb) + .bind(works_at) + .execute(pool) + .await?; + for (id, name) in [ + (mira, "Mira"), + (acme, "Acme"), + (globex, "Globex"), + (initech, "Initech"), + ] { + sqlx::query( + "INSERT INTO entities (id, kb_id, type_id, canonical_name) VALUES ($1, $2, $3, $4)", + ) + .bind(id) + .bind(kb) + .bind(etype) + .bind(name) + .execute(pool) + .await?; + } + Ok(Fixture { + org, + user, + kb, + ceo_of, + works_at, + mira, + acme, + globex, + initech, + }) +} + +async fn asserted( + pool: &PgPool, + f: &Fixture, + subject: Uuid, + predicate: Uuid, + object: Uuid, + from: Option<&str>, +) -> anyhow::Result { + let id = Uuid::now_v7(); + sqlx::query( + "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, confidence, + valid_from, valid_from_precision) + VALUES ($1, $2, $3, $4, $5, 0.9, $6::timestamptz, CASE WHEN $6 IS NULL THEN NULL ELSE 'day' END)", + ) + .bind(id) + .bind(f.kb) + .bind(subject) + .bind(predicate) + .bind(object) + .bind(from) + .execute(pool) + .await?; + Ok(id) +} + +async fn live_derived(pool: &PgPool, f: &Fixture) -> anyhow::Result> { + Ok(sqlx::query_as( + "SELECT subject_id, object_id FROM derived_facts + WHERE kb_id = $1 AND invalidated_at IS NULL ORDER BY subject_id, object_id", + ) + .bind(f.kb) + .fetch_all(pool) + .await?) +} + +async fn open_contradictions( + pool: &PgPool, + f: &Fixture, +) -> anyhow::Result> { + Ok(sqlx::query_as( + "SELECT id, left_fact, detail FROM axiom_violations + WHERE kb_id = $1 AND kind = 'derived_contradiction' AND status = 'open'", + ) + .bind(f.kb) + .fetch_all(pool) + .await?) +} + +#[tokio::test] +async fn a_contradiction_points_upstream() -> anyhow::Result<()> { + let Ok(url) = std::env::var("UTOPIA_DATABASE_URL") else { + eprintln!("跳过：未设 UTOPIA_DATABASE_URL"); + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let f = seed(&pool).await?; + + let run = async { + // Mira works_at Globex（没写结束日期）；Mira ceo_of Acme 自 2024 起 + let old = asserted(&pool, &f, f.mira, f.works_at, f.globex, Some("2020-01-01")).await?; + let ceo = asserted(&pool, &f, f.mira, f.ceo_of, f.acme, Some("2024-01-01")).await?; + + // 1. 派生不落地，队列里有一行 + let m = reasoning::materialize(&pool, f.kb).await?; + assert_eq!(m.derived, 1); + assert_eq!( + m.blocked, 1, + "the derivation that hits an assertion stays out" + ); + assert_eq!(m.inserted, 0); + assert!(live_derived(&pool, &f).await?.is_empty()); + + let r = reasoning::run(&pool, f.kb).await?; + assert_eq!(r.contradictions, 1); + assert_eq!(r.rules_disagree, 0); + let rows = open_contradictions(&pool, &f).await?; + assert_eq!(rows.len(), 1); + let (vid, left, detail) = &rows[0]; + assert_eq!(*left, old, "left is the assertion that was hit"); + assert_eq!(detail["axiom"], "functional"); + assert_eq!(detail["rule"], "sub_property"); + assert_eq!(detail["subject"], "Mira"); + assert_eq!(detail["predicate"], "works at"); + assert_eq!(detail["object"], "Acme"); + assert_eq!(detail["via_label"], "CEO of"); + assert_eq!(detail["premises"][0], serde_json::json!(ceo)); + let (right,): (Uuid,) = + sqlx::query_as("SELECT right_fact FROM axiom_violations WHERE id = $1") + .bind(vid) + .fetch_one(&pool) + .await?; + assert_eq!(right, ceo, "right is the last premise"); + + // Review 给的线索：旧断言没写结束日期、派生起得更晚 → stale + let page = reasoning::open_violations(&pool, f.kb, 50, 0).await?; + let card = page + .iter() + .find(|v| v.id == *vid) + .expect("card on the page"); + assert_eq!(card.hint.as_deref(), Some("stale")); + assert_eq!(card.detail["subject"], "Mira"); + + // 争议在它坐的地方可见（0017 §3）：面板行挂 contested，图上有一条幽灵边， + // 「没落地的」一档有一行，它的证明链读得出前提 + let (_, facts) = utopia_store::graph::entity_detail(&pool, f.kb, f.mira).await?; + let hit = facts + .iter() + .find(|x| x.id == old) + .expect("the assertion is on the panel"); + let c = hit + .contested + .as_ref() + .expect("the hit assertion is contested"); + assert_eq!(c["kind"], "derived_contradiction"); + assert_eq!(c["ref_id"], serde_json::json!(vid)); + assert!( + facts + .iter() + .find(|x| x.id == ceo) + .unwrap() + .contested + .is_none(), + "the premise is not the disputed one" + ); + let (_, edges) = utopia_store::graph::neighborhood(&pool, f.kb, f.mira, 1, None).await?; + let ghost = edges + .iter() + .find(|e| e.blocked) + .expect("a ghost edge for the blocked derivation"); + assert_eq!(ghost.id, *vid); + assert!(ghost.derived && ghost.contested); + assert_eq!((ghost.source, ghost.target), (f.mira, f.acme)); + assert!(edges.iter().find(|e| e.id == old).unwrap().contested); + assert!(!edges.iter().find(|e| e.id == ceo).unwrap().contested); + let blocked = reasoning::blocked_for_entity(&pool, f.kb, f.acme).await?; + assert_eq!(blocked.len(), 1); + assert_eq!(blocked[0].violation_id, *vid); + assert_eq!(blocked[0].against_fact, old); + assert_eq!(blocked[0].premises, vec![ceo]); + let steps = reasoning::blocked_proof(&pool, f.kb, *vid) + .await? + .expect("the ghost has a proof"); + assert_eq!(steps.len(), 1); + assert_eq!(steps[0].fact_id, ceo); + + // 重跑幂等：还是那一行 + reasoning::run(&pool, f.kb).await?; + assert_eq!(open_contradictions(&pool, &f).await?.len(), 1); + + // 2. 修了就落：给旧断言一个结束日期，区间不再重叠 + sqlx::query( + "UPDATE facts SET valid_to = '2023-06-30'::timestamptz, valid_to_precision = 'day' + WHERE id = $1", + ) + .bind(old) + .execute(&pool) + .await?; + let m = reasoning::materialize(&pool, f.kb).await?; + assert_eq!(m.blocked, 0); + assert_eq!( + m.inserted, 1, + "once the assertion ends, the derivation lands" + ); + assert_eq!(live_derived(&pool, &f).await?, vec![(f.mira, f.acme)]); + reasoning::run(&pool, f.kb).await?; + assert!( + open_contradictions(&pool, &f).await?.is_empty(), + "the queue row clears with the contradiction" + ); + + // 3. 认可就落：把结束日期拿掉，矛盾回来；人说两边都对，派生照常落地 + sqlx::query("UPDATE facts SET valid_to = NULL, valid_to_precision = NULL WHERE id = $1") + .bind(old) + .execute(&pool) + .await?; + let m = reasoning::materialize(&pool, f.kb).await?; + assert_eq!(m.blocked, 1); + assert_eq!(m.invalidated, 1, "the landed derivation is withdrawn again"); + reasoning::run(&pool, f.kb).await?; + let rows = open_contradictions(&pool, &f).await?; + assert_eq!(rows.len(), 1); + reasoning::decide(&pool, f.kb, rows[0].0, "accepted", f.user).await?; + let m = reasoning::materialize(&pool, f.kb).await?; + assert_eq!(m.blocked, 0, "an accepted pair lands"); + assert_eq!(live_derived(&pool, &f).await?, vec![(f.mira, f.acme)]); + let r = reasoning::run(&pool, f.kb).await?; + assert_eq!(r.contradictions, 1, "still counted"); + assert!( + open_contradictions(&pool, &f).await?.is_empty(), + "but the accepted row stays resolved and nothing new is opened" + ); + + // 4. 派生之间互撞：先把旧断言闭合掉，让断言不再参与；再来一个 ceo_of Initech， + // 两条 works_at 由同一条规则推出、互斥——按规则对报一次，两条都不落 + sqlx::query( + "UPDATE facts SET valid_to = '2023-06-30'::timestamptz, valid_to_precision = 'day' + WHERE id = $1", + ) + .bind(old) + .execute(&pool) + .await?; + let ceo2 = asserted(&pool, &f, f.mira, f.ceo_of, f.initech, Some("2024-01-01")).await?; + let m = reasoning::materialize(&pool, f.kb).await?; + assert_eq!(m.derived, 2); + assert_eq!(m.blocked, 2, "both sides of a rule clash stay out"); + assert_eq!(m.invalidated, 1, "the one that had landed is withdrawn"); + assert!(live_derived(&pool, &f).await?.is_empty()); + let r = reasoning::run(&pool, f.kb).await?; + assert_eq!(r.contradictions, 0); + assert_eq!(r.rules_disagree, 1); + let defects: Vec<(Uuid, Option, serde_json::Value)> = sqlx::query_as( + "SELECT subject, other, detail FROM ontology_defects + WHERE kb_id = $1 AND kind = 'rules_disagree' AND status = 'open'", + ) + .bind(f.kb) + .fetch_all(&pool) + .await?; + assert_eq!(defects.len(), 1); + assert_eq!(defects[0].0, f.ceo_of); + assert_eq!(defects[0].1, Some(f.ceo_of)); + assert_eq!(defects[0].2["count"], 1); + assert_eq!(defects[0].2["rules"][0]["axiom"], "functional"); + assert_eq!(defects[0].2["rules"][0]["rule_a"], "sub_property"); + assert_eq!(defects[0].2["rules"][0]["via_a"], "CEO of"); + let page = reasoning::open_defects(&pool, f.kb, 50, 0).await?; + let card = page + .iter() + .find(|d| d.kind == "rules_disagree") + .expect("the rule clash is on the page"); + assert_eq!(card.subject_label.as_deref(), Some("CEO of")); + assert_eq!(card.other_label.as_deref(), Some("CEO of")); + + // 撤掉第二个 ceo：规则对的那一行清掉，第一条派生重新落地 + sqlx::query("UPDATE facts SET invalidated_at = now() WHERE id = $1") + .bind(ceo2) + .execute(&pool) + .await?; + reasoning::run(&pool, f.kb).await?; + let (n,): (i64,) = sqlx::query_as( + "SELECT count(*) FROM ontology_defects + WHERE kb_id = $1 AND kind = 'rules_disagree' AND status = 'open'", + ) + .bind(f.kb) + .fetch_one(&pool) + .await?; + assert_eq!(n, 0, "a rule clash clears when its derivations go"); + let m = reasoning::materialize(&pool, f.kb).await?; + assert_eq!(m.blocked, 0); + assert_eq!(live_derived(&pool, &f).await?, vec![(f.mira, f.acme)]); + anyhow::Ok(()) + } + .await; + + let _ = sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(f.org) + .execute(&pool) + .await; + run +} diff --git a/crates/utopia-store/tests/a_declared_disjointness_keeps_names_apart.rs b/crates/utopia-store/tests/a_declared_disjointness_keeps_names_apart.rs new file mode 100644 index 00000000..112f8323 --- /dev/null +++ b/crates/utopia-store/tests/a_declared_disjointness_keeps_names_apart.rs @@ -0,0 +1,198 @@ +//! 0016 B3：`owl:disjointWith` 进消解——本体声明了互斥的两个类，同名也不进审阅队列。 +//! +//! 消解判「两个类能不能指同一个东西」有三层：硬表 `CONFUSABLE_TYPE_KEYS`、类层级 +//! （同一支系当易混，#226）、本体声明的互斥。这里守的是**声明优先于前两层**： +//! +//! 1. 没声明时行为不变：organization vs project 照硬表进队列，corporation vs +//! federal_agency 照类层级（共有非根祖先 organization）进队列。 +//! 2. 声明 organization ⟂ project 之后，同名的 organization / project 分开，不进队列。 +//! 3. 声明 corporation ⟂ agency 之后，federal_agency（agency 的子类）跟 corporation +//! 也分开——**互斥是继承的**，声明在父类上就够。 +//! +//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆，绝不碰已有的库。 + +use sqlx::PgPool; +use utopia_store::{ontology, resolution}; +use uuid::Uuid; + +struct Fx { + org: Uuid, + kb: Uuid, + organization: Uuid, + project: Uuid, + corporation: Uuid, + agency: Uuid, + federal_agency: Uuid, +} + +async fn seed(pool: &PgPool) -> anyhow::Result { + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + // 顶上要有一个根：类层级那条规则里「共有的祖先」不算根（schema.org 里万物皆 + // Thing，算上它 Person 与 Organization 也成了一家），所以 organization 得有父类 + let (thing, organization, project, corporation, agency, federal_agency) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'disjoint-test')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'disjoint-test')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'disjoint-test')", + ) + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + for (id, key) in [ + (thing, "thing"), + (organization, "organization"), + (project, "project"), + (corporation, "corporation"), + (agency, "agency"), + (federal_agency, "federal_agency"), + ] { + sqlx::query("INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, $3, $3)") + .bind(id) + .bind(kb) + .bind(key) + .execute(pool) + .await?; + } + for (child, parent) in [ + (organization, thing), + (project, thing), + (corporation, organization), + (agency, organization), + (federal_agency, agency), + ] { + sqlx::query("INSERT INTO entity_type_parents (child_id, parent_id) VALUES ($1, $2)") + .bind(child) + .bind(parent) + .execute(pool) + .await?; + } + Ok(Fx { + org, + kb, + organization, + project, + corporation, + agency, + federal_agency, + }) +} + +async fn entity(pool: &PgPool, f: &Fx, name: &str, type_id: Uuid) -> anyhow::Result { + let id = Uuid::now_v7(); + sqlx::query( + "INSERT INTO entities (id, kb_id, type_id, canonical_name) VALUES ($1, $2, $3, $4)", + ) + .bind(id) + .bind(f.kb) + .bind(type_id) + .bind(name) + .execute(pool) + .await?; + Ok(id) +} + +/// 消解一条 mention，回它挂上的「类型漂移」审核对指向谁 +async fn drift_reviews( + pool: &PgPool, + f: &Fx, + name: &str, + type_id: Uuid, +) -> anyhow::Result> { + let r = resolution::resolve_mention(pool, f.kb, Some(type_id), name, None).await?; + assert!( + r.created, + "a cross-type same name is a new entity: keep apart, never merge" + ); + Ok(r.reviews + .iter() + .filter(|x| x.reason.starts_with("type_drift|")) + .map(|x| x.other_id) + .collect()) +} + +#[tokio::test] +async fn a_declared_disjointness_keeps_names_apart() -> anyhow::Result<()> { + let Ok(url) = std::env::var("UTOPIA_DATABASE_URL") else { + eprintln!("跳过：未设 UTOPIA_DATABASE_URL"); + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let f = seed(&pool).await?; + + let run = async { + // 1. 没声明：硬表与类层级照旧 + let orion = entity(&pool, &f, "Orion", f.organization).await?; + assert_eq!( + drift_reviews(&pool, &f, "Orion", f.project).await?, + vec![orion], + "organization vs project is confusable by the hard-coded list" + ); + let acme = entity(&pool, &f, "Acme", f.corporation).await?; + assert_eq!( + drift_reviews(&pool, &f, "Acme", f.federal_agency).await?, + vec![acme], + "corporation vs federal_agency share the ancestor organization: kin, so Review" + ); + + // 2. 声明 organization ⟂ project：硬表说易混，本体说互斥——本体赢 + ontology::set_disjoint_for(&pool, f.kb, f.organization, &[f.project]).await?; + let _vega = entity(&pool, &f, "Vega", f.organization).await?; + assert!( + drift_reviews(&pool, &f, "Vega", f.project) + .await? + .is_empty(), + "a declared disjointness wins over the hard-coded list" + ); + + // 3. 声明 corporation ⟂ agency：federal_agency 是 agency 的子类，互斥继承下来， + // 类层级说一家也不算 + ontology::set_disjoint_for(&pool, f.kb, f.corporation, &[f.agency]).await?; + let _beta = entity(&pool, &f, "Beta", f.corporation).await?; + assert!( + drift_reviews(&pool, &f, "Beta", f.federal_agency) + .await? + .is_empty(), + "a disjointness declared on the parent reaches the child and wins over kinship" + ); + // 反过来问也一样：表里两个方向各一行，继承沿另一头的祖先链走 + let _gamma = entity(&pool, &f, "Gamma", f.federal_agency).await?; + assert!( + drift_reviews(&pool, &f, "Gamma", f.corporation) + .await? + .is_empty(), + "the declaration holds from either side" + ); + + // 4. 取消声明，回到没声明时的行为——编辑必须能撤 + ontology::set_disjoint_for(&pool, f.kb, f.corporation, &[]).await?; + let delta = entity(&pool, &f, "Delta", f.corporation).await?; + assert_eq!( + drift_reviews(&pool, &f, "Delta", f.federal_agency).await?, + vec![delta], + "with the declaration gone, kinship sends the pair to Review again" + ); + Ok::<_, anyhow::Error>(()) + } + .await; + + let _ = sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(f.org) + .execute(&pool) + .await; + run +} diff --git a/crates/utopia-store/tests/a_proof_reaches_the_sentence.rs b/crates/utopia-store/tests/a_proof_reaches_the_sentence.rs new file mode 100644 index 00000000..e7139a27 --- /dev/null +++ b/crates/utopia-store/tests/a_proof_reaches_the_sentence.rs @@ -0,0 +1,254 @@ +//! R2：一条派生的证明要能一路读到原句（`docs/decisions/0002`）。 +//! +//! 前提一律是断言（`fact_derivations` 不记派生），所以证明是一条链： +//! 派生 → 按 `seq` 排好的断言 → 每条断言的证据 → chunk。这里守三件事： +//! +//! 1. **顺序对**。`A part_of B`、`B part_of C` 推出 `A part_of C`，证明第一步是 A→B。 +//! 2. **叶子是原句**。每一步带着它的证据，引句就是当初抽出它的那句话。 +//! 3. **撤了的前提照样列出并打标记**。派生随之失效，`proof` 仍能回看当时靠的是什么。 +//! +//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆，绝不碰已有的库。 + +use sqlx::PgPool; +use utopia_store::reasoning; +use uuid::Uuid; + +struct Fixture { + org: Uuid, + kb: Uuid, + part_of: Uuid, + a: Uuid, + b: Uuid, + c: Uuid, + doc: Uuid, + chunk_ab: Uuid, + chunk_bc: Uuid, +} + +async fn seed(pool: &PgPool) -> anyhow::Result { + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let etype = Uuid::now_v7(); + let part_of = Uuid::now_v7(); + let (a, b, c) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let (src, doc, chunk_ab, chunk_bc) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'proof-test')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'proof-test')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases (id, workspace_id, name, materialize_inferences) + VALUES ($1, $2, 'proof-test', TRUE)", + ) + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, 'thing', 'Thing')", + ) + .bind(etype) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO relation_types (id, kb_id, key, label, is_transitive) + VALUES ($1, $2, 'part_of', 'part of', TRUE)", + ) + .bind(part_of) + .bind(kb) + .execute(pool) + .await?; + for (id, name) in [(a, "FarmBeats"), (b, "Azure"), (c, "Microsoft")] { + sqlx::query( + "INSERT INTO entities (id, kb_id, type_id, canonical_name) VALUES ($1, $2, $3, $4)", + ) + .bind(id) + .bind(kb) + .bind(etype) + .bind(name) + .execute(pool) + .await?; + } + sqlx::query("INSERT INTO sources (id, kb_id, name) VALUES ($1, $2, 'proof-test')") + .bind(src) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO documents (id, kb_id, source_id, filename, sha256, status) + VALUES ($1, $2, $3, 'press.md', 'proof', 'ready')", + ) + .bind(doc) + .bind(kb) + .bind(src) + .execute(pool) + .await?; + for (id, seq, text) in [ + (chunk_ab, 0i32, "FarmBeats is part of Azure."), + (chunk_bc, 1i32, "Azure is part of Microsoft."), + ] { + sqlx::query( + "INSERT INTO chunks (id, kb_id, document_id, seq, text) VALUES ($1, $2, $3, $4, $5)", + ) + .bind(id) + .bind(kb) + .bind(doc) + .bind(seq) + .bind(text) + .execute(pool) + .await?; + } + Ok(Fixture { + org, + kb, + part_of, + a, + b, + c, + doc, + chunk_ab, + chunk_bc, + }) +} + +/// 一条断言，带一句原文当证据 +async fn asserted( + pool: &PgPool, + f: &Fixture, + subject: Uuid, + object: Uuid, + chunk: Uuid, + quote: &str, +) -> anyhow::Result { + let id = Uuid::now_v7(); + sqlx::query( + "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, confidence) + VALUES ($1, $2, $3, $4, $5, 0.9)", + ) + .bind(id) + .bind(f.kb) + .bind(subject) + .bind(f.part_of) + .bind(object) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO fact_evidence (fact_id, chunk_id, quote, proposed_predicate, document_id, doc_version) + VALUES ($1, $2, $3, 'part of', $4, 1)", + ) + .bind(id) + .bind(chunk) + .bind(quote) + .bind(f.doc) + .execute(pool) + .await?; + Ok(id) +} + +#[tokio::test] +async fn a_proof_reaches_the_sentence() -> anyhow::Result<()> { + let Ok(url) = std::env::var("UTOPIA_DATABASE_URL") else { + eprintln!("跳过：未设 UTOPIA_DATABASE_URL"); + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let f = seed(&pool).await?; + + let run = async { + let ab = asserted( + &pool, + &f, + f.a, + f.b, + f.chunk_ab, + "FarmBeats is part of Azure", + ) + .await?; + let bc = asserted( + &pool, + &f, + f.b, + f.c, + f.chunk_bc, + "Azure is part of Microsoft", + ) + .await?; + reasoning::materialize(&pool, f.kb).await?; + + let derived: Vec = + reasoning::derived_for_entity(&pool, f.kb, f.a).await?; + let ac = derived + .iter() + .find(|d| d.subject_id == f.a && d.object_id == f.c) + .expect("A part_of C should be derived"); + + // 1. 顺序对，2. 叶子是原句 + let proof = reasoning::proof(&pool, f.kb, ac.id) + .await? + .expect("live derivation has a proof"); + assert_eq!(proof.derived.id, ac.id); + assert_eq!(proof.steps.len(), 2); + assert_eq!( + proof.steps[0].fact_id, ab, + "the chain starts where the derivation starts" + ); + assert_eq!(proof.steps[1].fact_id, bc); + assert_eq!(proof.steps[0].subject, "FarmBeats"); + assert_eq!(proof.steps[0].object.as_deref(), Some("Azure")); + assert_eq!(proof.steps[0].predicate.as_deref(), Some("part of")); + assert_eq!(proof.steps[0].evidence.len(), 1); + assert_eq!( + proof.steps[0].evidence[0].quote.as_deref(), + Some("FarmBeats is part of Azure"), + "the leaf of a proof is the sentence it was extracted from" + ); + assert_eq!(proof.steps[0].evidence[0].chunk_id, f.chunk_ab); + assert_eq!(proof.steps[1].evidence[0].chunk_id, f.chunk_bc); + assert!(proof.steps.iter().all(|s| !s.retracted)); + + // 一个不存在的 id 不是错误，是「没有证明」 + assert!(reasoning::proof(&pool, f.kb, Uuid::now_v7()) + .await? + .is_none()); + + // 3. 撤掉一条前提：派生失效，证明还在，且那一步打上标记 + sqlx::query("UPDATE facts SET invalidated_at = now() WHERE id = $1") + .bind(bc) + .execute(&pool) + .await?; + reasoning::materialize(&pool, f.kb).await?; + let (gone,): (bool,) = + sqlx::query_as("SELECT invalidated_at IS NOT NULL FROM derived_facts WHERE id = $1") + .bind(ac.id) + .fetch_one(&pool) + .await?; + assert!(gone, "a derivation falls with its premise"); + let proof = reasoning::proof(&pool, f.kb, ac.id) + .await? + .expect("an invalidated derivation still explains itself"); + assert!(!proof.steps[0].retracted); + assert!( + proof.steps[1].retracted, + "the retracted premise is marked, not hidden" + ); + anyhow::Ok(()) + } + .await; + + let _ = sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(f.org) + .execute(&pool) + .await; + run +} diff --git a/crates/utopia-store/tests/a_source_kind_is_listed_once.rs b/crates/utopia-store/tests/a_source_kind_is_listed_once.rs new file mode 100644 index 00000000..4e616411 --- /dev/null +++ b/crates/utopia-store/tests/a_source_kind_is_listed_once.rs @@ -0,0 +1,53 @@ +//! #247：来源的种类只在一处定义，前后端对表。 +//! +//! 后端 `SourceKind`（utopia-core）一个枚举出两份清单：创建时的白名单、同步时的分派 +//! （后者按枚举穷举匹配，编译器保证加了种类就得决定它怎么同步）。前端那一份在 +//! `web/src/sourceKinds.ts`，这个测试把它读出来跟枚举比——此前两边各自手写，五种 +//! 连接器进了界面、进了同步，却没进创建白名单，界面上选得到、建的时候报 +//! 「kind must be one of…」。单元测试与 tsc 都看不见的那种漂移，这里看得见。 +//! +//! 不需要数据库。 + +use std::path::Path; +use utopia_core::models::SourceKind; + +/// 从 `CREATABLE_SOURCE_KINDS = [ "…", … ] as const` 里把引号里的字面量按顺序读出来 +fn frontend_kinds(src: &str) -> Vec { + let start = src + .find("CREATABLE_SOURCE_KINDS = [") + .expect("web/src/sourceKinds.ts declares CREATABLE_SOURCE_KINDS"); + let body = &src[start..]; + let end = body.find(']').expect("the array closes"); + body[..end] + .split('"') + .skip(1) + .step_by(2) + .map(str::to_string) + .collect() +} + +#[test] +fn the_frontend_list_matches_the_backend_enum() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../web/src/sourceKinds.ts"); + let src = + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())); + let frontend = frontend_kinds(&src); + let backend: Vec = SourceKind::creatable() + .map(|k| k.as_str().to_string()) + .collect(); + assert_eq!( + frontend, backend, + "web/src/sourceKinds.ts and utopia_core::models::SourceKind list different kinds (order matters: it is the dialog's order)" + ); +} + +#[test] +fn every_kind_round_trips_through_its_string() { + for k in SourceKind::all() { + assert_eq!(SourceKind::parse(k.as_str()), Some(k), "{k:?}"); + } + assert_eq!(SourceKind::parse("watch_folder"), None); + assert!(!SourceKind::Memory.creatable_by_hand()); + assert!(!SourceKind::Upload.creatable_by_hand()); + assert!(SourceKind::S3.creatable_by_hand()); +} diff --git a/crates/utopia-store/tests/a_viewer_never_sees_a_credential.rs b/crates/utopia-store/tests/a_viewer_never_sees_a_credential.rs new file mode 100644 index 00000000..5efcab44 --- /dev/null +++ b/crates/utopia-store/tests/a_viewer_never_sees_a_credential.rs @@ -0,0 +1,139 @@ +//! #246：来源列表不带任何凭据。 +//! +//! 列表接口给 Viewer 看，此前只剔了 `auth_header`；对象存储、WebDAV、Notion 各自的 +//! 密钥原样下发。现在凭据键列在 `SOURCE_SECRET_KEYS` 一张表上，列表 SQL 按表剔。 +//! 这里守两件事： +//! +//! 1. **列表里一个凭据键都没有**，每一种连接器都试一遍。 +//! 2. **身份标识留着**（bucket、username、account_name），界面要显示得出「这是哪个账号」； +//! 而同步那条路（`sources::get`）拿到的仍是完整配置——凭据只是不出去，不是没了。 +//! +//! 直接插表而不走 `sources::create`：`KINDS` 少了五种（#247），那是另一个修复。 +//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆，绝不碰已有的库。 + +use sqlx::PgPool; +use utopia_core::models::SOURCE_SECRET_KEYS; +use utopia_store::sources; +use uuid::Uuid; + +async fn seed(pool: &PgPool) -> anyhow::Result<(Uuid, Uuid)> { + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'secret-test')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'secret-test')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'secret-test')", + ) + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + Ok((org, kb)) +} + +#[tokio::test] +async fn a_viewer_never_sees_a_credential() -> anyhow::Result<()> { + let Ok(url) = std::env::var("UTOPIA_DATABASE_URL") else { + eprintln!("跳过：未设 UTOPIA_DATABASE_URL"); + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let (org, kb) = seed(&pool).await?; + + let run = async { + // 每种连接器一条，配置按各自界面真正会写的键 + let fixtures: Vec<(&str, serde_json::Value)> = vec![ + ( + "custom", + serde_json::json!({ "endpoint": "https://x.test/items", "auth_header": "Bearer c" }), + ), + ( + "github_issues", + serde_json::json!({ "repo": "o/r", "auth_header": "Bearer g" }), + ), + ( + "jira_issues", + serde_json::json!({ "base_url": "https://j.test", "project": "P", "auth_header": "Basic j" }), + ), + ( + "s3", + serde_json::json!({ "bucket": "b", "region": "r", "access_key_id": "AKIA", + "secret_access_key": "s3-secret" }), + ), + ( + "azure_blob", + serde_json::json!({ "bucket": "c", "account_name": "acct", "account_key": "az-key" }), + ), + ( + "gcs", + serde_json::json!({ "bucket": "g", "service_account_key": "{\"private_key\":\"x\"}" }), + ), + ( + "webdav", + serde_json::json!({ "base_url": "https://d.test", "path": "/", "username": "u", + "password": "dav-pass" }), + ), + ("notion", serde_json::json!({ "token": "secret_n", "query": "q" })), + ]; + for (kind, config) in &fixtures { + sqlx::query( + "INSERT INTO sources (id, kb_id, kind, name, config) VALUES ($1, $2, $3, $3, $4)", + ) + .bind(Uuid::now_v7()) + .bind(kb) + .bind(kind) + .bind(config) + .execute(&pool) + .await?; + } + + let listed = sources::list(&pool, kb).await?; + assert_eq!(listed.len(), fixtures.len()); + for s in &listed { + let obj = s.config.as_object().expect("config is an object"); + for key in SOURCE_SECRET_KEYS { + assert!( + !obj.contains_key(*key), + "{}: `{key}` must not reach a viewer, got {:?}", + s.kind, + obj + ); + } + } + // 身份标识留着 + let by_kind = |k: &str| { + listed + .iter() + .find(|s| s.kind == k) + .map(|s| s.config.clone()) + .expect("listed") + }; + assert_eq!(by_kind("s3")["bucket"], "b"); + assert_eq!(by_kind("s3")["access_key_id"], "AKIA"); + assert_eq!(by_kind("azure_blob")["account_name"], "acct"); + assert_eq!(by_kind("webdav")["username"], "u"); + assert_eq!(by_kind("custom")["endpoint"], "https://x.test/items"); + assert_eq!(by_kind("notion")["query"], "q"); + + // 同步那条路仍拿完整配置：凭据只是不出去，不是没了 + for s in &listed { + let full = sources::get(&pool, s.id).await?; + let want = &fixtures.iter().find(|(k, _)| *k == s.kind).unwrap().1; + assert_eq!(&full.config, want, "{}: sync still sees the credentials", s.kind); + } + Ok::<_, anyhow::Error>(()) + } + .await; + + let _ = sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await; + run +} diff --git a/docker-compose.yml b/docker-compose.yml index b97180b3..32f74ca5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: # 写一个不存在的标签，clone 下来第一步就是 manifest unknown。 # 所以撤掉一个版本时必须连它一起改——0.1.0 撤了，现在指向 rc。 # 转正之后改回 0.1.0（或改成 latest 让它自动跟随）。 - image: ${UTOPIA_IMAGE:-ghcr.io/deeplethe/utopia:0.1.0-rc2} + image: ${UTOPIA_IMAGE:-ghcr.io/deeplethe/utopia:0.1.0-rc3} profiles: ["app"] environment: # 默认以 owner 身份运行。要启用受限角色（业务表随便读写、台账只增不改）， diff --git a/docs/decisions/0001-ontology-import-and-governance.md b/docs/decisions/0001-ontology-import-and-governance.md index c63cf84d..1d5f5dd4 100644 --- a/docs/decisions/0001-ontology-import-and-governance.md +++ b/docs/decisions/0001-ontology-import-and-governance.md @@ -1,581 +1,265 @@ -# 0001 · 本体导入与治理路线 - -- **状态**：进行中 · P0 / P1 / P2 / P2c 全部建成；P3 的「按预算切换」已落地（部署级 `ontology_prompt_budget`，缺省 24,000 字符，超了按块检索），P3a 建成但**只能手动触发**、没有类数阈值这根轴；P3b 两条腿都建成但形态与本文不同（原词落 `fact_evidence.proposed_predicate`，映射由 `predicate_match` + [0003](0003-ontology-growth-loop.md) 的采纳回路承担）；**P4a 已建成，P4b / P4c 仍待做**；**P5 已由 [0002](0002-reasoning-engine.md) 落地，`utopia-reason` 不再是空壳**；判据 2 关于「参数顺序」的那一半已被 [0012](0012-the-ontology-is-a-contract-not-a-suggestion.md) 推翻。2026-09-02 对照代码复核，修订就地 -- **成文**：2026-08-27 / 28，随调研持续修订（约定见 [README](README.md)） - -> 理由比清单重要——清单会过期，理由决定以后遇到新情况怎么判。所以本文把「贯穿性判据」放在分阶段之前，且推翻过的判断就地留痕（见 P1、P3 的修订记录）。 - -## 为什么是这条线 - -产品的差异化是"时间"与"可信"，而这两样的上游都是**本体**：抽取器按本体决定抽什么、归成什么类；时态引擎按本体的 `functional` 决定什么算矛盾；消解按类型决定谁能和谁是同一个。本体不对，下游全歪，而且歪得沉默。 - -企业已经有自己的本体（FIBO、行业标准、Protégé 自建），让他们在界面上一条条重建不现实。所以这条线的终点是：**把企业本体导进来，让抽取按他们的词汇工作，并且治理过程本身能沉淀。** - ---- - -## 已验证的事实（2026-08-28，真实语料测得） - -用公开企业新闻稿建了 `Industry Corpus` 库（NVIDIA 博客 18 篇 + 微软新闻室 10 篇，RSS 摄入）： - -| 指标 | 数值 | -|---|---| -| 文档 / 分块 | 28 / 181 | -| 实体 / 事实 | 917 / 922 | -| **实体密度** | **5.07 个 / 分块** | -| 消解：自动合并 / 待人工 | 10 对 / 5 对 | -| 攒批裁决任务数 | 22（处理 917 实体的消解对） | -| 图谱总览查询 | 103 ms | -| as-of 查询 | 107 ms | -| 实体变更历史（NVIDIA 122 事件） | 16 ms | -| 当前 9 类本体在提示词中的体积 | 466 字符 | - -**跨文档消解成立**：NVIDIA 聚合了 16 篇文档的 118 条事实，没有裂成同名副本。 - -**已修的三个缺陷**（分支见文末）： - -1. HTTP 客户端不发 User-Agent → 维基百科等站点 403，URL/RSS 对一大片真实网站直接失效 -2. HTML 提取全文档遍历 → 一篇维基条目 60 个分块，首块整块是侧栏菜单、末块是版权声明 -3. `part_of` 被标成 `functional` → 28 篇新闻稿积压 59 条假冲突（"属于 Microsoft Learn"与"属于 Microsoft"并存不是矛盾） - -**未解决的质量问题**：Product 类占 40%（368 个），抽样约四分之一是泛化短语（"row power center"、"financial databases"）。补救手段是给类写描述（description 进抽取提示词）。 - -> **原文这里写「目前只能整库重抽，因为实体不可单独编辑」——P0 建成后不再成立。** -> 今天有三条更细的路：实体面板直改（受 P4a 的 `type_source` 保护）、类型消解批量精化、 -> 本体长出新类后的 `adopt_proposed_types` 认领。整库重抽不再是唯一手段。 - ---- - -## 贯穿性判据 - -以后遇到新构造、新特性，用这几条判，不要重新辩论： - -1. **不是"OWL 有没有"，是"我们有没有机器消费它"**——但要加时间维度：**丢弃不可逆，保存近乎免费**。所以原文一律保真，投影只覆盖当下用得上的。推论：**保留了原文，投影就不必语义完整**——简化是 UI/提示词层的取舍，不是数据损失。 -2. **本体是引导，不是执法**。声明可能是错的（`part_of` 就是），所以本体应当影响提示词与候选排序，而不是驱动丢弃、覆盖、自动改写。错误的引导模型能靠原文覆盖；错误的执法会系统性地毁数据。 - > **修订（2026-09-02，据 [0012](0012-the-ontology-is-a-contract-not-a-suggestion.md)）**：这条要拆成两半。**哪些类型能参与**仍是引导——本体可能写错，原文说西雅图就写西雅图。**参数顺序**改成执法：它不是关于世界的断言，是这个 key 的编码约定，原文从来没有「说了别的方向」。主语违反 domain 而宾语符合时按签名对调，留 `direction_corrected` 痕迹；对调也不合法则丢掉谓词、保留主宾与证据。留痕的自动动作不属于本条反对的那一类。实测五轮：违反率 57% → 4%，真·反向 39 → 0。 -3. **提示词大小必须与本体规模脱钩**。把 N 个类塞进每个分块的提示词，成本是 O(分块 × 本体)，而且模型从 800 个选项里挑反而更差。规模问题用检索解，不用"少列几个"解。 -4. **同名不能传递任何结论**。判定为同一实体 → 本来就一个类型，无所谓传递；判定为不同 → 名字正是我们刻意不信任的信号（两个张伟）。能携带信息的是上下文。 -5. **不确定性浮到人面前**，不要悄悄猜。分层阈值 + 灰区进 Review，与消解的"宁分勿合"同源。 -6. **治理经验固化进 schema，不要固化成隐形规则**。改一次类描述是可见、可评审、进审计台账的；从点击里长出来的自动规则半年后没人知道为什么。 -7. **唯一 ≠ 身份**。`key` 有 `UNIQUE (kb_id, key)`，但它派生自会变的东西（label / 局部名），而身份要求跨时间稳定。只要两个不同的全局事物可能想要同一个 key，key 就必须附带"来自何处"才能当标识符——那就是 IRI。另：**IRI 是名字不是地址**，绝大多数不可访问，不要去抓取，也不要把 http/https、末尾斜杠"规范化"成同一个。 - ---- - -## 分阶段 - -### P0 · 实体可编辑 —— 地基，且现在就在流血 - -**问题**：实体是只读的。`/kbs/{id}/entities/{entity_id}` 只有 GET，store 层唯一的 `UPDATE entities SET type_id` 在 `resolution.rs:388`（合并时自动升格），前端面板没有任何编辑入口。类型判错、名字抽歪，只能整库重抽这把大锤。 - -**做** -- `PATCH /kbs/{id}/entities/{entity_id}`，Editor 权限，可改 `type_id` / `canonical_name` -- 落审计台账：`entity.retyped` / `entity.renamed`，detail 带前后值快照（沿用决策台账的自包含快照约定） -- 前端：实体面板头部加编辑入口 - -> **修订记录**：初稿写「`entities` 上有唯一索引，改名会撞 409」。**两处都错**——`entities_kb_type_name_idx` 是**普通索引不是唯一索引**，而且同类同名的重复**现在就存在**（`General` 库「张伟」×2、「rust compiler」×2）。更要紧的是：不该加那个约束。 - -**碰撞是提示，不是拒绝**：「张伟」×2 正是"宁分勿合"的设计产物——两个不同的人可以同名同类，消解的整个灰区逻辑依赖于能把他们分开存放。所以改名/改类型撞上同名时：**查出来、提示"已有同名实体，是否合并？"、但允许继续**。加唯一约束会砸掉消解的地基，报 409 会让用户无法录入第二个张伟。 - -**动机数据**：`General` 库里跨类型同名的——"rust" **5 个**、"java" 3 个、"c++" 3 个。同一个东西被分进五个类，今天无法收拾。 - -**验收**：能把 "row power center" 从 Product 改成 Concept，改动出现在审计台账，图谱与本体页的计数同步。 - ---- - -### P1 · 停止静默丢弃 —— 小，但现在就在流血 - -> **修订记录**：这一格原本是"关系的 range 校验"，理由是"对现有语料立刻生效"。**那句是错的**——查证后：23 个关系里只有 2 个有 domain（就是那两个属性），**range 一条都没有**，因为 UI 里没有这一项、没有任何东西会写它。range 数据只可能来自 OWL 导入，所以它不可能独立于 P2 生效。range 的正确位置见 P2（导入并保存）与 P3（作为信号之一消费）。 - -**真正现在就在流血的**：抽取器把事实**静默丢弃**——抽出来了、被挡掉、不留任何痕迹，用户看不见，只是图里少了东西。与"账本 append-only、每条事实都有证据、不确定性浮到人面前"三条原则直接冲突。 - -**范围比最初判断的大**（2026-08-28 逐行核过 `extraction.rs`）。属性路径不是一个 `continue`，是四个真丢弃： - -| 行 | 条件 | 性质 | -|---|---|---| -| 255 | 主语未在 `entities` 中声明 | 真丢弃 | -| 257 | `domain_type_id` 为 NULL | **不可达**——`ontology.rs:197` 强制 attribute 必有 domain，`update_relation_type` 又不许改 kind/domain（`ontology.rs:290`）。且无 domain 的属性连提示词都进不去（`extraction.rs:122` 的 `r.domain_type_id?`）。防御性代码，不需要信号 | -| 261 | domain 不匹配 | 真丢弃（最初只点了这一条） | -| 269 | 既无 `value` 也无 `object` | 真丢弃 | -| 274 | datatype 归一化失败 | 有 `debug!` 日志，但用户不可见 | - -关系路径另有两条值得记：`confidence < 0.6`（第 234 行，设计阈值，但同样不可见——用户无从知道"抽到了但不够自信"）与 `related_to` 不在本体时**整条事实消失**（第 380 行，删掉默认关系就会踩到）。〔**后者已不可能发生**：`related_to` 整个删掉，落不上就是 `predicate_id = NULL`，见 [0010](0010-no-relation-is-no-relation.md)。〕 - -**顺带发现一个不一致**：关系路径上主宾未声明时会**兜底按 `concept` 消解**（332-362 行），属性路径上同样的缺失却直接丢（255 行）。同一个原因，一边宽容一边静默丢弃。要么两边都兜底，要么两边都记信号——不能一边一个样。〔**已消除**：`concept` 兜底类随 [0009](0009-no-type-is-a-type.md) 删掉，两条路径现在都记 `subject_not_declared`。丢弃原因码今天有 12 个（`extraction_drops::reason`），含 `truncated_reply`、`malformed_item`、`not_an_entity_name`、`direction_corrected`，比本节设想的多一倍。〕 - -**做** -- 新表 `extraction_drops (kb_id, document_id, reason, detail, count, example, updated_at)`，PK `(kb_id, document_id, reason, detail)`，抽取开始时按 document 清空 -- 不复用 `ontology_misses`：那张表语义是"你的本体缺这些"（面向本体演进），丢弃是"这些事实没落地"（面向数据完整性），两者的读者和动作都不同 -- 按 document 归集顺带修好生命周期：`ontology_misses` 只在整库 rebuild 时清（`graph.rs:689`），来源级重抽不清，会攒陈旧条目。新表按 document 清则天然正确 - -**验收**：故意给 Person 专属属性喂一个 Organization 主语，Library 的文档行能看到"3 条事实未落地"，点开有原因与样例，事实不再无声消失。 - ---- - -### P2 · OWL 导入 - -**架构分三层，关键在第一层** - -1. **原文保真**：导入的本体文件原样进 blob store（内容寻址那套摄入管道在用），`ontology_imports` 记录 (kb_id, blob sha, 文件名, 时间, 投影版本)。这一步不理解语义，只负责不丢。 -2. **投影**：把今天能消费的子集映射进 `entity_types` / `relation_types`。**是可重跑的推导，不是一次性转换。** -3. **重投影**：推理机上线或我们补上新校验时重跑，用户什么都不用做。 - -这样"我们表达不了"从**能力缺口**降级成**投影暂未覆盖**。 - -**映射表** - -| OWL / RDFS | Utopia | 备注 | -|---|---|---| -| `owl:Class` | entity_type | | -| `rdfs:subClassOf` | **多父**（见下） | | -| `rdfs:label` | `label` | 优先 `@en`/`@zh` | -| **`rdfs:comment`** | **`description`** | 承重：进抽取提示词，也是 P3 检索的匹配依据 | -| `owl:ObjectProperty` | relation_type (`kind=relation`) | | -| `owl:DatatypeProperty` | relation_type (`kind=attribute`) | | -| `rdfs:domain` | `relation_type_domains`（多值关联表） | 见下"多值域" | -| `rdfs:range` | `relation_type_ranges`（多值关联表）/ `datatype` | 同上；**只存不强制**，消费者是 P3/P5 | -| `owl:FunctionalProperty` | `functional` | max-cardinality-1，时态引擎的命根子 | -| `owl:InverseFunctionalProperty` | `inverse_functional` | | -| IRI | 新增 `iri` 列 | 见下"IRI 与 key 的分工" | -| 其余公理 | **保留在原文，报告为"暂未投影"** | 不是"已跳过" | - -**IRI 与 key 的分工**：`key` 有硬约束——只允许 `[a-z0-9_]`、最长 40 字符（`ontology.rs:77`），而且它是**给模型读写的令牌**：提示词列的是 key，模型返回的 `type` 是 key，`type_ids` 按 key 查。IRI 塞不进去（冒号斜杠井号全非法），放宽约束的话提示词变成 800 行 URL，还要模型逐字复现——截断或幻觉一个字符就匹配失败。 - -所以：**IRI 是全局身份（权威），key 是模型标签**。key 从 IRI 派生（命名空间前缀 + 局部名，snake_case，冲突加后缀），短而稳定。存储：一个可空 TEXT 列 + `UNIQUE (kb_id, iri) WHERE iri IS NOT NULL`；手工建的类为 NULL，无负担。 - -**为什么 key 唯一了还要它**——重导入是本体功能上线就会撞到的第一件事： - -``` -v1: http://acme.com/hr#Employee rdfs:label "Employee" → key = employee -v2: http://acme.com/hr#Employee rdfs:label "Staff Member" → key = staff_member ← 同一个类 -``` - -没有 IRI 只能二选一，都错：建新类 `staff_member` 把 `employee` 变孤儿（实体全挂在上面），或按 label 匹配（label 正是刚变的那个）。有 IRI 就是一次 `WHERE iri = $1`，改名字，实体不动。 - -想「那 key 就从局部名派生」也撑不住：`foaf:Person` 与 `acme:Person` 都要 `person`，谁拿 `_2` 后缀取决于导入顺序，下次重导入分不出 `person_2` 是谁的——除非记下它来自哪个 IRI，那就是 IRI 列绕一圈重新发明。另两个用途：**溯源**（上游类被手改后重导入要调和，得先知道它是上游的）与 **P5 导出**（key 反推不出命名空间，有损）。 - -**多值域（domain / range）**：OWL 里并集是常态（`works_at` 的值域可能是 `Organization ∪ Project`），单列表达不了，用关联表存，判定走子类 DAG。 - -导入的语义陷阱：RDFS 中同一属性的**多条 `rdfs:range` 是交集**（"必须同时是两者"），不是并集——极常见的建模错误，但规范如此。`owl:unionOf` → 直接进表；多条独立 `rdfs:range` → 一者是另一者的子类则取更具体的，互不包含则**报告"暂未投影"，不猜**。 - -**domain/range 的正确用法是提示词里的类型签名，不是闸门** - -关系在提示词里从 `- works_at (works at): 描述` 变成 `- works_at (works at): Person → Organization。描述`。这是**给模型的类型签名**，在源头减少 "Alice works_at 西雅图" 这类三元组，而不是等错了再修——事后校验面对的是既成事实（丢掉可惜、留着是脏数据），签名是在生成那一刻掰正。 - -**是引导不是强制**：本体写错时模型看到原文说了别的仍可覆盖；强制闸门则会系统性丢数据（`part_of` 烧我们的正是这种方式）。 - -大本体下（P3 规模）它换位置继续有用：预测谓词时，**domain/range 与实际主宾类型匹配的候选关系排名更高**，作为检索排序信号。 - -三件事的必要性不同：**存**（是，P5 的正经输入，成本在解析器里本来就要写）、**进提示词当签名**（是，最高性价比，约十行）、**校验/自动升格/违规队列**（否，P3 用更丰富的证据做同一件事，且用可能错的声明驱动自动动作风险高）。 - -**交集不实现，理由是架构性的**：**因为保留了原文，投影就不必语义完整**。投影只服务提示词与界面展示，两者都接受简化；需要完整语义的是推理机，而它读的是 blob 里的原文。实现交集意味着要表达"既是 A 又是 B"的匿名类——往描述逻辑走的第一步，会把 schema 拖进类表达式的泥潭，而收益只是提示词多一行字。 - -**多继承要真支持**：`type_matches_domain`（`extraction.rs:137`）沿单父链上溯。丢掉一个父分支，那分支上的属性会**静默失配**——抽出来了、写入时被挡、不报错。补法：`entity_type_parents` 关联表存全部父类，domain/range 判定走 DAG（访问集防环），左栏仍按主父展示成树，避免同一个类出现多次。 - -**两个必须在预览里说清的** -- **哪些关系会因 `functional` 而开启冲突检测**——这正是 `part_of` 那个坑，企业本体声明为函数性但数据不遵守，导完就是一队假冲突 -- **有多少类没有 `rdfs:comment`**——它们在 P3 的自动分类里质量会明显偏低 - -**流程**：上传 → **dry-run 预览**（新建/更新/暂未投影各多少、上述两项警告）→ 确认才落库。绝不让上传一个文件就不可逆地改掉本体。 - -**v1 砍掉**：OWL/XML 与 Manchester 语法（Turtle + RDF/XML 覆盖 Protégé 导出的绝大多数）、任何推理、个体导入、反向导出。 - -**个体（ABox）永不作为事实导入**——不是能力问题，是原则：每条事实必须有证据链和出处，凭空塞进来的实例破坏的正是这个地基。它们随原文件保存，将来推理机可当背景知识读。 - -**依赖**：`oxttl` + `oxrdfxml`（Oxigraph 那套小而专的 crate）。不引 `horned-owl`，我们不需要推理。工作区目前无任何 RDF 依赖。〔已引入，住在 `utopia-ingest`。〕 - -> **修订记录 · P2a + P2b 已落地**（`feat/owl-import`） -> -> 落地的是三层里的前两层与整条预览流程：解析（`utopia-ingest/src/ontology_rdf.rs`）、 -> 计划与执行（`utopia-server/src/owl_import.rs`，预览与落库**共用同一个 `plan()`**）、 -> 界面（本体页左栏底部 Import 入口 → 选文件 → 计划 → 确认）。 -> `ontology_imports` 表与 `entity_types.iri` / `relation_types.iri` 见 `ontology_imports` 的建表注释。 -> -> **对着真实词汇表验证，不是自己写的样例**。FOAF（RDF/XML，635 三元组）： -> 15 类 + 89 属性，`functional` / `inverse_functional` / domain / range 全部读出； -> DCTerms（Turtle，700 三元组）：22 类 + 55 属性。**自己手写的样例文件全部通过， -> 而真实的 FOAF 在第一行就死了**——格式判定写反了，而且文件开头的 ` Microsoft -``` - -而 schema.org 声明 `employee (organization → person)`。**130 条可校验的事实里 102 条这样反着落库。** - -这恰恰是选 schema.org 的理由失效——`ontology_packs.rs` 写着「1488 个属性带 -domain + range，方向是**声明的**不是描述的」。声明了，然后没人执行。 - -而且它比空谓词严重得多：**空谓词是诚实的沉默，反向边是自信的错误**。图上写着 -Musk 雇佣了 Microsoft，而 0002 说推理机是缺陷放大器——这正是它会放大的那种输入。 - -## 三、一个根因，两个症状 - -查下去，类型判错与方向反向是同一件事的两副面孔： - -``` -#128 撤掉种子 → seed_classes 空（它取的是 builtin 的类） - ↓ -检索偏爱字面出现在正文里的叶子类 -（讲 Sutskever 的分块，976 个类里 researcher 第 4、person 第 359、organization 第 795） - ↓ - ┌──────────────────────┬──────────────────────────┐ -实体判成 researcher employee 的签名退化成 (* → *) -（schema.org 里它是 （sig_of 只认铺出去的类， - Audience 的子类，不是人） 一侧没铺就写 *） -``` - -`build_lists` 的注释早写过这道地板：「**内置类恒在**。检索漏掉的分块仍然要有地方 -落脚，否则模型无类可选」。种子退场后判据悬空了——**它当初碰巧等价，只因为种子类 -正好是那几个通用类**。 - -## 四、五轮实验 - -同语料同本体，每次只动一个变量，全部 60/60 块： - -| 变体 | 事实 | 空谓词 | 可校验 | 违反 | **真·反向** | 违反率 | -|---|---|---|---|---|---|---| -| 只改提示词 | 429 | 138 | 172 | 98 | 39 | 57.0% | -| + 祖先地板 | 451 | 138 | 181 | 63 | 31 | 34.8% | -| + 签名类恒在 | — | — | — | — | 26 | 37.2% | -| + 按库内类型掰正 | 447 | 139 | 179 | **0** | **0** | 29.6% | -| + 前缀归并 + 不适用则沉默 | 447 | 179 | 149 | **6** | **0** | **4.0%** | - -（第三行停在 41/60 块——服务器被杀，只能看比率。） - -### 提示词能做的，和不能做的 - -第一版把两件事混成一句 "hint, not a rule — when the text says otherwise, write what -the text says"，于是模型连参数顺序也按原文说法写。两件事的可覆盖性根本不同： - -- **哪些类型能参与**：提示不是闸门。本体可能写错，原文说西雅图就写西雅图。 - [0001](0001-ontology-import-and-governance.md) 的判断在这里不变——硬闸门会 - 系统性丢数据，`part_of` 烧我们的正是那样。 -- **参数顺序**：由签名定。**顺序不是关于世界的断言，是这个 key 的编码约定**； - 原文从来没有「说了别的方向」，它只说两个实体之间存在某种关系。 - -改了之后有效，但**不是主力**：三轮提示词把违反率从 57% 压到 35%，压下去的 -**全是类型判错那一半**，真·反向纹丝不动（22.7% → 17.1% → 17.6%，后两个在噪声里）。 - -**模型看得见签名，就是不照做**——英语的 "X is an employee of Y" 太强。 -继续加措辞不会赢。 - -### 于是在写入时掰正 - -主语违反 domain **而宾语符合**时，按签名交换主宾。 - -这不是新原则：`produced_by` 命中 `produces` 时（#109）早就在自动翻转主宾了， -区别只在触发条件是**措辞**还是**签名**。 - -当初反对自动掰正的理由是实体类型不可靠——实测 Elon Musk 被判成 `researcher`。 -**祖先地板修好之后那个前提不成立了**，同一批人判成了 `person`。 - -类型**从库里的实体读**，不用抽取器手上那份 `entity_type_of`——那份只覆盖模型在 -这一块里声明过的实体，而宾语常是别处已存在的实体。**这一处差别就是反向从 10 降到 0 -的原因。** - -**绝不静默**：每次掰正落一条 `direction_corrected`。0001 反对的是「用可能错的声明 -驱动**自动动作**」，而留痕的动作不属于那一类。29 次里有 1 次掰错（`spatial`， -schema.org 对它的 domain 定义本就模糊）——这是这个机制的固有成本： -**它信任本体的声明，而声明可能不适用于这对实体。** - -### 不适用的关系该沉默 - -对调也不合法时，从前照原样落库，等于**用本体的名义说一件本体不同意的事**： - -``` -OpenAI --affectedBy--> … schema.org 的 affectedBy 是医学检验用的 -Mistral --amount--> €105 million amount 属于融资工具，不属于公司 -Anthropic --competitor--> OpenAI schema.org 的 competitor 是 SportsEvent 的属性 -``` - -现在丢掉谓词，保留主宾、时间、证据，原词留在 `fact_evidence.proposed_predicate`， -显示时由 `fact_surface_predicate()` 取回（[0010](0010-no-relation-is-no-relation.md)）。 -验过没有变哑：**退回的 179 条空谓词事实，179 条全部拿得出原文说法。** - -## 五、剩下的不是抽取的错 - -- **具化建模缺口**（`amount` 13 / `target` 8 / `participant` 5）：schema.org 用中间 - 节点建模（Action / LoanOrCredit / Offer），我们是扁平二元关系。 -- **同名词**（`affected_by` / `competitor` / `uses_device`）：包里有很多名字通用、 - 含义狭窄的词，按名字或向量相似度挑必然撞上。跟 `Researcher` 被拿来标人同源。 - -**这两类是本体包的本质成本**，0008 该记进去：**包的代价不在提示词长度，在选择难度。** - -### 为什么不引入更强的建模语言 - -数据模型**已经支持**中间节点——`entities` + `facts` 就是属性图， -`[某轮融资] --amount--> €105M` 今天就能存，库里也确实有 10 个 `event`、 -9 个 `monetary_amount` 实体。卡住的不是表达力。 - -卡住的是**提示词的规则 1 与 2**：「用文中写出的规范全名」+「每条事实的主宾都必须 -出现在 entities 里」。而一轮融资在原文里**没有名字**——要满足签名就得凭空造一个实体， -而提示词从头到尾在教它不要造。**模型做的是对的。** - -真要支持具化，动的是这两条规则的地基，还得给无名节点一套命名与去重办法 -（同一轮融资在两篇文章里怎么认成一个？它没有名字可比）——那是实体消解的新问题。 - -判据应该是**限定词本身需不需要被查询**：「2024 年 A 轮里由 General Catalyst -领投的公司有哪些」——那个「领投」挂在融资轮上。**今天问不了这种问题，那才是该做的信号**， -而不是「schema.org 这么建模所以我们也要」。 - -## 六、顺带修掉的两处 - -**领头的轻动词不算区别**：`has_funding` 与 `funding` 是同一个关系。实测空谓词里 -`has_funding` ×4 落空而本体有 `funding`，`product` ×2 落空而本体有 `has_product`。 -过度归并不会造成错配——撞车即作废，最坏退回「匹配不上」。 - -**「实体名」是一句话时不该造成实体**。实测 421 个实体里 76 个无类型，最长的 111 字符 -是一整个从句。守卫本来就有（`> 100 字符` 就 `continue`），但**只装在声明实体那条路上**， -而未声明的主宾会绕过去直接 `resolve()` 造实体——**前门有锁，后门开着**。而且那个 -`continue` 是静默的，正是 `drop_signal` 当初为之而建的七处之一。 - -判据改成**词数 + 限定动词**而不是字符数：`US District Court for the Northern District -of California`（57 字符）是真实体，`removal was driven by growing discontent and -distrust with Altman`（65 字符）是从句——字符数分不开，**谓语分得开**。 - -效果：最长实体名 111 → 57 字符（57 那个正是上面那个法院），无类型实体 76 → 60。 -被挡下的全是真句子，没有误伤。 - -**孤点（一条事实都没有的实体）14.0% → 17.5%，但这不能归因于守卫**：守卫总共只挡了 -个位数，而孤点多了 14 个，更可能是跑次方差（两次跑的模型输出本来就不同， -后一次还中途重启过、被回收的文档整篇重抽）。孤点的真实成因是另一回事—— -**模型在 `entities` 里声明了实体却没产出用到它的事实**，孤点列表里是法院、SEC、 -特斯拉总部这些正当实体。这个比例值得单独记，但它跟这次的守卫无关。 - -## 待做 - -- **写入时的守卫挡不住写入之后的改动**：实体合并会把主语换成另一个类型的实体， - 于是事实「变成」违反。实测 6 条残留里 4 条如此。要治得把同样的检查放进合并路径。〔**已做**（#190 / #196，`fix/one-domain-check-for-three-paths`）：写入时的判断抽成 `ontology::judge_direction`，抽取与**采纳**共用——采纳是第二条写谓词的路，实测把违反率从 0 抬到 12.3%，全在包关系上；两边都不合的采纳**不挂谓词**，条数随 `facts_left_off` 上报。**合并**不掰不改只报：搬动过的事实查一遍 domain / range，违反的进 `axiom_violations`（新 kind `signature`）。一致性检查（0002 R0）也多了这一类，全量量、事实撤了就清，所以本体事后改 domain 也逃不过。未分类实体不算违反——「不知道」不是「不符合」。〕 -- 另外 2 条残留（`Stability AI Ltd`、`Colossus 2 data center`）没有合并也没有改类型， - **原因未查明**。 -- 导入本体包时过滤掉需要中间节点才能用的关系（domain 全是 `Action`/`Offer`/ - `LoanOrCredit` 这类具化壳的），它们铺给模型只会产出违反。〔仍未做。〕 -- **（2026-09-02 补）`bootstrap_ontology.rs` 开头的模块注释仍在说「新建的库只有 10 个默认关系……降级成 related_to」**，那是三次退场之前的世界。不属于本篇，但与本条线同源，该改。 +# 0012 · The ontology is a contract + +- **Status**: implemented · five controlled runs on `ai-timeline-ends` × schema.org + W3C Org took the violation rate from 57% to 4% and true reversals from 39 to 0 · the write-time judgment (`ontology::judge_direction`) now also covers adoption and merge (#190 / #196; merge reports `signature` rows into `axiom_violations`, R0 re-checks the same class) · filtering reified-shell relations out of pack import is still not done · since this record the ontology can also declare `inverseOf` / `subPropertyOf` (#177 / #179) and the pack list grew to five; the runs cover only the first two +- **Written**: 2026-08-31 (inferred: the source is undated and sits between 0011 and 0013, both dated 2026-08-31) · condensed into English 2026-09-03 +- **Related**: [0008](0008-ontology-packs-as-cold-start.md) built the packs but never asked whether a large ontology holds up on real text; criterion 2 of [0001](0001-ontology-import-and-governance.md) ("the ontology guides, it does not enforce") is overturned by half here; [0010](0010-no-relation-is-no-relation.md) supplies the empty-predicate behavior; [0013](0013-a-source-should-hand-over-its-history.md) is the other end of the same line — how things come in, versus the rules they land by + +## The problem + +The packs earn their keep. Six Wikipedia articles with schema.org + W3C Org (1655 relations, +973 classes) and zero seeds: all 215 predicated facts and all 41 distinct relations came from +the pack, and the empty-predicate share fell to 25.6% (55.5% `related_to` in the ten-seed days). + +But not one declared constraint was enforced. schema.org declares +`employee (organization → person)`; the graph said `Elon Musk --employee--> Microsoft`, and +102 of 130 checkable facts were written backwards — the reason for choosing schema.org, 1488 +properties with declared domain and range, went unused. An empty predicate is honest silence; +a reversed edge is a confident error, exactly the input the engine of +[0002](0002-reasoning-engine.md) amplifies. + +One root cause, two symptoms. After #128 removed the seeds, `seed_classes` was empty and +retrieval favored leaf classes that appear literally in the text (in a chunk about Sutskever, +`researcher` ranked 4th of 976 classes, `person` 359th, `organization` 795th). Entities got +typed `researcher` (a subclass of `Audience` in schema.org), and `sig_of`, which only knows +classes that were laid out, degraded the signature of `employee` to `(* → *)`. + +## Decisions + +1. **Which types may participate stays guidance; argument order is enforced by the + signature.** The first prompt merged the two into "hint, not a rule — when the text says + otherwise, write what the text says", and the model applied that to argument order too. + Order is not a claim about the world, it is the encoding convention of the key; the text + never "says another direction", it only says a relation exists. For types, 0001 still + holds: a hard gate loses data systematically, as `part_of` showed. + +2. **The ancestor floor.** The ancestors of every retrieved class join the list, so `person` + and `organization` are always available. This removed the type half of the violations and + made entity types trustworthy: the people typed `researcher` became `person`. + +3. **Correct direction at write time.** When the subject violates the domain and the object + satisfies it, swap them by signature — the move `produced_by` → `produces` already made + (#109), triggered by the signature instead of the wording. Types are read from the entity + in the store, not from the extractor's `entity_type_of`, which only covers entities + declared in the current chunk while the object usually already exists. That single + difference took reversals from 10 to 0. + +4. **Never silently.** Every swap leaves a `direction_corrected` trace. 0001 objected to + automatic action driven by possibly wrong declarations; a traced action is a different + thing. One of 29 swaps was wrong (`spatial`, whose schema.org domain is vague) — the + built-in cost of trusting a declaration that may not apply to this pair. + +5. **A relation that does not apply falls silent.** When swapping is also illegal, keeping + the predicate would assert in the ontology's name something the ontology disagrees with + (`OpenAI --affectedBy--> …` is a medical-test property; `competitor` belongs to + `SportsEvent`). The predicate is dropped, subject, object, time and evidence stay, the + model's word goes to `fact_evidence.proposed_predicate` and `fact_surface_predicate()` + shows it (0010). All 179 facts pushed back to an empty predicate still surface their wording. + +6. **Two side repairs.** A leading light verb is not a distinction: `has_funding` and + `funding` merge, and over-merging is harmless because a collision voids the match. A clause + is not an entity name: the 100-character guard sat only on the declared-entities path while + undeclared subjects and objects went straight to `resolve()`; the criterion is now word + count plus a finite verb, since a 57-character court name and a 65-character clause cannot + be told apart by length. Longest name 111 → 57 characters, untyped entities 76 → 60. + +## Dead ends + +- **More prompt wording.** Three rounds moved the violation rate from 57% to 35%, all of it + type errors; true reversals stayed flat (22.7% → 17.1% → 17.6%, within noise). The model + sees the signature and does not follow it — English "X is an employee of Y" is too strong. +- **Automatic swapping before the floor.** Rejected earlier because entity types were + unreliable (Musk typed `researcher`); once the floor landed the premise fell away. +- **A richer modeling language for reification.** `amount` (13), `target` (8) and + `participant` (5) fail because schema.org models them through intermediate nodes (Action / + LoanOrCredit / Offer) and we write flat binary relations. Expressiveness is not the blocker: + `entities` + `facts` already form a property graph. The blocker is prompt rules 1 and 2 — + canonical names from the text, every subject and object an entity — and a funding round has + no name in the text, so the model would have to invent one. The model is right. Revisit when + a query needs the qualifier itself ("companies whose 2024 Series A was led by General + Catalyst"); unnamed nodes would also need their own naming and dedup. +- **Blaming the guard for orphans** (entities with no fact, 14.0% → 17.5%). The guard blocked + single digits; the run was restarted mid-way. The real cause is the model declaring + entities it never uses (courts, the SEC, Tesla HQ) — worth measuring separately. + +The remaining cost of a pack is selection difficulty, not prompt length: many names are +generic with narrow meanings (`affected_by`, `competitor`, `uses_device`, `Researcher`) and +get picked by name or vector similarity. 0008 should carry this. + +## Revisions + +- 2026-09-02: the write-time guard could not stop later edits — a merge swaps the subject for + an entity of another type and the fact "becomes" a violation (4 of the 6 residuals). Done in + #190 / #196: `ontology::judge_direction` is shared by extraction and adoption (adoption was + a second path writing predicates and had pushed the rate back to 12.3%); adoption that fits + neither way leaves the predicate off, counted under `facts_left_off`; merge does not swap, + it reports `signature` violations on moved facts, and R0 re-checks them so a later change of + domain cannot hide. An unclassified entity is not a violation — "unknown" is not "does not fit". + +## Open questions + +- Two residual violations (`Stability AI Ltd`, `Colossus 2 data center`) were neither merged + nor retyped; the cause is unknown. +- Pack import still lays out relations whose domain is a reified shell (`Action`, `Offer`, + `LoanOrCredit`); shown to the model they can only produce violations. diff --git a/docs/decisions/0013-a-source-should-hand-over-its-history.md b/docs/decisions/0013-a-source-should-hand-over-its-history.md index 3d6dc251..ef7f42ce 100644 --- a/docs/decisions/0013-a-source-should-hand-over-its-history.md +++ b/docs/decisions/0013-a-source-should-hand-over-its-history.md @@ -1,202 +1,114 @@ -# 0013 · 一个来源该交出它的历史，不是它的现状 - -- **状态**：已实施两个（`github_issues` #134、`jira_issues` #135）· 文档协作类（飞书 / Confluence / Notion）仍未开工，`instant` 精度未触发（2026-09-02 核，其余陈述逐条复核为真） -- **成文**：2026-08-31（约定见 [README](README.md)） -- **相关**：[0001](0001-ontology-import-and-governance.md) 的双时态地基； - 语料侧的同一个判断见 `scripts/bench/fetch-wiki-history.mjs`（#122）； - [0012](0012-the-ontology-is-a-contract-not-a-suggestion.md) 是同一件事的另一头—— - 这一篇管**东西怎么进来**，那一篇管**进来之后按什么规矩落** - -## 判据：一个来源值不值得接 - -这产品是双时态账本，所以看四条，缺一条就退化成"又一个抓网页的"： - -1. **有没有真实时间戳** —— 认知时间那根轴靠它 -2. **会不会自我推翻** —— `supersedes` 才有事可做 -3. **有没有稳定身份** —— 同一份东西的新版能认出来（`external_key`） -4. **企业知识是不是真的住在那儿** - -工单系统四条全中：每张工单的状态天生随时间变，而且信噪比远高于聊天记录。 - -## 核心判断：别取现在，取变化 - -**工单最有价值的部分不是"它现在是 closed"**，而是"8-18 开出、8-20 关掉、 -中间被指派给谁、优先级怎么变的"。 - -只抓当前状态，这条时间线要靠一次次同步慢慢攒——第一次同步只能看见此刻， -之前发生的全丢了。而多数系统**已经把变更史准备好了**，只是要多问一句。 - -这与 #122 给维基百科语料做的是同一个判断。区别是维基要从修订列表里采样， -工单系统直接把变更事件给你。 - -落进正文的形状（两个来源一致，刻意的）： - -``` -# KAFKA-9 Consumer logs ERROR during close - -Type Bug. -Currently Closed. -Resolved on 2011-07-19. - -## History -- 2011-08-05 — Alan Cabrera changed Workflow: jira → no-reopen-closed -- 2015-09-01 — Tony Stevenson changed Workflow: no-reopen-closed → Apache Kafka Workflow - -## Comments -### Luke Chen on 2026-08-24 -… -``` - -**抬头写成带日期的陈述句，不是键值对。** `Opened by X on 2026-08-18` 能抽出带 -`valid_from` 的事实；`created_at: 2026-08-18` 则要模型自己猜这是什么意思。 - -## 两个供应商教的不同的事 - -| | GitHub | Jira | -|---|---|---| -| 取法 | 三次拉取 + **逐工单**取事件 | **一次调用**取全（`expand=changelog`）| -| 增量 | `since` 参数 | JQL `updated >= "…"` | -| 变更史 | 事件级（"发生了 labeled"）| **字段级**（`Workflow: A → B`）| -| 时间戳 | RFC3339 | `+0000` **无冒号**，不是 RFC3339 | - -### GitHub：聪明的设计悄悄丢掉了全部价值 - -第一版想让三样都走仓库级端点、一次分页取全，避免 200 张工单 401 次请求。 -**拿真实数据一跑就发现事件那一路是错的**：`issues/events` 不支持 `since`， -只能从最新往回翻，而 GitHub 的模型里 PR 也产生 issue 事件——实测本仓库的工单事件 -埋在**第 5 页**，换一个 PR 活跃的仓库就会被推到翻页上限之外。 - -于是"状态变更史"悄悄变成空的，**而它正是这个来源存在的理由**。改成逐工单取， -N 只是本轮要写入的工单数。用一次准确换一次省事，这里该换。 - -### Jira：三个坑都藏在字段形状里 - -- **时间戳不是 RFC3339**（`2026-08-24T11:11:52.944+0000`，时区偏移没有冒号）。 - chrono 默认解不了，解不出来就是整页工单丢掉。 -- **增量没有 `since` 参数**，只能靠 JQL，时间格式是 Jira 自己的且**要带引号**—— - 两点写错的症状都是 400，不是"查不到"。 -- **`fields` 必须显式列**：不列 `comment` 就不返回评论；默认返回全部字段会把 - 一条响应撑到几百 KB。 - -## 拿真实响应钉住字段形状 - -手写的 JSON 只能证明"我以为的形状"解得出来。GitHub 一条 issue 有上百个字段， -我们只声明十个；哪个其实叫别的名、哪个在某些情况下是 null，只有真数据说得清。 - -两个连接器的夹具都取自**公开可匿名读的实例**（`deeplethe/utopia`、 -`issues.apache.org`），字段裁到我们声明的那些——裁剪本身顺带证明了未声明的 -字段不会让 serde 失败。 - -它当场抓到一个手写测试全绿而真实数据报错的 bug：**逐工单事件端点不返回 `issue` -字段**（上下文已经在 URL 里）。第一版按仓库级响应写成必填，换端点后整片解不出来。 - -**这条对后来者是硬要求**：接一个没有公开实例、拿不到真实响应的来源， -交付时必须说明"未经真实实例验证"，别跟这两个混为一谈。 - -## 截断要说出来 - -Kafka 项目有 **14506** 张工单，一轮翻页上限取 500。不报的话界面上"同步完成" -是一句误导——所以取回数小于 `total` 时落一条 warn，说明本轮只覆盖了一段。 - -与 #108「部分抽取报告成完成」、#127「一条坏记录不该毁掉一整块」同一条原则： -**限了覆盖面就要说出来**。 - -## 已知取舍：时刻被截成了日期 - -连接器把时间戳写进正文时是 `created_at.format("%Y-%m-%d")`——**按 UTC 截到天**。 -抽取器从 "Opened by X on 2026-08-18." 这句话里读出的 `valid_from` 因此是 -day 精度，时刻在入库前就没了。 - -**代价说清楚**：跨 UTC 午夜的事件会差一天。一个 UTC+8 的人在 8 月 19 日 -早上七点开的 issue，GitHub 界面上写着 8-19，我们的文档里写的是 8-18。 - -**为什么仍然接受**： - -- 它没有真的丢。源头原样还在，而两个连接器都是**幂等可重同步**的—— - 真要精确到时刻，重同步一次就能拿回来。这与"不可逆丢失"差一个量级。 -- 精度模型本来就在说实话。`precision = day` 的含义正是"知道哪天、不知哪刻"， - day 精度的日期存成 UTC 午夜、按 UTC 渲染，是自洽的。**真正的错是假装知道**。 -- 这产品目前不问"几点"。它回答"某个时刻世界是什么样"，粒度到天。 - -**什么时候该回头做**：有人要按"当地营业日"统计，或者接了一个事件天然聚在 -午夜附近的源（交接班、开盘收盘）。那时 day 精度不够，才值得加一档 `instant` -精度——连带 CHECK 约束、抽取提示词、渲染分支一起改。 - -顺带记一条同源的规矩，它已经落在代码里：**世界时间与记录时间的时区待遇相反**。 -`valid_from`/`valid_to` 来自文档里的陈述，是日历日期不是时刻，一律按 UTC 渲染 -（转本地会让 UTC-5 的读者看到前一天）；`recorded_at` 这类"我们何时这么认为" -是真实时刻，按看的人的时区渲染。这两处**故意不一样，别统一**。 - -## 为什么没有抽象成 `issues` + provider - -接第二个供应商正是为了看清该不该抽象。答案是**暂时不该**：取法根本不同 -（三次拉取 vs 一次调用、`since` vs JQL、事件 vs 字段变更）。共同的只有 -"一张工单一篇文档、正文带变更史"这条判断——而它已经体现在两边一致的**文档形状**上， -不需要一个共同的 trait 来强制。 - -硬抽一个 provider 接口，会把这些真实差异挤进一堆 `match`。等第三个来了再看： -如果它的取法落在已有两种之一，那时抽象才有形状可依。 - -## 这条线后来延伸到了哪 - -「说清楚一样东西是哪来的」不止管来源。同一句话在本体侧也成立， -而且是这篇写完之后才补上的： - -- **#141**：类的形状开始承载来历——**方 = 词表声明的（有 IRI）， - 圆 = 语料里长出来的**。从前所有自动建的类都写死 `circle`， - 一张图上看不出哪些类是本体里定义过的、哪些是从文档里长出来的。 -- **#145**：一个类被词汇表**认领**（`adopt_iri_onto_key`）之后形状要跟着改。 - 漏了这一处的症状很刺眼：类拿到了 IRI 却仍画成圆——**有 IRI 却是圆的， - 画面就在说谎**。这个缝隙只有真导入一次才撞得到，实测撞上过。 - -两件都不是这篇的主题，但判据是同一条：**来历要看得见**。 -接新来源时值得一并想：这个来源带进来的东西，读者能不能一眼看出它是从哪来的。 - -## 相邻的一层：一个源能被谁挂载 - -本文谈「东西怎么进来」。写完之后加了一层本文没有的东西——**授权**（#142，`data_source_grants`）：注册数据源是部署级动作，而它携带的连接串会到达每一个被授权的工作区，所以挂载只能在授权过的集合里挑，守卫在两侧都有（列表按工作区过滤 + 挂载端点复核）。 -判据与本文一致：来历要看得见，去向也要管得住。 - -## 待做：飞书 / Confluence / Notion - -文档协作类是下一个该接的——它是 #122 那份维基历史快照语料的**企业版**： -同一份文档在不同时刻的版本，天然带版本号，不用采样也不用跟埋在 PR 里的事件较劲。 -四条判据全中。 - -**飞书优先**（`feishu_docs`）：中文用户的实际落点，而且它的 API 直接给文档版本。 - -三个已知的未知，接的人要先答： - -1. **拿不到公开实例。** 飞书/Confluence/Notion 都要租户与凭据，所以做不到 - GitHub/Jira 那样的真实端到端。要么用测试租户，要么明说未经真实验证。 -2. **富文本不是字符串。** 飞书文档是块结构（`docx` 的 block 树）， - Notion 是 block、Confluence Cloud 是 ADF——都需要一个"块树 → 纯文本"的渲染器。 - Jira Cloud 的 v3 也是同一个问题，本轮绕开了（走 v2）。 -3. **"变更史"在这里是什么。** 工单的变更史是事件流；文档的变更史是**版本序列**， - 更接近 #122 那份语料的做法——可能要按"变了多少"采样，而不是每个版本都取。 - -### 照着走的骨架 - -两个已有实现是同一个形状，抄它就行：`github_issues.rs`（三次拉取 + 逐工单事件）、 -`jira_issues.rs`（一次调用取全，`expand=changelog`）。 - -1. **纯函数 `render()`** 把一条记录排成 Markdown——不联网，于是测得动 -2. **`fetch_all()`** 负责分页与鉴权 -3. `ingest_sources.rs` 里加一个 `sync_*` 分支，调 `ingest_item()`——身份、 - sha256 去重、版本记录都由它负责 -4. `sources::KINDS` 白名单 **加前端三处**（`api.ts` 的 `SourceView["kind"]`、 - `Library.tsx` 的建来源对话框、`SourcesRail.tsx` 的图标与 `SYNCING_KINDS`） - -第 4 步最容易漏，症状是**界面上选得到、建的时候报 `kind must be one of…`**—— -单元测试与 tsc 都看不见，只有端到端会撞上（#134 就是这么撞的）。 - -另外注意凭据只进不出：编辑来源时留空 = 保留库里原值，写法见 `sync_custom`。 - -### 验收 - -- 单元测试覆盖 `render()`（含"空评论不留空节"这类边界） -- 拿得到真实响应就加夹具测试；**拿不到就在 PR 里明说未经真实实例验证** -- 端到端：建来源 → 同步 → 文档带版本/变更信息 → 二次同步幂等（新增 0） -- `cargo clippy --workspace --all-targets` 与 `npm run typecheck` 干净 - -Confluence / Notion 是同一类，上面第 2 条未知三家共通。 +# 0013 · A source hands over its history + +- **Status**: implemented for `github_issues` (#134), `jira_issues` (#135) and `notion` (#213, pages keep their own clock); WebDAV shares and object storage (S3 / Azure / GCS) arrived as plain file sources (#200, #207, #209) · Feishu and Confluence not started · the `instant` precision has not been triggered +- **Written**: 2026-08-31 · condensed into English 2026-09-03 +- **Related**: the bitemporal ground of [0001](0001-ontology-import-and-governance.md); the same judgment on the corpus side in `scripts/bench/fetch-wiki-history.mjs` (#122); [0012](0012-the-ontology-is-a-contract-not-a-suggestion.md) is the other end of the line — this record is about how things come in, that one about the rules they land by; the grant layer added afterwards (#142, `data_source_grants`) decides who may mount a source: provenance visible, destination governed + +## The problem + +The product is a bitemporal ledger, so a source is worth connecting only if it passes four +tests: real timestamps (the recorded-time axis depends on them); the ability to contradict +itself (otherwise `supersedes` has nothing to do); a stable identity so a new version of the +same thing is recognized (`external_key`); and enterprise knowledge actually living there. +Miss one and it degrades into another web scraper. Issue trackers pass all four. + +The valuable part of a ticket is not "it is closed now" but "opened 08-18, closed 08-20, +assigned to whom in between, priority changed how". Syncing only the current state builds +that timeline sync by sync, and everything before the first sync is lost — while most systems +already keep the change history and only need to be asked. #122 made the same call for +Wikipedia; there revisions have to be sampled, an issue tracker hands the events over. + +## Decisions + +1. **One ticket is one document, and the body carries the history.** Both connectors render + the same shape on purpose: a heading, dated declarative sentences (`Currently Closed.`, + `Resolved on 2011-07-19.`), a `## History` list of `date — who changed Field: A → B`, then + `## Comments`. Sentences with dates, never key-value pairs: `Opened by X on 2026-08-18` + extracts to a fact with `valid_from`; `created_at: 2026-08-18` leaves the model to guess. + +2. **GitHub fetches events per ticket.** The first version routed everything through + repository-level endpoints to spare 200 tickets 401 requests. Real data showed the events + leg was wrong: `issues/events` has no `since`, pages from newest backwards, and pull + requests produce issue events too — this repository's issue events sat on page 5, and a + PR-heavy repository pushes them past the page cap. The state history came back silently + empty, and it is the reason the source exists. Per-ticket fetching costs N requests, N + being the tickets written this round; accuracy over economy here. + +3. **Jira's traps are all in field shape.** Timestamps are `2026-08-24T11:11:52.944+0000`, + no colon in the offset, not RFC3339; an unparsed one drops a whole page. Incremental sync + has no `since` and goes through JQL `updated >= "…"` in Jira's own format with quotes, and + both mistakes surface as 400, not as an empty result. `fields` must be listed explicitly: + omit `comment` and no comments come back; list nothing and one response runs to hundreds + of KB. `expand=changelog` returns the field-level history in one call. + +4. **Fixtures come from real responses.** Hand-written JSON only proves that the shape I + imagined parses. Both fixtures come from publicly readable instances (`deeplethe/utopia`, + `issues.apache.org`), trimmed to the declared fields — the trimming itself shows that + undeclared fields do not break serde. It caught a bug the hand-written tests passed: the + per-ticket events endpoint does not return the `issue` field (the context is in the URL), + and the first version had it as required. A source without a public instance must be + delivered with "not verified against a real instance" stated. + +5. **Truncation is reported.** Kafka has 14506 tickets; a round caps at 500 pages. When the + fetched count is below `total`, a warning says this round covered a slice, otherwise "sync + complete" misleads. Same principle as #108 (partial extraction reported as complete) and + #127 (one bad record must not sink a chunk). + +6. **Timestamps are cut to the day, deliberately.** The connectors write + `created_at.format("%Y-%m-%d")` in UTC, so `valid_from` extracted from + "Opened by X on 2026-08-18." has day precision, and an event across UTC midnight is off by + one day for a UTC+8 reader. Accepted because nothing is truly lost (both connectors are + idempotent and resyncable); because `precision = day` says exactly "the day is known, the + instant is not" — the real error would be pretending to know; and because the product does + not yet ask "at what hour". Add an `instant` precision (CHECK constraint, extraction prompt + and rendering branch together) only when someone needs local business days or a source + whose events cluster around midnight. + + A related rule is already in code: world time and record time get opposite timezone + treatment. `valid_from` / `valid_to` are calendar dates from statements in documents and + render in UTC (local conversion shows a UTC-5 reader the previous day); `recorded_at` is a + real instant and renders in the viewer's timezone. The difference is intentional. + +## Dead ends + +- **An `issues` + provider abstraction.** The second vendor was connected to find out whether + to abstract. Not yet: the fetch strategies differ at the root (three pulls vs one call, + `since` vs JQL, event-level vs field-level history). What they share — one ticket, one + document, history in the body — already lives in the identical document shape and needs no + trait; a forced interface would push the real differences into a pile of `match`. + Reconsider when a third source lands in one of the two existing shapes. + +## The same judgment elsewhere + +Provenance has to be visible, and that reached the ontology after this record: a class's +shape carries its origin (#141: square = declared by a vocabulary with an IRI, circle = grown +from the corpus), and a class adopted by a vocabulary (`adopt_iri_onto_key`, #145) changes +shape — a class with an IRI drawn as a circle is a picture that lies. + +## Open questions + +- **Feishu / Confluence / Notion** are next: the enterprise version of the #122 Wikipedia + history corpus — versions of one document at different times, numbered, no sampling and no + events buried in PRs. Feishu first (`feishu_docs`): it is where Chinese users are, and its + API hands out document versions. Three unknowns to answer first: no public instance (a test + tenant, or "not verified" stated); rich text is a block tree (Feishu `docx` blocks, Notion + blocks, Confluence ADF; Jira v3 has the same problem, avoided this round via v2) and needs a + block-tree-to-text renderer; and "history" here is a version sequence, closer to #122 — + probably sampled by how much changed rather than every version. + + The skeleton is the two existing connectors: a pure `render()` turning one record into + Markdown (no network, so testable); `fetch_all()` for paging and auth; a `sync_*` branch in + `ingest_sources.rs` calling `ingest_item()`, which owns identity, sha256 dedup and version + records; and the `sources::KINDS` whitelist plus three frontend places (`SourceView["kind"]` + in `api.ts`, the create dialog in `Library.tsx`, icons and `SYNCING_KINDS` in + `SourcesRail.tsx`). The fourth step is the one that gets missed: the kind is selectable and + creation fails with `kind must be one of…`, invisible to unit tests and tsc (#134 hit it). + Credentials go in and never out — an empty field on edit keeps the stored value, as in + `sync_custom`. Acceptance: unit tests on `render()`, fixture tests where a real response is + obtainable, create → sync → versions present → second sync adds 0, and clean + `cargo clippy --workspace --all-targets` and `npm run typecheck`. + +## Revisions + +- 2026-09-03: every connector's credentials stay on the server (#246). Until now only `auth_header` was stripped from responses; the object-storage, WebDAV and Notion keys went out to every viewer. The keys now live in one list, `SOURCE_SECRET_KEYS`, shared by the listing, the create / update responses and the update merge (blank or missing keeps the stored value, an explicit `null` removes it). Adding a connector means adding its keys there first. +- 2026-09-03: the five connectors added under this record could not be created (#247): the store's hand-written `KINDS` allowlist stopped at seven kinds while the sync dispatcher and the UI knew twelve. The kinds now come from one enum, `SourceKind` in `utopia-core`; the allowlist is derived from it, the dispatcher matches it exhaustively, and a test compares the frontend's `web/src/sourceKinds.ts` against it, so the three can no longer drift apart. diff --git a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md index 14dd5db7..486655dd 100644 --- a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md +++ b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md @@ -1,139 +1,82 @@ -# 0014 · 身份跟着人，范围跟着令牌 - -- **状态**：已实施（#161 记录、#180 落地）· `personal_tokens` + Streamable HTTP 的 MCP 服务端已上线，暴露五个只读工具；界面在 `feat/tokens-have-a-page`（0016 A2）：账户层「Agents & tokens」页，发放时明文与 MCP 客户端配置片段一起只显示一次，列表只剩前缀，撤销留痕（2026-09-02 核，修订见文末） -- **成文**：2026-09-01（约定见 [README](README.md)） -- **相关**：迁移 `0014_data_source_grants` 刚给数据源补上授权层——本篇是同一个问题 - 换到「机器来敲门」这一侧；[0004](0004-language-and-localization.md) 定下服务端只说英文， - 错误码这一条对 MCP 同样适用 - -> 起因是要把 Utopia 的七个工具暴露成 MCP 服务端。**第一个要回答的不是传输层选哪个， -> 是客户端以什么身份接进来。** 这一篇只答这个。 - -## 现状：两种凭据，都不合身 - -| | 跟着谁 | 存法 | 过期 | 能撤吗 | -|---|---|---|---|---| -| JWT | 人 | 无状态 | 7 天 | **不能**——签出去就管不了 | -| `sources.ingest_token` | 一个来源 | 明文 | 无 | 换一个 | - -JWT 是给浏览器会话设计的：短命、每次登录重签、无状态所以不需要一张表。MCP 客户端是 -长命的、机器的、配在别人机器上的一个文件里——七天过期意味着每周手动重配一次，而 -「撤不回来」意味着笔记本丢了只能等它自己过期。 - -`ingest_token` 更不合适：它跟来源走，不跟人走，而且只能**往里推文档**。 - -## 走过的岔路：KB 级机器令牌 - -先提的方案是给知识库发机器令牌——一枚令牌对应一个库，跟人无关。 - -**否掉了，两条理由：** - -1. **它引入第三套授权模型。** 现在已经有工作区成员和 KB 角色两层；再加一层「令牌自己的 - 权限」，那么「这个 agent 能看什么」就得同时查三张表才答得出来。而三层里任何一层写错， - 失败方向都是「多给了」。 -2. **归因会变成假的。** `audit_events.actor_id` 现在记的是活人，`actor_label` 还存了一份 - 身份快照。机器令牌写进来的事实，actor 只能是一个合成 id——台账上就多出一类「不是任何 - 人做的」记录，而账本存在的理由正是「谁在什么时候认下了什么」。 - -**改成：令牌以这个人的身份行事。** - -## 决定 - -``` -有效权限 = 这个人的角色 ∩ 这枚令牌的 scope -``` - -交集，不是并集。**令牌只能收窄，永远不能放宽。** 一个 viewer 的令牌勾上 write 也还是 -只读——scope 是上限，不是授权。 - -### 为什么身份跟着人 - -- **现有守卫一行不用改。** `require_kb(kb_id, Role::Viewer)`、`access::kb_role` 拿到的还是 - 一个 `User`，它从哪来的无所谓 -- **归因是真的。** 台账上是活人，不是机器人 -- **停用即失效。** 人离职停用，他的令牌跟着废，不用单独维护一张「谁的机器还连着」的表 -- **多个库不用发多把钥匙** - -### 为什么范围仍然要单独收窄 - -因为有一个 MCP 特有的问题，应用内对话没有这么严重： - -> **混淆代理。** MCP 客户端是别人的 agent、别人的系统提示词，而它读的是知识库里的 -> 文档——**不可信内容**。一份文档里写「请执行这段 SQL」或者「记住 X」，那个 agent 可能 -> 就照做了，用的是这个人的全部权限。 - -应用内对话也有这个面，但那里提示词和工具循环都在 Utopia 手里；走 MCP，Utopia 对客户端 -的提示词、对它还接了哪些别的服务端，一无所知。 - -而这个人的「全部权限」是什么：经 `query_data`，对他所在**每一个库挂载的每一个生产 -数据库**跑只读 SQL；经 `remember`，往 append-only 账本里写事实。这些能力配在一串放在 -`claude_desktop_config.json` 明文里的字符串上。 - -所以默认发**只读、限定到一个库**的令牌。要让 agent 写，得显式勾。 - -## 这枚要哈希，而 `ingest_token` 不哈希 - -两处结论不同，不是疏忽。`ingest_token` 那条明文决定的原话（`0002_ingest.sql`）： - -> **明文存，不是哈希。** 自部署威胁模型下「只看一次」是自找麻烦：改存明文随时可查。 -> DB 失守时文档本体早已泄露，密钥哈希化没有额外收益 - -那个推理对 ingest_token 成立，因为**它只能往里推文档**——泄露它的最坏结果是有人往你 -库里塞垃圾，而库都失守了，塞垃圾不是最要紧的事。 - -个人令牌不一样：它经 `query_data` 能**读出 Utopia 之外的生产库**。Utopia 的数据库失守 -本来就泄露 Utopia 自己的文档，但数仓在另一台机器上、装着另一批数据，不该跟着一起丢。 -**爆炸半径不同，所以存法不同。** - -## 形状 - -```sql -CREATE TABLE personal_tokens ( - id UUID PRIMARY KEY, - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - name TEXT NOT NULL, -- 人自己起的，"我的笔记本" - token_hash TEXT NOT NULL, -- 哈希，理由见上 - scope TEXT NOT NULL DEFAULT 'read' - CHECK (scope IN ('read', 'write')), - kb_ids UUID[], -- NULL = 这个人能进的全部 - expires_at TIMESTAMPTZ, -- NULL = 不过期，但 UI 默认给 90 天 - last_used_at TIMESTAMPTZ, -- 「这枚还在用吗」，撤之前要答得出 - revoked_at TIMESTAMPTZ, -- 撤销不删行：撤过这件事本身要留痕 - created_at TIMESTAMPTZ NOT NULL DEFAULT now() -); -``` - -`user_id` 这条有外键且级联，与 `audit_events.actor_id` 的裸外键相反——**台账要活得比 -用户久，令牌不该**。人没了，他的钥匙就该一起没。 - -## 一条实施纪律 - -**每个工具入口都要校验 scope，不能只在连接握手时校验一次。** - -这是 `0014_data_source_grants` 那轮的教训，原话写在测试里： - -> 列表过滤只挡「看得见」，而挂载端点是照着 id 调的——守卫必须在两侧都有 - -MCP 的对应形态是：握手时校验一次，然后整条连接生命周期里都信任它。工具调用是一个个 -独立请求，`revoked_at` 在中途被写上时，正在跑的连接必须立刻失效。 - -## 修订记录（2026-09-02）：落地之后对照本文 - -**形状**多一列 `token_prefix`（`utp_pat_…`，与 ingest 的 `utp_` 区分——日志与配置文件里一眼要认得出是哪一种），`token_hash` 加了 `UNIQUE`。哈希选 SHA-256 不选 argon2：高熵串不怕爆破，而每行盐不同就查不了唯一索引。 - -**「每个工具入口都校验 scope」那条纪律被满足了，但形态与预想不同**：做成完全无状态——每个 POST 重跑一次认证（撤销 / 过期在 SQL 的 `WHERE` 里判）+ `covers()` + `require_kb`，没有「连接」这个东西可以被信任。`scope` 在这一版没有分支：`can_write` 硬编码 `false`，就算令牌勾了 write 也不放开。每次工具调用写一条审计（`mcp.tool_called`，target 是令牌），归因是真的。 - -**前置条件本文没记**：#175 把七个工具的执行从 `chat.rs` 的 `match` 里抽成 `tools.rs`，对话与 MCP 共用同一份实现——否则「对话里的 `entity_facts` 和 MCP 里的不是同一个东西」。工具的 JSON schema 仍留在 `chat.rs`，MCP 复用它做转换，已知带一处瑕疵：`search_chunks` 的描述里还写着「可以引用成 [n]」，而 MCP 客户端拿不到引用编号。 - -**一处会误导人的陈述**：`crates/utopia-mcp` 曾是三行占位，宣称的三个工具名（`add_memory` / `search_memory` / `get_entity_timeline`）从未实现，MCP 服务端住在 `utopia-server/src/api/mcp.rs`。〔已删（2026-09-02，连同 `utopia-graph`、`utopia-connectors`），理由见 [0016](0016-close-the-open-seams-before-cutting-new-ones.md) A2。〕 - -## 未决 - -- **`query_data` 与 `remember` 要不要进第一版。** 倾向不进——先发四个只读工具 - （`search_chunks` / `find_entities` / `entity_facts` / `changes`），把身份这条路走通再说。 - 这两个各自还有没答的问题：外部 agent 写进来的事实挂什么证据？跑 SQL 的审计怎么记？〔**已答：不进**。实际发的是五个而非四个，多一个 `search_docs`。`remember` 的前提是 [0015](0015-recording-a-sentence-is-not-asserting-a-fact.md) 那道闸，而闸还没接上。〕 -- **传输层**：stdio（本地，配 Claude Desktop 最省事）还是 streamable HTTP（远程， - 和 Utopia 已经是个服务端相称）。这个选择不影响本篇的结论，两种都要认令牌。〔**已答：Streamable HTTP**，一条路由 `POST /api/v1/kbs/{kb_id}/mcp`，响应用 `application/json` 不用 SSE——工具一问一答，没有服务端主动推的东西。〕 -- **令牌能不能跨工作区。** 现在的 `kb_ids` 是库级白名单；如果将来要按工作区发， - 和数据源授权那张表会长得很像，届时看要不要合并概念。〔仍未做。〕 -- **（2026-09-02 补）没有界面。** 「UI 默认给 90 天」的 90 天在服务端，而前端根本没有令牌那一页。这是 MCP 今天对用户不可用的直接原因。〔**已做**，同日：`/account/tokens`。三个决定落在界面上——scope 缺省只读、库多选、过期缺省 90 天且「不过期」要显式选；配置片段按库出端点（令牌限定到库，端点也按库分），写法取 Claude Code / Claude Desktop 一族的 Streamable HTTP 键名，别的客户端要自己改键。〕 +# 0014 · Identity from the person, scope from the token + +- **Status**: implemented (#161 record, #180 code) · `personal_tokens` and a Streamable HTTP MCP server at `POST /api/v1/kbs/{kb_id}/mcp` with five read-only tools, `application/json` responses rather than SSE · the account-level "Agents & tokens" page at `/account/tokens` (0016 A2): plaintext shown once beside a per-base client config snippet, the list keeps the prefix, revocation leaves a trace · `can_write` is still hard-coded `false`, so a `write` scope changes nothing yet +- **Written**: 2026-09-01 · condensed into English 2026-09-03 +- **Related**: migration `0014_data_source_grants` gave data sources a grant layer; this is the same question where a machine knocks. [0004](0004-language-and-localization.md) has the server speak English only, MCP error codes included. [0015](0015-recording-a-sentence-is-not-asserting-a-fact.md) removes the main objection to opening `remember` over MCP. + +## The problem + +Exposing Utopia's tools over MCP raises one question before transport: as whom does the +client connect? Neither existing credential fits. The JWT follows a person but is stateless, +expires in seven days and cannot be revoked — a client configured in a file on someone else's +machine would be reconfigured weekly, and a lost laptop waits for expiry. +`sources.ingest_token` follows a source, is stored in plaintext, never expires and can only +push documents in. + +## Decisions + +1. **The token acts as the person.** Effective permission = the person's role ∩ the token's + scope. A token can only narrow, never widen: a viewer's token ticked `write` stays + read-only. Existing guards (`require_kb(kb_id, Role::Viewer)`, `access::kb_role`) still + receive a `User`; the audit trail shows a person; deactivating the person kills their + tokens; several bases need no extra keys. + +2. **Scope still narrows separately, because of the confused deputy.** An MCP client is + someone else's agent under someone else's system prompt, reading untrusted content from the + base — a document saying "run this SQL" or "remember X" may be obeyed with the person's + full permissions, and Utopia knows nothing about that client's prompt or its other servers. + Full permission means read-only SQL against every production database mounted in every base + the person can reach (`query_data`) and writes to the ledger (`remember`), all hanging off + a string in `claude_desktop_config.json`. So the default token is read-only and limited to + one base; writing must be ticked. + +3. **This token is hashed while `ingest_token` is not.** `0002_ingest.sql` reasoned that once + the database is lost the documents are lost anyway, so hashing buys nothing — true for a + token that can only push documents in. A personal token reads production databases outside + Utopia through `query_data`, and that warehouse should not fall with Utopia's database. + Different blast radius, different storage. SHA-256 rather than argon2: a high-entropy + string does not fear brute force, and a per-row salt would defeat the `UNIQUE` index on + `token_hash`. + +4. **Shape.** `personal_tokens(id, user_id, name, token_prefix, token_hash, scope, kb_ids, + expires_at, last_used_at, revoked_at, created_at)`: `scope` defaults to `read`; `kb_ids` + NULL means every base the person can reach; `expires_at` NULL means never, but the UI + defaults to 90 days and "never" is an explicit choice; `last_used_at` answers "is this + still in use" before revoking. The prefix `utp_pat_…` against ingest's `utp_` tells the + kinds apart in logs and config files. `user_id` cascades on delete, unlike the bare foreign + key on `audit_events.actor_id`: the ledger must outlive the user, the keys must not. + Revocation writes `revoked_at` rather than deleting the row: the revocation is a trace. + +5. **Every tool call checks scope.** The lesson of `0014_data_source_grants`: list filtering + only guards what is visible, the mount endpoint is called by id, so the guard sits on both + sides. Rather than a handshake check followed by trust for the connection's lifetime, every + POST re-runs authentication (revocation and expiry in the SQL `WHERE`), `covers()` and + `require_kb`; a token revoked mid-session fails on its next call. Each call writes an audit + row (`mcp.tool_called`, target = the token), so attribution is real. + +## Dead ends + +- **Base-level machine tokens**, one per base and unrelated to a person. It would be a third + authorization model beside workspace membership and base roles — "what can this agent see" + needs three tables, and a mistake in any fails towards "gave too much". And attribution + becomes fake: `audit_events.actor_id` records living people with an `actor_label` snapshot, + while a machine token's facts would belong to a synthetic id. + +## Revisions + +- 2026-09-02: an unstated precondition — #175 moved the seven tools out of the `match` in + `chat.rs` into `tools.rs`, so chat and MCP share one implementation. The JSON schemas stay + in `chat.rs`; `search_chunks` still says results can be cited as `[n]`, which MCP clients + cannot see. +- 2026-09-02: the placeholder crate `utopia-mcp` advertised three tools that never existed; + the server lives in `utopia-server/src/api/mcp.rs`. Deleted with `utopia-graph` and + `utopia-connectors` ([0016](0016-close-the-open-seams-before-cutting-new-ones.md) A2). + +## Open questions + +- **`query_data` and `remember` over MCP.** The first version ships `search_chunks`, + `search_docs`, `find_entities`, `entity_facts` and `changes`. `remember` waits on the gate + of 0015; what evidence an external agent's fact carries and how SQL runs are audited are + still unanswered. +- **Tokens across workspaces.** `kb_ids` is a base-level whitelist; a workspace-level grant + would look much like `data_source_grants`, and the two concepts may merge then. diff --git a/docs/decisions/0015-recording-a-sentence-is-not-asserting-a-fact.md b/docs/decisions/0015-recording-a-sentence-is-not-asserting-a-fact.md index 3c9a3863..1d33c584 100644 --- a/docs/decisions/0015-recording-a-sentence-is-not-asserting-a-fact.md +++ b/docs/decisions/0015-recording-a-sentence-is-not-asserting-a-fact.md @@ -1,168 +1,90 @@ -# 0015 · 记下一句话，不等于断言一个事实 - -- **状态**：已实施 · schema 在迁移 `0018`（#180），运行时随 [0016](0016-close-the-open-seams-before-cutting-new-ones.md) A1 接上：抽取分流进 `pending_facts`、Review 新一档、对话里跟在 remember 后面的确认卡、`remember` 重新打开。**决定 3 的措辞与本文写的不同**，见文末最后一条修订 -- **成文**：2026-09-01（约定见 [README](README.md)） -- **相关**：[0010](0010-no-relation-is-no-relation.md) 删掉兜底关系（本篇那条空谓词正是它的 - 正确行为）、[0011](0011-a-mapping-is-not-a-fact.md) 批过「拿浮点数编码二值状态」—— - 本篇的实现红线；[0014](0014-identity-from-the-person-scope-from-the-token.md) 里 - 「MCP 不开写」的主要顾虑，被本篇的闸消掉 - -> 起因是一次实测，不是推演。跑通 `remember` 之后回头查库，看到的东西和助手说的 -> 对不上。 - -## 那一次实测 - -对话里说： - -> Please remember this: Acme moved its headquarters to Shenzhen on 2026-03-15. - -助手答： - -> I've recorded that **Acme moved its headquarters to Shenzhen** on March 15, 2026. - -而图里落下的是： - -``` -Acme --(空谓词)-> Shenzhen confidence 0.9 invalidated_at 空 -``` - -谓词是空的——本体里没有「搬迁到 / 总部位于」这类关系，抽取落不上就留空。**按 0010 -这是正确行为**（不编一个 `related_to` 出来）。但结果是图上多了一条 0.9 置信、没有 -语义的边，而且**说的和进去的不是一回事，人没有任何办法发现**。 - -## 现状：根本没有确认这道闸 - -查之前以为 `unconfirmed` 队列就是它。不是，它的定义是： - -```sql --- 待确认 = 有证据、但证据所在的分块全被新版本取代了 -``` - -那是**证据过期**队列。`lowconf` 是 `confidence < 0.75`。两个都是事后清理：低置信事实 -在队列里躺着的时候，它已经是图上一条活边（`invalidated_at IS NULL`）。 - -所以：**任何抽取出来的事实，一落库就生效，没有任何一步需要人点头。** - -## 这对批量摄入是对的，对 `remember` 不是 - -灌 500 篇文档抽出一万条事实，让人逐条确认是不可能的。乐观写入 + 事后审阅是那条路上 -唯一可行的设计，本篇不动它。 - -`remember` 是另一回事，三点都不一样： - -| | 文档摄入 | remember | -|---|---|---| -| 量 | 一次上万条 | 一次一句 | -| 人在哪 | 上传完就走了 | **就在对话里** | -| 素材 | 外部证据 | 人自己特意说的一句话 | - -确认成本最低的那一刻，恰好就是他说完那句话的那一刻。 - -## 真正的病不是「少一道闸」 - -是**助手宣称的和图里得到的不一致**。 - -「已记录 Acme 把总部搬到深圳」听起来像是那件事进去了；实际进去的是一条无谓词的边。 -所以二次确认的价值主要不在多点一下，在于**让人看见即将断言的是什么**——看到 -`Acme --?-> Shenzhen`，人会立刻说这不对。 - -这也决定了确认界面该长什么样：**原句在上，抽出的三元组在下**，两者并排。只列三元组 -等于要人凭空判断它对不对。 - -## 决定 - -1. **`remember` 写文档这一步不变。** 那只是记录「你说过这句话」，本身无害，也确实 - 该立刻可检索。 -2. **从记忆抽出的事实，进图前要人确认。** 未确认的不参与检索、不上图、不进推理。 -3. **助手的话照实说**：「已记录，抽出 N 条待你确认」，不宣称完成。 - -## 修订记录：第一版把状态加在 `facts` 上，错了 - -**原本的方案**是给 `facts` 加三列（`nod` / `nodded_by` / `nodded_at`）， -`nod = 'pending'` 表示等人点头。迁移写完、`insert_fact` 的参数也穿好了， -才发现这是**这个仓库半个月前刚踩过的坑**。 - -`0013_reasoning.sql` 里那段话，把这个形状的问题说得比我清楚： - -> 试过塞进 `facts` 加一位 `derived_by_rule` 标记，那一版的问题是**失败方向反了**： -> 仓库里有四十多处读 `facts` 的查询，其中只有一处认识那个标记，于是新写一条 -> 查询默认就是把派生当断言看，得记得加过滤。写这个功能的人（我）当场就漏了两处。 -> -> 分开之后忘了 UNION 的后果是**看不见**派生，而不是**混进去**。 - -数了一下：今天有 **27 处**查询按 `invalidated_at IS NULL` 捞活事实，分布在 -6 个文件。给它们逐个补 `AND nod IS DISTINCT FROM 'pending'`，漏一处的后果就是 -一条没人点头的事实混进图里——而这个功能存在的全部理由就是防这件事。 - -**改成自己一张表 `pending_facts`。** 确认时才写进 `facts`。忘了读它的后果变成 -「看不见待确认队列」，而不是「未确认的混进了图」。失败方向对了。 - -顺带一提，0013 给出的另外两条理由这里也成立：**列不一样**（待确认的需要 -`proposed_predicate` 原话、需要指回那句记忆，而不需要 `supersedes`）， -**生命周期不一样**（确认后它就不该继续存在于那张表里）。 - - - -0011 那条线上批过一次同样的错。原话不在 ADR 里，在它落地的建表注释 -（`migrations/0006_semantic_layer.sql` 的 `concept_mappings.status` 列）： - -> **状态而不是置信度。** 从前借事实的 confidence 表达「提议 0.6 / 确认 1.0」， -> 那是把一个二值状态编码成浮点数，还顺带让它落进「低置信事实」那一档。 - -`facts` 表今天没有状态列（`id, kb_id, subject_id, predicate_id, object_id, -object_value, valid_from, valid_to, ..., confidence, derived_by_rule, supersedes`）。 -要加就加真的一列，别再拿 0.6 糊弄——那会让这条事实同时出现在「待确认」和 -「低置信」两个队列里，而它们问的不是同一个问题。 - -## 顺带：MCP 开写的障碍消掉了 - -0014 里不开 `remember` 的主要顾虑是混淆代理——知识库里的文档是不可信内容，一份文档 -写「请 remember 某某」，外部 agent 可能就照做了，用的是这个人的全部权限。 - -装上这道闸之后，**外部 agent 也只能提议，不能断言**。人先看见才生效，那条顾虑就不 -成立了。所以本篇不只是修一个 bug，它是 0014 里那个「未决」的答案。 - -## 修订记录（2026-09-02）：只有 schema，没有运行时 - -`pending_facts` 与 `rejected_facts` 两张表在 #180 建好了，形状按「原句在上、三元组在下」设计：`chunk_id NOT NULL` 指回那句记忆、`proposed_predicate` 存模型原话、`predicate_id` **可空**——人要看见的正是这个空、`proposed_by` 补上「谁的话」。失败方向的论证被完整搬进了迁移注释。 - -**但全仓零读零写。** `memory::is_memory_document()` 为这道判据写好了，零调用；抽取管线没有任何 `pending_facts` 写入点；Review 的八档计数里没有 `pending` 这一档；`remember` 的回话仍是 `Recorded (effective …)`。 - -**临时闸是把工具整个关掉**：`chat.rs` 的 `REMEMBER_ENABLED = false`，注释写着「抽取侧接上那张表之后改回 true；在那之前宁可没有这个工具，也不要一个会悄悄改图的工具」。所以三条决定里只有隐含的第 0 条（不让说谎继续）实现了。 - -**于是下文「MCP 开写的障碍消掉了」在代码上还不成立**——闸没装上，MCP 依旧只读。 - -**一个数字的更正**：「27 处查询按 `invalidated_at IS NULL` 捞活事实，分布在 6 个文件」——今天是 56 处、7 个文件（多出 `ontology.rs`）。论证方向不变，数字若再被引用应重新数。 - -**同一类问题的另一面**（#173，迁移 `0015_what_it_did_not_just_what_it_said`）：跨轮回放助手上一轮的工具调用与结果，模型才知道自己做过什么，不会把上一轮的检索重跑一遍落到另一批同名实体上。本文讲的是「助手说的」与「图里得到的」之间的落差，那一条讲的是「助手说的」与「助手做的」之间的落差。 - -## 修订记录：接上运行时之后，三处与本文不同 - -**一、助手说不出「抽出 N 条」。** 决定 3 写的是「已记录，抽出 N 条待你确认」。抽取是异步走队列的， -`remember` 回话的那一刻抽取还没跑，N 不存在。两条路：把记忆抽取改成同步（一句话十来秒，卡住整个对话循环）， -或者如实说「这句话记下了，从中抽出的事实会先给你确认，确认前不进图」，等抽取完成时再把确认卡**长到对话里**。 -选了后者。卡片靠 SSE 的 `pending` 事件触发，按那句记忆的 chunk 取待确认项；回放旧会话时同样按 chunk 取， -还有没点头的就照样显示，都处理完了就不占地方。 - -**二、确认界面在两处，同一个组件。** Review 页新一档「等你点头」排在最前——它是人自己说的话，而且这一档里的 -东西还没进图，别处每一档审的都是已经在图上的。对话里那张卡是同一行组件。载荷是声明式的（chunk + 三元组列表）， -将来若要让别人的客户端渲染它（MCP 那一侧），换的是渲染器不是服务端。 - -**三、确认走的正是抽取那条路。** 事实 + 证据（指回那句记忆，引句就是整句）+ 时态对账。所以「Mira 交给 -Devin 了」点头之后，Mira 那条会像从文档里抽出来时一样被闭合。置信度不动，人的态度在台账里 -（`fact.nod_confirmed` / `fact.nod_rejected`，快照自包含）。 - -**顺带定下的**：拒绝记录按（主语, 谓词, 宾语实体）查，**字面值事实不查**——`rejected_facts` 没有 `object_value` -列，按（主语, 谓词）挡会把「薪水 28000 被拒」扩大成「薪水这个属性永远不再提」，宁可多问一次。 -实体照常消解并创建，`pending_facts.subject_id` 是外键，这是 0018 定下的取舍；代价是几个暂时孤立的节点。 -「谁说的」从 `remember` 经 `memory_ingest` 与 `extract_document` 的任务载荷一路传到 `proposed_by`。 - -## 未决 - -- **闸只拦记忆，还是拦所有「单条、交互式」的写入？** 今天只有 `remember` 这一条路， - 但将来若有「在图上手工加一条边」的界面，它该走哪边？ -- **确认之后置信度取什么。** 人点了头还留 0.9，还是升到 1.0？0011 的教训说别拿 - confidence 表达人的态度，那大概是「不动它」——但要想清楚。〔**已定：不动**。人的态度在台账里。〕 -- **拒绝之后那条记忆怎么办。** 文档还在（人确实说过那句话），只是没抽出可用的事实。 - 下一轮重抽会不会又提议一遍？`concept_mappings` 那边靠 `status='rejected'` 挡住了 - 重复提议，这里需要对应的东西。〔答了一半：`rejected_facts` 表与查重索引建好了，语义就是「这个三元组在这个库里被拒过」，但没有任何代码写它或查它。〕 +# 0015 · A recorded sentence waits for a nod + +- **Status**: implemented · schema in migration `0018` (#180: `pending_facts`, `rejected_facts`) · runtime wired in [0016](0016-close-the-open-seams-before-cutting-new-ones.md) A1: extraction from a memory document goes to `pending_facts`, Review has a "waiting for your nod" queue placed first, a confirmation card grows into the chat after `remember`, `REMEMBER_ENABLED` is `true` again · decision 3 landed with different wording, see Revisions · MCP is still read-only; opening `remember` there is the next cut +- **Written**: 2026-09-01 · condensed into English 2026-09-03 +- **Related**: [0010](0010-no-relation-is-no-relation.md) removed the fallback relation (the empty predicate below is its correct behavior); [0011](0011-a-mapping-is-not-a-fact.md) rejected encoding a binary state as a float, the red line for this implementation; [0014](0014-identity-from-the-person-scope-from-the-token.md) kept MCP read-only mainly because of the confused deputy, which this gate removes + +## The problem + +A real run, not a thought experiment. The user said "Please remember this: Acme moved its +headquarters to Shenzhen on 2026-03-15." The assistant answered "I've recorded that Acme +moved its headquarters to Shenzhen on March 15, 2026." The graph got +`Acme --(empty predicate)--> Shenzhen`, confidence 0.9, live. The ontology has no "moved to / +headquartered in", so the predicate stayed empty — correct by 0010 — but the graph gained a +meaningless 0.9 edge, and what was said and what went in differed with no way to notice. + +There was no confirmation gate at all. `unconfirmed` is the queue for facts whose evidence +chunks were all superseded; `lowconf` is `confidence < 0.75`; both are after-the-fact cleanup +on facts that are already live (`invalidated_at IS NULL`). Every extracted fact takes effect +on insert. That is right for bulk ingest — nobody confirms ten thousand facts from five +hundred documents one by one. `remember` differs on all three axes: one sentence at a time, +the person is still in the conversation, and the material is something they said on purpose. +The cheapest moment to confirm is right after they said it. + +The disease is not the missing gate but the gap between what the assistant claims and what +the graph gets. The value of confirming is mostly that the person sees what is about to be +asserted — shown `Acme --?-> Shenzhen`, they say "that's wrong" at once. So the card shows the +original sentence above the extracted triples; triples alone ask for a judgment from nothing. + +## Decisions + +1. **`remember` still writes the document.** That only records "you said this", is harmless + and should be searchable at once. +2. **Facts extracted from a memory wait for a nod before entering the graph.** Unconfirmed + facts take no part in retrieval, the graph or reasoning. +3. **The assistant says what happened** and does not claim completion. +4. **Pending facts get their own table, `pending_facts`,** written to `facts` only on + confirmation. The columns differ (`chunk_id NOT NULL` pointing back at the memory, + `proposed_predicate` with the model's wording, `predicate_id` nullable — the emptiness is + exactly what the person must see — and `proposed_by`), and so does the lifecycle: after + confirmation the row should not exist there. The failure direction is right: forgetting to + read the table hides the queue instead of leaking an unconfirmed fact into the graph. +5. **Confirmation takes the extraction path**: fact plus evidence (the memory chunk, the + whole sentence as the quote) plus temporal reconciliation, so a nod on "Mira handed over to + Devin" closes Mira's fact as a document extraction would. Confidence is untouched; the + person's stance lives in the audit ledger (`fact.nod_confirmed` / `fact.nod_rejected`). + Rejections go to `rejected_facts` and are checked on the next extraction by (subject, + predicate, object entity); literal-value facts are not checked, since `rejected_facts` has + no `object_value` and blocking on (subject, predicate) would turn "salary 28000 rejected" + into "never propose salary again" — better to ask once more. Entities are resolved and + created as usual (`pending_facts.subject_id` is a foreign key), at the cost of a few + temporarily orphaned nodes. `proposed_by` travels from `remember` through the + `memory_ingest` and `extract_document` job payloads. +6. **The MCP objection is gone.** With the gate an external agent can only propose, never + assert; a document saying "please remember X" reaches the person before the graph. This + answers the open item in 0014. + +## Dead ends + +- **Status columns on `facts`** (`nod` / `nodded_by` / `nodded_at`, `pending` = awaiting a + nod). The migration was written and `insert_fact` threaded before it turned out to be the + trap `0013_reasoning.sql` describes for `derived_by_rule`: the failure direction is + reversed. Dozens of queries fetch live facts by `invalidated_at IS NULL` (27 in 6 files when + first counted, 56 in 7 files by 2026-09-02); each would need + `AND nod IS DISTINCT FROM 'pending'`, and missing one leaks an unconfirmed fact — the one + thing the feature exists to prevent. +- **Confidence 0.6 for "proposed".** Rejected once already for `concept_mappings.status` + (0011): a binary state encoded as a float, which also lands the fact in the low-confidence + queue, and the two queues ask different questions. + +## Revisions + +- 2026-09-02: only the schema existed — zero reads and writes, `memory::is_memory_document()` + written but uncalled, and the interim gate was `REMEMBER_ENABLED = false` in `chat.rs` + ("better no tool than one that silently changes the graph"). +- On wiring the runtime, decision 3 changed. The assistant cannot say "N facts extracted": + extraction runs asynchronously and N does not exist when `remember` replies. Making it + synchronous would block the chat loop for ten seconds a sentence; instead the reply says the + sentence is recorded and its facts will be shown for confirmation first, and the card grows + into the conversation on the SSE `pending` event, fetched by the memory's chunk (replayed + sessions fetch the same way). The Review queue and the chat card are one component with a + declarative payload (chunk + triple list), so an external client would swap the renderer. +- The same disease from the other side (#173, migration + `0015_what_it_did_not_just_what_it_said`): replaying the previous turn's tool calls so the + model knows what it did and does not rerun a search onto another set of same-name entities. + This record is "said" versus "got"; that one is "said" versus "did". + +## Open questions + +- Does the gate hold only memories, or every single-item interactive write? Today `remember` + is the only such path; a future "add an edge by hand" interface should take the same table. diff --git a/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md b/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md index eca4adf5..40a9d6c9 100644 --- a/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md +++ b/docs/decisions/0016-close-the-open-seams-before-cutting-new-ones.md @@ -1,123 +1,102 @@ -# 0016 · 先把开着的口子收上，再开新的 +# 0016 · Close the open seams before cutting new ones -- **状态**：规划中 · v0.1.0 之后的第一份排期 -- **成文**：2026-09-02（约定见 [README](README.md)） -- **相关**：本文是对 [0001](0001-ontology-import-and-governance.md) 到 [0015](0015-recording-a-sentence-is-not-asserting-a-fact.md) 逐篇对照代码复核之后写的；每一条「待做」都能在那十五篇里找到出处。复核本身的产物是各篇里 2026-09-02 的修订记录，本文只排顺序、只讲为什么是这个顺序 +- **Status**: in progress · the first schedule after v0.1.0 · A1, A2, A3 done · B1 done (#227) · B2 done (#238 / #243, [0017](0017-a-contradiction-points-upstream.md)) · B3 done: the signature half (#190 / #196), cross-pack signatures and range-aware direction (#233), `disjointWith` into resolution · D2's blank-base problem worked around with builtin `metric` / `dimension` classes (#231), the pack itself still planned · the lakehouse engines landed ahead of D4 (#239, [0018](0018-the-lakehouse-is-one-protocol-away.md)) +- **Written**: 2026-09-02 · condensed into English 2026-09-03 +- **Related**: written after checking [0001](0001-ontology-import-and-governance.md) through [0015](0015-recording-a-sentence-is-not-asserting-a-fact.md) against the code; every item below has its source in those fifteen records, whose 2026-09-02 revision notes are the check's product. This record only orders them and says why this order. -> 这一篇没有实验数字。它是一次盘点：v0.1.0 发出去了，README 的能力表写了十二行， -> 十五篇决策记录里有三篇的状态行落后于同一个 PR 里的代码。**在开下一条大线之前， -> 先把「说的」和「做的」对齐一遍。** +## The problem -## 现状，一段话 +v0.1.0 shipped with twelve capability rows in the README while three of fifteen records had +status lines behind the code in the same PR. Before the next big line, align "said" with +"done". The half-built pieces still open: -一个 Rust 二进制加一个 Postgres：摄入（十种格式、URL / RSS / GitHub / Jira 同步、按源令牌推送）→ -混合检索 → 抽取（本体驱动、预算内全铺、超预算按块检索加祖先补齐、写入时按签名掰正方向）→ -三段消解（等值召回、画像相似、包含关系）加攒批裁决 → 双时态账本（证据、supersedes、合并可回滚）→ -本体从语料长出来（计数采纳、可撤销、默认开）→ 推理机 R0 全建、R1 带开关 → 语义层（概念映射另存、口径有历史）→ -问数（Ontology2SQL，源授权到工作区）→ 对话是 agentic 的，七个工具，跨轮回放自己做过什么 → MCP 只读五工具、个人令牌。 -告警五种，审计台账，每库角色矩阵。约 4.7 万行 Rust、2.2 万行前端、18 个连库测试、8 份可重跑语料。 - -**这些都是真的。** 下面这些也是真的： - -| 半成品 | 出处 | 症状 | +| Half-built | Source | Symptom | |---|---|---| -| `pending_facts` 只有表没有运行时；`remember` 整个停用 | 0015 | README 里有 Chat 记忆，用户用不到 | -| MCP 只有 API，没有令牌界面 | 0014 | README 里有 MCP，用户配不了 | -| `utopia-mcp` / `utopia-connectors` / `utopia-graph` 三个 crate 是三行占位，其中一个宣称的工具名从未实现 | 0014 | 读源码的人被误导 | -| 证明树只展开一层；增量维护没做；派生 vs 断言矛盾不记信号 | 0002 | README 说 derivation path expands back to the sentence，只对了一层 | -| `disjointWith` 落库了，消解侧仍用硬编码 `CONFUSABLE_TYPE_KEYS` | 0009 / 0001 P5 | 本体声明了不相交，合并候选照样跨过去 | -| 合并路径没有签名复核〔已做，#196；采纳那条路也没有，#190，一并做了〕 | 0012 待做 | 写入时掰正的方向，合并一次就能再反回去 | -| 类型消解只能手动跑 | 0001 P3a | 大本体下新实体的细化靠人记得点一下 | -| 中文库拿到纯英文本体；`zh.ts` 没追平所以界面默认英文 | 0008 / 0004 | 中文用户两头都是英文 | -| `merge_key` 不折 `_by`；叙述动词进本体 | 0007 | 同一关系两个方向并存 | -| 预算 24,000 与每块 40 / 30 / 30 标着「待测」；答案键仍是人填的；混装准确率没测 | 0006 / 0008 | 所有「更好」都说不清是改动还是方差 | -| 语义层映射不留证据；多源同概念全给模型挑 | 0011 | 口径是黑箱 | -| 映射探查硬依赖 `metric` / `dimension` 两个 key，空库静默零条 | 0009 未决 | 不装包就没有问数语义层 | -| 关掉自动扩本体后没有「新说法」提醒 | 0003 | 索引铺好了，提醒没做 | -| `document.no_text_layer` 未接，OCR 端点不存在 | 0005 | 扫描件全绿 | -| 死代码与陈旧注释：`graph.rs` 的 `confirmed_mappings()`、`confirm_fact` 里的 `mapped_to` 连接、`bootstrap_ontology.rs` 开头的注释、`reasoning.rs` 里写错的迁移号、测试抬头写错的迁移号 | 0010 / 0012 / 0002 / 0014 | 下一个读的人会信 | - -## 判据 - -以后再排期，用这几条判，不重新辩论： - -1. **说的和做的要一致。** README 的一行承诺、决策记录的一个状态行、代码里的一句注释——三者不一致时，先改到一致，再谈新功能。[0015](0015-recording-a-sentence-is-not-asserting-a-fact.md) 讲的是助手对用户，这条是我们对读者。 -2. **写图的东西先于读图的东西。** 一个会悄悄改图的半成品比没有更坏；一个只读的半成品只是少一个功能。所以 `pending_facts` 排在证明树前面。 -3. **能量的先做。** `scripts/bench/` 在，语料在，缺的是外部答案键。没有它，本体与抽取侧的任何调参都是把过拟合调得更紧（[0006](0006-ontology-scale-and-the-prompt.md) 未决）。 -4. **模拟与执行门等地基。** README 路线图的头两条——决策推演、执行校验——本质是「**不落账的提议叠加在账本之上**」。0015 的 `pending_facts` 就是这个形状的第一块砖：一条提议、原句在上三元组在下、人点头才进账。先把这块砖砌稳，模拟引擎才有可以叠的东西。 - -## 四条线，按这个顺序 - -### A · 收口——两到三周，不加新概念 - -**A1 · 把 0015 接上。**〔已做，形态与下文有两处不同，见 0015 末尾的修订：助手说不出 N，确认卡在对话里随抽取完成长出来。〕 抽取遇到记忆文档（`memory::is_memory_document`，已写好零调用）时写 `pending_facts` 而不是 `facts`； -Review 加 `pending` 一档，界面**原句在上、三元组在下**；确认 → `insert_fact`，**置信度不动**（0011 的教训：别拿浮点数表达人的态度，人的态度在审计台账里）； -拒绝 → 写 `rejected_facts`，下一轮重抽先查它；`remember` 的回话改成「已记录，抽出 N 条待你确认」；然后把 `REMEMBER_ENABLED` 改回 true。 -**验收**：那条实测（「Acme 把总部搬到深圳」）跑一遍，图上不出现空谓词活边，Review 里出现一条带原句的待确认项。 -做完这一条，MCP 可以放 `remember`（scope = write，令牌以人的身份提议，人自己确认）——0014 未决的答案由此成立。 - -**A2 · 令牌有界面。**〔已做，`feat/tokens-have-a-page`〕Account 页：列表（前缀、名字、scope、库、最后使用、过期）、发放（明文只显示一次）、撤销；旁边给一段可复制的 MCP 客户端配置（URL + Authorization 头）。 -三个占位 crate **删掉**（已做，与 A3 同一刀）：`utopia-mcp` 宣称的三个工具从未存在，服务端住在 `utopia-server/src/api/mcp.rs`；`utopia-graph` 没有对应物；`utopia-connectors` 的意图是对的，但两个连接器看不出边界该划在哪，[0013](0013-a-source-should-hand-over-its-history.md) 说等第三个来了再看抽象，crate 是同一个判断。占位是对读者的承诺，兑现不了就别留，需要时 `cargo new` 是几分钟的事。 - -**A3 · 清死代码与陈旧注释。**〔已做，分两刀：#188 那批清了五处死代码与三个占位 crate；`chore/comments-catch-up` 补了还在描述「兜底谓词」的五处注释和一处指向仓库外 DESIGN.md 的引用〕上表最后一行那五处，加上 `resolution.rs` 里 `CONFUSABLE_TYPE_KEYS` 上方停留在旧世界的注释。一个 PR。 - -**A4 · README 与代码对表。** Reasoning 那一行的 "derivation path expands all the way back to the original sentence" 在 B1 做完之前改成只承诺一层； -Chat memory 与 MCP 两处在 A1 / A2 落地前标 in development。中英两份一起改。 - -### B · 推理机补完——A 之后，与 C 并行 - -**B1 · R2 证明树。** 递归展开到叶子 chunk 的 API + 实体面板里可展开的树。数据结构（`fact_derivations`）已够。 -**B2 · 派生 vs 断言矛盾要有信号。** `axiom_violations` 加一种 kind（`derived_contradiction`），进 Review 同一档——0002 写了没做的那一行。 -**B3 · `disjointWith` 进消解，合并路径复核签名。** `classify_type_drift` 改从 `entity_type_disjoint` 读（没声明就退回今天的行为，不硬编码）； -`merge_entities` 搬事实前跑一遍与写入时相同的 domain / range 检查，违反的进 `axiom_violations` 而不是静默搬过去。这是 0009 与 0012 各自待做的同一件事。 -〔**签名那一半已提前做了**（#190 / #196 逼出来的，A 线里插队）：`ontology::judge_direction` 一处判断，抽取与采纳共用；合并后对搬动过的事实报 `signature` 违规；R0 多一类。剩下 `disjointWith` 进消解那一半仍在这里。〕 -**B4 · R3 增量维护——后置。** 只在测量台上全量重推超过一个明确阈值（建议：`ai-timeline-ends` 语料超过 10 秒）时才做。今天每次全量重推活得下去。 - -### C · 本体与抽取质量——先造尺子，再调参 - -**C1 · 外部答案键。** 0006 未决的那条：维基条目配 Wikidata `P31` 作类型答案，生成脚本入库。有了它，才能跑 0008 的混装对照（`run.mjs --packs` 已支持）与 0006 的预算曲线。 -**C2 · 类型消解自动触发。** 抽取结束入队一次类型消解（与 `bootstrap_ontology` 并列），只自动落「在原类子树里」的那一档，跨轴仍进人工。**排在 C1 后面**：没有尺子，自动跑等于把噪声规模化（0001 P3 的原话）。 -**C3 · 中文本体。** 三选一，倾向第一个：装包时若 `ontology_lang = zh`，由 LLM 批量生成中文 **description**（模型读的那一行，0004 说它该跟语料语言），存进库、原文 IRI 不动、`label` 不动（label 是 key 的展示，key 永不翻译）； -另两个是「等社区出中文包」与「什么都不做只提示」。同时把 `zh.ts` 追平，把 `navigator.language` 加回 `detect()`。 -**C4 · 0007 的两条尾巴。** `merge_key` 折 `_by` 并把整组标成需对调；叙述动词交给那次同义归并的 LLM 调用一并审（「这些里哪些是文章的口吻」），可撤销。 -**C5 · 0003 的提醒。** 关掉开关后 Ontology 页显示「自上次以来有 N 个新说法」，数据从 `ontology_misses` 按 `dismissed_at IS NULL` 算，索引已在。 - -### D · 语义层与问数——C1 之后 - -**D1 · 映射留证据。** `concept_mappings` 加 `evidence`（探查时读了哪几张表的 schema 文档，chunk id 数组），数据映射页展示。 -**D2 · `metric` / `dimension` 出包。** 0009 未决的答案：做一个「Utopia 语义层」包，带 IRI、可选、可替换，与其它包同一条装法；`mappings.rs` 改按 IRI 找类型，不按 key。 -**D3 · 多源同概念的选法。** 先做最简单的规则：问数时若挂载的源只有一个，只给那个源的映射；多个源全给。规则写在 `mappings::confirmed` 一处。 -**D4 · MySQL 线协议。** 白捡 TiDB / OceanBase / Doris / StarRocks，`sqlparser` 换方言。README 路线图里的那条。 - -### E · 企业交付——v0.2 的门槛，可穿插 - -- **凭据静态加密**（SECURITY.md 承诺「1.0 之前」；LLM key 与连接串同待遇）——小，可提前。 -- **OIDC SSO**；**备份与恢复命令**；**十万文档基准**（先跑再说，数字比设计重要）。 -- **`document.no_text_layer` + OCR**：告警那条接上要先有 OCR 端点（docling-serve sidecar），两件一起做，提示语才完整（0005 留到第二批的那条）。 -- **`instant` 精度**：0013 说了触发条件（按当地营业日统计、或事件聚在午夜附近的源），没触发前不做。 - -### 后置：模拟引擎与执行门 - -README 路线图的头两条**不在这份排期里**，理由见判据 4。它们需要三样地基：`pending_facts` 的提议语义（A1）、派生的可解释性（B1）、派生与断言的矛盾信号（B2）。 -A 与 B 做完之后单独写一篇决策记录，那时再排。 - -## 顺序与理由，一句话 - -**A → B ∥ C → D → E。** A 是对齐「说的和做的」，不做新东西；B 与 C 互不依赖，B 动写图的路径、C 动尺子；D 等 C1 的尺子；E 可穿插，加密先。 - -## 一条新约定 - -**状态行由实现它的 PR 负责更新。** 0011、0014、0015 三篇的状态行都落后于同一个 PR 里的代码——写决策记录的人和写代码的人是同一个人，在同一个提交里，仍然漏了。 -所以不靠记性：PR 模板里加一行「这个改动实现或推翻了哪篇决策记录，状态行改了没有」。已写进 [README](README.md) 的约定。 - -## 开放问题 - -- **A1 的闸只拦记忆，还是拦所有单条交互式写入？** 0015 未决。今天只有 `remember` 一条路，先拦它；将来「在图上手工加一条边」的界面出现时，按同一张表走。 -- **C3 生成的中文描述算不算「原文保真」的例外？** 它不是导入的原文，是我们生成的投影。倾向记成投影（可重跑、可丢弃），原文 IRI 与英文描述保留在包里。 -- **B4 的阈值**：10 秒是拍的。 -- **对话内 agent 生成的界面走哪个协议。** A1 的确认卡载荷是声明式的（chunk + 三元组列表），渲染器是我们的 React。 - 讨论过 A2UI（2026-09-02）：它解决的是「你不控制的客户端渲染你 agent 的 UI」，而 Chat 页两头都是我们的； - 卖点成立的地方在 MCP 那一侧，那里对应的是 MCP 自己的 UI 扩展，不是 A2UI。**判据是谁的客户端在看**， - 等 MCP 放开写工具、有外部客户端要渲染这张卡时再定。 -- **要不要把抽取层拆出 `utopia-server`。** 今天约 6,500 行领域代码（extraction、type_resolution、bootstrap_ontology、predicate_match、adjudication、owl_import、连接器）住在 HTTP crate 里，依赖方向仍是干净的一条线。讨论过（2026-09-02）决定**不拆**：唯一实在的收益是脱离 server 测纯函数，而它们的单测今天已经不起 server；搬动 1,500 行的 `extraction.rs` 会让紧接着的 A1 diff 和 blame 变难看。判据留下：**等它造成实际损失再拆，损失的定义是「有东西没法脱离 server 测试」**。连接器那一块的边界等第三个连接器（飞书）来了再定。 -- **十万文档基准跑出来之后**，单进程 worker 与内嵌 Tantivy 的天花板在哪，决定了要不要在 v0.2 谈横向扩展。现在不谈。 +| `disjointWith` is stored; resolution still uses the hard-coded `CONFUSABLE_TYPE_KEYS` | 0009 / 0001 P5 | merge candidates cross declared disjointness | +| type resolution runs only by hand | 0001 P3a | refining new entities depends on someone clicking | +| a Chinese base gets an English ontology; `zh.ts` lags, so the interface defaults to English | 0008 / 0004 | Chinese users get English at both ends | +| `merge_key` does not fold `_by`; narrative verbs enter the ontology | 0007 | one relation in two directions | +| the 24,000 budget and 40 / 30 / 30 per-chunk counts are untested; the answer key is hand-filled; mixed packs unmeasured | 0006 / 0008 | no "better" can be told from variance | +| mappings carry no evidence; with several sources every same-name concept goes to the model | 0011 | the calculation is a black box | +| the mapping probe depends on the `metric` / `dimension` keys | 0009 | a blank base found nothing until the builtin classes of #231; the pack is D2 | +| no "new phrasings" reminder once auto-extension is off | 0003 | the index is there, the reminder is not | +| `document.no_text_layer` unwired, no OCR endpoint | 0005 | scanned documents show all green | + +## Criteria + +1. **Said and done agree.** README line, status line, code comment: when they disagree, fix + that before adding features. 0015 is the assistant to the user; this is us to the reader. +2. **What writes the graph comes before what reads it.** A half-built thing that silently + changes the graph is worse than nothing; a half-built read is a missing feature. So + `pending_facts` came before the proof tree. +3. **Measurable first.** `scripts/bench/` and the corpora exist; the external answer key does + not, and without it tuning tightens overfitting ([0006](0006-ontology-scale-and-the-prompt.md)). +4. **Simulation and execution gates wait for the ground.** The README roadmap's first two + items are proposals overlaid on the ledger; `pending_facts` (A1), the proof chain (B1) and + the contradiction signal (B2) are what they stack on. + +## The lines: A → B ∥ C → D → E + +**A · Close the seams**, no new concepts. A1 wire 0015 (done). A2 a tokens page and delete +the three placeholder crates (done; a placeholder is a promise to the reader, and `cargo new` +takes minutes). A3 dead code and stale comments (done, #188 and `chore/comments-catch-up`). +A4 README against code, both languages: promise only what has landed. + +**B · Finish the reasoning engine**, after A, parallel with C. B1 the R2 proof chain (done, +#227; a chain, not a tree — see [0002](0002-reasoning-engine.md)). B2 the derived-vs-asserted +contradiction signal, `axiom_violations` kind `derived_contradiction` (done: B2a engine and +queue #238, B2b visibility on graph and panel #243 — 0017). B3 resolution reads `entity_type_disjoint` +ahead of the hard-coded list (done: a declared disjointness, inherited, wins over kinship and the +list; nothing declared → today's behavior; class kinship from the hierarchy landed first, #226). B4 R3 incremental +maintenance, deferred until a full re-derivation of `ai-timeline-ends` exceeds a threshold +(proposal: 10 seconds). + +**C · Ontology and extraction quality**: build the ruler, then tune. C1 an external answer +key, Wikidata `P31` as the type answer for Wikipedia articles, generated by script; only then +can the 0008 mixed-pack comparison (`run.mjs --packs`) and the 0006 budget curve run. C2 type +resolution queued after extraction, auto-applying only the "within the original subtree" +tier; after C1, or it scales noise. C3 with `ontology_lang = zh`, generate Chinese +`description` rows by LLM (the line the model reads follows the corpus, +[0004](0004-language-and-localization.md)), IRIs and `label` untouched; catch `zh.ts` up and +put `navigator.language` back into `detect()`. C4 fold `_by` in `merge_key` and let the +synonym-merging LLM call review narrative verbs, reversibly. C5 "N new phrasings since last +time" from `ontology_misses` where `dismissed_at IS NULL`. + +**D · Semantic layer and data questions**, after C1. D1 `concept_mappings` gains `evidence` +(the schema-doc chunk ids read during probing). D2 a "Utopia semantic layer" pack with IRIs, +optional and replaceable; `mappings.rs` finds types by IRI. D3 one mounted source → only its +mappings, several → all, in `mappings::confirmed`. D4 the MySQL wire protocol (TiDB / +OceanBase / Doris / StarRocks, `sqlparser` dialect switch); the lakehouse engines came first. + +**E · Enterprise delivery**, the v0.2 bar, can interleave. Credentials encrypted at rest +(SECURITY.md promises it before 1.0) — small, do it early. OIDC SSO; backup and restore; a +hundred-thousand-document benchmark (numbers beat design). `document.no_text_layer` together +with an OCR endpoint (docling-serve sidecar). `instant` precision only when the trigger in +[0013](0013-a-source-should-hand-over-its-history.md) fires. + +**Deferred: simulation engine and execution gate** (criterion 4); a separate record once A +and B are done. + +## A new convention + +The PR that implements a record updates its status line. 0011, 0014 and 0015 fell behind the +code in the same PR by the same author, so the PR template asks which record the change +implements or overturns and whether the status line moved. Written into the [README](README.md). + +## Open questions + +- Does the A1 gate hold only memories or every single-item interactive write? Only `remember` + today; a hand-added edge should take the same table. +- Are C3's generated descriptions an exception to "keep the original text"? Leaning towards a + projection — rerunnable, disposable — with the IRI and English description kept. +- B4's 10-second threshold is a guess. +- Which protocol for agent-generated interfaces in chat. A2UI (discussed 2026-09-02) solves + "a client you do not control renders your agent's UI"; on the Chat page both ends are ours, + on the MCP side the counterpart is MCP's own UI extension. Decide when an external client + needs the A1 card. +- Whether to split extraction (about 6,500 lines) out of `utopia-server`. Decided not to + (2026-09-02): the only real gain is testing pure functions without the server, which their + unit tests already do. Split when something cannot be tested without the server; the + connector boundary waits for Feishu. +- After the benchmark: where the single-process worker and embedded Tantivy hit their ceiling + decides whether v0.2 discusses horizontal scaling. diff --git a/docs/decisions/0017-a-contradiction-points-upstream.md b/docs/decisions/0017-a-contradiction-points-upstream.md new file mode 100644 index 00000000..808c2ecd --- /dev/null +++ b/docs/decisions/0017-a-contradiction-points-upstream.md @@ -0,0 +1,205 @@ +# 0017 · A contradiction points at an error upstream + +- **Status**: implemented · B2a (#238): engine and queue — `derive::contradictions`, migration 0020, the Review card with clues and repairs · B2b: contested edges in the alert colour and ghost edges for blocked derivations on the graph, the disputed chip on panel rows, the "did not land" section of the Derived tab with its proof chain · B2 of 0016, wider than the one line written there: contradictions become visible everywhere, not only as a new kind in the queue +- **Written**: 2026-09-03 (conventions in [README](README.md)) +- **Related**: the two unbuilt rows of the "derived vs asserted" table in [0002](0002-reasoning-engine.md) §2; [0016](0016-close-the-open-seams-before-cutting-new-ones.md) B2; the proof chain (B1, #227) supplies the "premises expand to the sentence" half of the card below + +> This record changes one phrase in 0002. There, a contradiction "goes to Review for a ruling". +> In discussion (2026-09-03) it became clear what a person actually does when two edges +> disagree: follow the signal to **where something went wrong upstream** — a fact went +> stale, extraction misread, resolution merged the wrong pair, the ontology is too strict. +> So this queue is built as an audit, and every button is a repair. + +## The problem + +R1 derivation only yields to "this exact triple is already asserted". It does not consult +axioms: `part_of` transitivity derives `A part_of C` while the ledger asserts `C part_of A` +(asymmetric); `ceo_of` is functional and `Acme ceo_of Zhang San` is asserted, yet the +engine still lands `Acme ceo_of Li Si`. R0 scans `facts` only, derivations live in their +own table, so this class of contradiction is **invisible on both sides**: absent from +Review, absent from the graph. + +It is not the only invisible class. Temporal conflicts and axiom violations are known to +the Review page alone; on the graph and in the entity panel a contested edge looks exactly +like any other. + +## Criteria + +1. **A contradiction points at an error upstream.** The error is in one of four places: + stale knowledge (most common), a misread extraction, a wrong merge, an over-strict + ontology. The interface's job is to lay out the clues and offer the repairs. +2. **Write less to the graph rather than write wrong** (0002). A derivation that hits a + contradiction does not land; when two derivations collide, neither lands. +3. **Anything reported item by item needs an upper bound.** Only "one assertion against one + concrete edge" — a volume that can be predicted — goes into the queue individually, and + with a per-predicate cap; whatever is produced in batches is aggregated by its cause. +4. **A disputed fact is visible where it sits.** The assertion stays live, but someone + passing it on the graph or in the panel should see that it is questioned, and reach the + place to fix it in one step. + +## Decisions + +### 1. Detection: two tiers, pure logic in `utopia-reason` + +`derive::contradictions(derived, edges, axioms)` checks every derivation against the axioms +of its predicate, once against assertions and once against other derivations: + +| Axiom | What counts as a contradiction | +|---|---| +| functional | same subject and predicate, different object, **validity intervals overlap** (Mira leaving and Devin taking over is a succession; the half-open interval semantics of `validity` are reused) | +| inverse_functional | the dual | +| asymmetric | the reverse edge exists with overlapping validity | +| irreflexive | the derivation is a self-loop | + +**Derived vs asserted**: per item. The derivation does not land; one `axiom_violations` +row of kind `derived_contradiction`. Capped per predicate (same shape as R1's +`MAX_DERIVED_PER_PREDICATE`; 50 suggested); the overflow stays out of the queue and is +counted in the report. + +**Derived vs derived**: aggregated. The cause is a rule set that contradicts itself — +`ceo_of ⊑ works_at` together with `works_at` functional necessarily produces contradictions +in batches. Queuing every pair would flood Review, and a person facing a hundred identical +"two derivations disagree" cards cannot decide anything from them. So one row per **pair of +rules** goes into `ontology_defects`, kind `rules_disagree`, recording both rules, the +count, and two or three examples. It is the dynamic form of the existing static defects +`transitive_and_functional` and `symmetric_and_asymmetric`: invisible in the declarations, +surfacing once data arrives. Neither derivation lands. + +`run()` (R0) and `materialize` (R1) share the one function: R0 already holds the edges and +axioms and adds a `derive` + `contradictions` pass to report; R1 uses the same result to +decide what stays unlanded. **Both must compute it** — otherwise R0's "clear open rows not +recomputed this round" would sweep away what R1 wrote. + +### 2. The Review card: lay out the clues, lay out the repairs + +**The derived-vs-asserted card**, three parts: + +1. **The contradiction itself**: the derived edge (rule, premise chain expandable to the + sentence — from B1) beside the asserted edge (evidence quote, interval, confidence). + Most errors are visible at a glance. +2. **A diagnostic hint**, one line when a clue can be computed, none when it cannot: + - the assertion has no end date and the derivation starts after it → "looks stale: did + Zhang San's tenure end in 2024-07?" + - an entity on either side has same-name neighbours (the existing `same_name` machinery) + → "looks like a wrong merge: are these two Acmes one company?" + - the assertion's confidence is below 0.75 → "the extraction was unsure to begin with" + - none of the above → "read both sentences" +3. **Actions that are repairs**, all on existing endpoints; once a repair is made the + violation clears on the next recomputation, with no separate resolve step: + +| Action | What it repairs | Where it lands | +|---|---|---| +| Close the assertion at a date | stale knowledge | `POST /facts/{id}/close` (same as the temporal-conflict queue, with a date input) | +| Retract the assertion | misread extraction / wrong merge | `reject_fact`, append-only | +| Go to possible duplicates | wrong merge | the Duplicates queue in Review | +| The ontology is too strict | wrong definition | the relation on the Ontology page | +| Both hold; let the derivation through | the world is like that | `resolution = accepted`; the next `materialize` lets that pair land | + +**The derived-vs-derived card** (in the ontology-defects queue): "these two declarations +together produced N contradicting pairs; examples below — usually a sub-property attached +in the wrong place, or a functional declaration that is too strict". Two actions: go to the +Ontology page and change a declaration; or "accept" (contradicting derivations from this +pair of rules never land and are not reported again). Both sides are derived and the +assertions beneath may each be right, so there is no "the data is wrong" here; "both hold" +would mean giving up the meaning of functional, which belongs on the Ontology page, so +there is no such button either. + +### 3. Disputed facts are visible where they sit + +"Disputed" becomes one status with three sources: open temporal conflicts +(`fact_conflicts`), open axiom violations (`axiom_violations`, including the new kind), and +blocked derivations (the `detail` of `derived_contradiction` rows). B2 makes it a first-class +state of the browse pages and wires up the first two sources along the way — they are +equally invisible today. + +**Rows in the entity panel**: `EntityFact` already carries `stale` and `corrected`; a third +flag `contested: Option` is computed with one EXISTS. The chip +reads "disputed"; hover gives one sentence ("derived works_at Li Si contradicts this"); +click goes to the matching Review item. The row is not dimmed: the assertion is still live. + +**The Derived tab of the panel**: a new section, "derivations that did not land", read from +`axiom_violations.detail`, each row naming what blocked it and expanding to its premise +chain. Here a person sees "the engine could have drawn this edge, and what stopped it". + +**Edges on the graph**: + +- A contested assertion switches to the **alert color**, edge and all, with no reliance on + a node ring. The request was explicit: a ring sits on the node while the edge stays gray, + and peripheral vision cannot tell them apart. +- A blocked derivation is drawn as a **ghost edge**: the same hue mixed toward `EDGE_DIM` + (under premultiplied blending alpha cannot darken an edge; only the RGB can be mixed — see + the note at `EDGE_DIM`), thinner. It follows the Derived toggle; clicking it opens the + entity panel at that row. sigma's default edge program does not draw dashes, and no custom + program is introduced for this. +- The hover label chip gets a "⚠" prefix, and the tooltip states the dispute. + +**Color**: one new token, `--u-contest`, set to **#ff6a3d** (hot coral orange). It has to +stand apart from three colors already in use: derived edges are gold +(`rgb(231,197,124)`), warning chips are amber (`--u-warn` #f2b66d, too close to gold to +double as an edge color), and danger is pink (`--u-danger` #ff9daf, reserved for +destructive actions). Coral orange is bright enough on the dark ground, its hue is far from +all three, and it is no common type color (the type palette leans blue, green and violet). +Edge color `rgba(255,106,61,0.55)`; ghost edge `lerp(#ff6a3d, EDGE_DIM, 0.55)`. This is +the only new color; chips on the Ontology and Review pages use the same token. + +### 4. Data + +Migration `0020` (0019 was taken by #199 / #233): + +- `axiom_violations.kind` CHECK gains `derived_contradiction` +- `axiom_violations` gains `detail JSONB NOT NULL DEFAULT '{}'`: the derived triple (three + ids and three names), the rule kind, the premise ids — without it the interface cannot + say what was derived; `path` holds the premises for the proof chain but cannot draw the + ghost edge +- `axiom_violations.resolution` CHECK gains `fact_closed` (closing an assertion is a repair + that leaves its own trace, kept apart from `fact_retracted`) +- `ontology_defects.kind` CHECK gains `rules_disagree`, plus `detail JSONB` (both rules, the + count, the examples) + +`left_fact` / `right_fact` stay non-null and keep pointing at `facts`: for derived vs +asserted, `left` is the contradicted assertion and `right` the derivation's last premise; +`rules_disagree` does not live in this table. + +### 5. Interfaces + +- `GET /kbs/{id}/graph`: edges gain `contested: bool`; a new class of edge with + `blocked: true` (the ghosts) +- `GET /kbs/{id}/entities/{id}`: `facts[].contested`; `blocked: [...]` next to `derived` +- Review `violations` queue: rows gain `detail` and `hint` (the diagnostic as a code; wording + belongs to the frontend) +- `POST /kbs/{id}/review/violations/{id}`: `resolution` gains `fact_closed` (with + `close_at`); the server calls `close_fact` and then marks the row resolved + +### 6. Tests + +- `utopia-reason` unit tests: one per axiom, non-overlapping intervals are no contradiction, + derived-vs-derived aggregates by rule pair, the cap is counted +- Store integration: a functional assertion stands and a derivation collides → the + derivation stays out and one violation carries `detail`; retract the assertion and rerun → + the derivation lands and the violation clears; `accepted` → both coexist; `fact_closed` + runs the close path; R0 and R1 agree +- Browser: an edge turns coral, ghost edges follow the toggle, the panel chip jumps to the + Review item, the Review card shows three parts and five actions + +## Two cuts + +**B2a · engine and queue**: §1, §2, §4, and the Review parts of §5, with the integration +tests. Two days. +**B2b · visibility**: §3 plus the graph and panel interfaces, wiring existing temporal +conflicts and axiom violations along the way. A day and a half. + +B1 (#227) merges first after a rebase; B2a branches from it. + +## Open questions + +- **The per-predicate cap of 50 is a guess**, awaiting a bench number like R1's twenty + thousand. +- **Granularity of `accepted`**: per pair (one derivation against one assertion). An + assertion hit by several derivations needs several clicks; aggregating the exemption per + assertion would generalize "this axiom does not apply to this assertion" too far. Per + pair first, measure later. +- **Could ghost edges be too many?** Their number is bounded by the (capped) violation + count, so it should stay manageable; failing that, draw them only when a related node is + selected. +- **Does the disputed status need its own SSE event?** The `review` event is already sent; + graph and panel can refetch on it. No new event. diff --git a/docs/decisions/0018-the-lakehouse-is-one-protocol-away.md b/docs/decisions/0018-the-lakehouse-is-one-protocol-away.md new file mode 100644 index 00000000..91ccdd46 --- /dev/null +++ b/docs/decisions/0018-the-lakehouse-is-one-protocol-away.md @@ -0,0 +1,58 @@ +# 0018 · The lakehouse is one protocol away + +- **Status**: implemented · `trino` / `databricks` / `snowflake` engines in `query_engine/` (migration `0021` widens the `engine` CHECK) · every engine is covered by protocol replays only (wiremock); **none has run against a real cluster** (#240, #241, #242) · MaxCompute is not done, see the last section +- **Written**: 2026-09-03 (conventions in the [README](README.md)) +- **Related**: [0011](0011-a-mapping-is-not-a-fact.md) placed data sources at the deployment level and mounts at the base level; this record leaves that layer alone. [0016](0016-close-the-open-seams-before-cutting-new-ones.md) D4 put the MySQL wire protocol ahead of the lakehouse; the first section explains why the order flipped + +> The roadmap line reads "Iceberg / Delta Lake, Databricks, Snowflake and MaxCompute": four names, three kinds of thing. The first two are table formats, the next two are services, the last is a service on another cloud. Treating them as four engines would be writing code per product name. This record pins down what an engine is first, then decides which ones to build. + +## Engines follow protocols + +The header of `query_engine` (written for 0011) already gave the direction: the Postgres wire protocol, then the MySQL wire family, then the HTTP family. This step lands the HTTP family and skips MySQL. 0016 D4's "TiDB / OceanBase / Doris / StarRocks for free" is a fine list, but the lakehouse is what is wanted now, and on the protocol axis it is closer than it looks: + +| Wanted | What it is | Which protocol that is for us | +|---|---|---| +| Iceberg, Delta Lake, Hive, Hudi | table formats plus a catalog, with no query endpoint of their own | one **Trino** catalog each; `POST /v1/statement` with `nextUri` paging | +| Databricks | a Delta lakehouse behind a SQL warehouse | its own **SQL Statement Execution API** (`/api/2.0/sql/statements`) | +| Snowflake | a cloud warehouse that also reads Iceberg | its own **SQL API v2** (`/api/v2/statements`) | +| MaxCompute | Alibaba Cloud's warehouse | signed REST, asynchronous instances, results through Tunnel | + +The first three rows are all "JSON in, JSON out, Bearer or Basic auth". `reqwest` is already a dependency; each engine is about two hundred lines. The binary still carries no native database driver, which is the promise in the README's first sentence. It is also why the answer to Iceberg is Trino rather than an Iceberg reader: reading Iceberg directly pulls in Arrow, Parquet, object-storage SDKs and a query planner, and that is a different product. + +## The connection string is the only input + +The data-source page has a name and a connection string. Three new engines add no dropdown: **the scheme picks the engine** (`engine_from_conn`), and each engine parses the rest (`conn.rs`). The shape follows `postgres://user:pass@host/db`: credentials in the userinfo, the path is "catalog / database / schema", engine-specific switches go in the query string: + +``` +trino://alice[:password]@host[:8080]/catalog[/schema][?ssl=true] +databricks://:TOKEN@workspace-host/sql/1.0/warehouses/ID[?catalog=main&schema=default] +snowflake://:TOKEN@account.snowflakecomputing.com/DB[/SCHEMA][?warehouse=WH&role=R&token_type=pat|oauth] +``` + +The Databricks path is the httpPath shown in the console, so it can be pasted as is. All three tokens sit in the password position; `TOKEN@` with the colon missing is the most common slip, so the username position is accepted too. The shape is validated at registration, and the error carries the expected form. `ssl=false` exists for local proxies and stand-ins; the three services themselves only speak https. + +Left out on purpose: Snowflake key-pair JWT (local RSA signing, a dependency for a second login method, wait for someone to need it) and Trino Kerberos / OAuth2 (same reasoning). Passwords and tokens are the whole surface. + +## The HTTP family has three of the four gates + +0011 set up defense in depth: parse and admit only SELECT, wrap a LIMIT, a read-only session with a timeout, JSON Lines out. The HTTP family **has no session**, so the third gate is a timeout alone (Trino's `query_max_execution_time` session property, Databricks' `wait_timeout`, Snowflake's `timeout`), and read-only rests entirely on the first gate. The first gate therefore parses with each engine's dialect: `DatabricksDialect`, `SnowflakeDialect`, and `GenericDialect` for Trino (sqlparser has no Trino dialect; Generic is a superset). One test runs the same three checks under all four dialects: SELECT passes, DELETE fails, two statements fail. + +The fourth gate is assembled here for the HTTP family. Databricks' `JSON_ARRAY` and Snowflake's `data` return every value as a string; `coerce` restores numbers and booleans from the manifest / rowType column types, otherwise a model handed `"42"` stops doing arithmetic. Key order is assembled by hand instead of through `serde_json::Map`, which sorts keys unless `preserve_order` is on, and column order is the order the query wrote. + +## Loopback goes direct + +reqwest's system-proxy detection on Windows reads the registry and did not honor a `127.*` bypass, so a stand-in on the loopback address went through the proxy and came back as 502. The engine client now carries an explicit policy: loopback and `NO_PROXY` hosts go direct, everything else follows `HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY`. A server process reads its environment; that matches how docker-compose configures it. The other connectors keep reqwest's default. + +## Replays are the only tests so far + +No stand-in for any of the three runs on this machine: Docker Hub cannot be reached here (see memory), and Databricks and Snowflake are cloud-only anyway. The tests replay each vendor's documented protocol with wiremock: Trino's two `nextUri` pages, Databricks' PENDING → SUCCEEDED polling, Snowflake's 202 → 200, and each error body. **They prove our reading of the protocol, and only that.** Until one real cluster has answered, the README keeps these three marked as awaiting a real run, handled the way #214 / #215 handle GCS and Notion: an issue per engine labeled help wanted. + +## MaxCompute waits + +It is the one name of the four that is not "JSON in, JSON out": requests are signed with an AccessKey (HMAC-SHA1 over canonicalized headers), SQL runs as an asynchronous instance, and results come either through Tunnel (another protocol) or `GetInstanceResult` as CSV capped at ten thousand rows. Together that is a connector's worth of work, and this machine has no account that could sign a request, so the result could only be "probably like this". It stays on the roadmap until someone with an account arrives, or until its MySQL-compatible entry (MCQA) can ride the MySQL wire protocol of 0016 D4. + +## Open questions + +- **How much schema to fetch.** All three expose `information_schema.columns`, and a lakehouse catalog can hold thousands of tables; `sync_schema_doc` caps at 200. With a schema in the connection string only that schema is read, otherwise the whole catalog. Whether that is enough waits for a real cluster. +- **The type-restoration table** in `coerce` is hand-written from the three vendors' docs. Snowflake's `fixed` with a scale returns `"42.10"`, which becomes 42.1 and loses the trailing zero; harmless for a model, possibly not for an "exact definition". Revisit when the semantic layer keeps evidence (0016 D1) and decide whether to keep the raw string alongside. +- **Trino's `ssl` inference**: a password, `ssl=true`, or port 443 / 8443 means https, anything else is plaintext. That is trino-python's rule, and someone who gets it wrong sees a TLS error instead of a hint. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index ce399d59..a4c39010 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -1,57 +1,52 @@ -# 决策记录 +# Decision records -代码记录了**做成什么样**，git 记录了**什么时候改的**。两者都不记录**为什么这样而不是那样**，以及**哪些路走过发现是死路**。这个目录记那个。 +Code records what was built and git records when it changed. Neither records why it was built this way and which roads turned out to be dead ends. This directory does. -判断标准：如果半年后有人（包括我们自己）看着某段代码问"当初为什么不直接……"，而答案不在代码里，那就该有一篇。 +The test for writing one: if someone (including us) looks at a piece of code in six months and asks "why not simply…", and the answer is not in the code, there should be a record. -## 约定 +## Conventions -**文件名** `NNNN-短横线英文标题.md`，四位序号，按创建顺序递增，不留空号。序号只是稳定的引用锚点，不表示优先级。 +**File names** are `NNNN-short-english-title.md`, four digits, increasing in creation order, no gaps. The number is a stable anchor for references and says nothing about priority. -**目录扁平**。在出现第二种性质不同的文档之前，不建子目录。 +**The directory is flat.** No subdirectories until a second kind of document appears. -**修订就地留痕，不静默改写**。结论变了——尤其是因为查证推翻了原判断——**在原处保留「修订记录」块**写清：原本是什么、为什么错、依据什么改的。不要把错误的那版删干净。 +**Revisions stay in place.** When a conclusion changes, especially because checking the code overturned it, keep a dated revision note where the original claim stood: what it said, why it was wrong, what changed it. The ledger this product keeps never updates a fact in place; a correction inserts a new row that supersedes the old one, because the change of mind is information. Records follow the same rule: knowing "we assumed a range would take effect immediately, checked, and found it never did" tells the next reader what to check first. -> 这条不是文档洁癖，是同一条产品原则。我们的账本从不原地 UPDATE 事实，纠正是插入新行 + `supersedes` 旧行，因为**认知变更本身是信息**。决策记录同理：知道"我们曾经以为 range 能立刻生效，查完发现一条都没有"，比只看到最终结论有用得多——它告诉你下次该先查什么。 +**When too much has changed**, write a new record and mark the old one `superseded by NNNN` at the top instead of rewriting it. -**结论变得太多时**，另起一篇并在旧篇顶部标 `已被 NNNN 取代`，而不是把旧篇改成新的。 +**The PR that implements a record updates its status line.** Three status lines once lagged behind the code in the same PR, written by the same person. So the PR description answers one question: which record does this change implement or overturn, and is its status line updated. (Added 2026-09-02, from [0016](0016-close-the-open-seams-before-cutting-new-ones.md).) -**状态行由实现它的 PR 负责更新。** 0011、0014、0015 三篇的状态行曾落后于同一个 PR 里的代码——写记录的人和写代码的人是同一个人、同一个提交，仍然漏了。所以 PR 描述里要答一句：这个改动实现或推翻了哪篇决策记录，状态行改了没有。（2026-09-02 加，出处见 [0016](0016-close-the-open-seams-before-cutting-new-ones.md)。） +**Line numbers drift and file names change.** Prefer function, table and constant names over `file.rs:123`. Migrations were consolidated from 53 files into one per domain (#130, #131); older references to migration files are by domain. -**行号会漂，文件名会换。** 正文里的 `file.rs:123` 是成文时的坐标，不保证仍然准确；迁移在 #130 / #131 从 53 份折成 10 份、一个域一份，所以 2026-08-31 之前写的迁移文件名都要按域重找。引用时优先写函数名、表名、常量名。 +**Language: English.** The first sixteen records were written in Chinese and condensed into English on 2026-09-03; the Chinese originals remain in git history. Code comments are still Chinese; UI, README and records are English. -**语言**：中文，与代码注释一致（UI 与 README 保持英文）。目前的读者是维护者，不是外部贡献者；将来需要时再译。 +## Index -## 索引 - -| | 文档 | 状态 | +| | Record | Status | |---|---|---| -| 0001 | [本体导入与治理路线](0001-ontology-import-and-governance.md) | 进行中 · P0–P2c 全建成；P3 按预算落地、P3a 只能手动跑；P3b 建成但形态不同；**P4b/P4c 待做**；P5 已由 0002 落地；判据 2 被 0012 推翻一半 | -| 0002 | [推理机](0002-reasoning-engine.md) | R0 建成（事实层五类含签名、本体自检八类）· R1 建成带开关默认关 · **R2 只一层、R3 未做** · 派生 vs 断言矛盾无信号 | -| 0003 | [本体从语料里长出来，人站在哪一环](0003-ontology-growth-loop.md) | 已建成且仍在跑 · 起点已被 0010 与种子退场改写 · 「拒绝有记忆」被 0007 推翻重做 · 新说法提醒待做 | -| 0004 | [语言：哪些字跟着看的人走，哪些跟着语料走](0004-language-and-localization.md) | 已建成 · L0–L3 全落地 · 界面刻意不猜浏览器语言 · 「中文内置本体」随播种退场作废 | -| 0005 | [告警中心](0005-alert-center.md) | 已建成 · 五种告警 · 三个决定推翻两个（就地留痕）· `no_text_layer` 待接 | -| 0006 | [本体规模与抽取提示词](0006-ontology-scale-and-the-prompt.md) | 已建成 · 「内置类恒在」换成祖先补齐 · 预算与候选数仍待测 · 外部答案键未做 | -| 0007 | [谁来决定一个说法值不值得成为关系](0007-who-decides-what-becomes-a-relation.md) | 已建成 · 六条缺陷全修 · 起点（种子、`related_to`）已不存在 · `_by` 折叠与叙述动词待解 | -| 0008 | [预制本体包作为冷启动](0008-ontology-packs-as-cold-start.md) | 已建成 · 五包内嵌、多选、对齐表 22 条 · 三个开放问题全开，中文标签变严重 | -| 0009 | [「还没判出来」不该是一个类](0009-no-type-is-a-type.md) | 已实施 · `disjointWith` 落库但消解侧未消费 · `metric`/`dimension` 去处未答且有可见代价 | -| 0010 | [「说不出是什么关系」不该是一个关系](0010-no-relation-is-no-relation.md) | 已实施 · 待做两条随 0011 完成 · 两处死代码待清 | -| 0011 | [「怎么算」不是「有什么」](0011-a-mapping-is-not-a-fact.md) | 已实施（#126 / #140 / #148）· 证据链未做 · 多源选法未做 | -| 0012 | [本体是一份契约，不只是一份建议](0012-the-ontology-is-a-contract-not-a-suggestion.md) | 已实施 · 违反率 57%→4%、反向 39→0 · 守卫已扩到采纳与合并（#190 / #196），另两条待做仍在 | -| 0013 | [一个来源该交出它的历史，不是它的现状](0013-a-source-should-hand-over-its-history.md) | 已实施两个（GitHub / Jira）· 文档协作类未开工 | -| 0014 | [身份跟着人，范围跟着令牌](0014-identity-from-the-person-scope-from-the-token.md) | 已实施（#180）· MCP 只读五工具 · 令牌页在账户层（A2）· 误导性的占位 crate 已删 | -| 0015 | [记下一句话，不等于断言一个事实](0015-recording-a-sentence-is-not-asserting-a-fact.md) | 已实施 · 记忆抽出的事实进 `pending_facts`，Review 新档 + 跟在 remember 步骤后的确认卡 · `remember` 重新打开 · MCP 放开写是下一刀 | -| 0016 | [先把开着的口子收上，再开新的](0016-close-the-open-seams-before-cutting-new-ones.md) | 规划中 · v0.1.0 之后的排期：A 收口 → B 推理机 ∥ C 尺子与本体 → D 语义层 → E 企业交付；模拟引擎后置 | - -## 不是决策记录的那些 - -**[../pipeline.md](../pipeline.md) —— 一份文档如何变成图谱。** 决策记录讲「为什么这样 -而不是那样」；那一篇讲「东西怎么流的、在哪一步会被丢掉」，带五张 mermaid 图（2026-09-02 加了公理那一张）。 -新来的人先看它，再回来看这里的理由。 - -这是本目录约定里说的「第二种性质不同的文档」。它没进子目录——放在 `docs/` 根上， -`.gitignore` 里单独开口，跟 `decisions/` 并列。 - -## 不放这里的东西 - -`docs/` 根目录是**本地草稿区**（`.gitignore` 里 `/docs/*` 忽略，只对 `/docs/decisions/` 开口）。随手的调研笔记、临时清单、跑测试的中间产物放那儿，不入库。等某份草稿沉淀出了值得留的判断，再作为一篇决策记录搬进来。 +| 0001 | [Ontology import and governance](0001-ontology-import-and-governance.md) | In progress · P0–P2c built; the P3 budget built, P3a by hand only; P3b built in a different shape; P4b / P4c pending; P5 delivered by 0002; criterion 2 half overturned by 0012 | +| 0002 | [Reasoning engine](0002-reasoning-engine.md) | R0 checker and R1 materialization (KB switch, default off) built; R2 proof chain (#227); contradiction signals per 0017; R3 incremental maintenance not built | +| 0003 | [The ontology grows out of the corpus](0003-ontology-growth-loop.md) | Built and running, default on · starting point rewritten by 0010 and the retired seeds · dismissal redone per 0007 · the "new phrasings" reminder pending | +| 0004 | [Language follows the reader of each text](0004-language-and-localization.md) | Built · UI strings, coded server errors, ontology description language and the locale of generated text · the browser language is not guessed yet | +| 0005 | [The alert center](0005-alert-center.md) | Built · five alert kinds live, search and paging in the panel · `document.no_text_layer` still unwired | +| 0006 | [Ontology scale and the extraction prompt](0006-ontology-scale-and-the-prompt.md) | Built · the character budget (24,000) and per-chunk retrieval live, values untested · answer keys still hand-filled | +| 0007 | [Counting decides what becomes a relation](0007-who-decides-what-becomes-a-relation.md) | Built · adoption decided by counting (`MIN_DOCS = 2`, `MIN_SIGNALS = 3`), proposals persist (#112) · narrative verbs and `_by` folding still open | +| 0008 | [Ontology packs as the cold start](0008-ontology-packs-as-cold-start.md) | Built · five packs embedded, multi-select at creation, schema.org by default · three open questions stay open; Chinese labels got worse | +| 0009 | [An undecided type stays empty](0009-no-type-is-a-type.md) | Implemented · `type_id` nullable, builtin classes gone · kin classes go to Review (#226), declared `disjointWith` keeps them apart (0016 B3) · `metric` / `dimension` builtin on demand (#231) | +| 0010 | [An unnamed relation stays empty](0010-no-relation-is-no-relation.md) | Implemented · `predicate_id` nullable, `related_to` gone, wording recovered by `fact_surface_predicate` · follow-ups done with 0011 | +| 0011 | [A mapping is configuration](0011-a-mapping-is-not-a-fact.md) | Implemented (#126 / #140 / #148) · Review flow and revision history rebuilt · the evidence chain not built | +| 0012 | [The ontology is a contract](0012-the-ontology-is-a-contract-not-a-suggestion.md) | Implemented · violation rate 57% → 4%, reversals 39 → 0 · guard extended to adoption and merge (#190 / #196) · reified-shell filter at pack import open | +| 0013 | [A source hands over its history](0013-a-source-should-hand-over-its-history.md) | Implemented for GitHub, Jira and Notion (#134 / #135 / #213) · Feishu and Confluence not started · `instant` precision not triggered | +| 0014 | [Identity from the person, scope from the token](0014-identity-from-the-person-scope-from-the-token.md) | Implemented (#180) · five read-only MCP tools over Streamable HTTP · tokens page at `/account/tokens` · `can_write` still hard-coded false | +| 0015 | [A recorded sentence waits for a nod](0015-recording-a-sentence-is-not-asserting-a-fact.md) | Implemented · memory facts wait in `pending_facts`, nod queue and chat card, `remember` reopened · MCP write is the next cut | +| 0016 | [Close the open seams before cutting new ones](0016-close-the-open-seams-before-cutting-new-ones.md) | In progress · A done · B done (B4 deferred) · C untouched · D2 worked around (#231); the lakehouse landed ahead of D4 (#239) | +| 0017 | [A contradiction points at an error upstream](0017-a-contradiction-points-upstream.md) | Implemented · B2a: engine and queue, per-item cap, aggregation by rule pair, cards with clues and repairs (#238) · B2b: contested edges in the alert colour, ghost edges for blocked derivations, the disputed chip and the "did not land" section in the panel (#243) | +| 0018 | [The lakehouse is one protocol away](0018-the-lakehouse-is-one-protocol-away.md) | Implemented: Trino (Iceberg / Delta / Hive), Databricks and Snowflake behind the same trait, scheme picks the engine (#239) · replay tests only, real clusters wanted (#240–#242) · MaxCompute waits | + +## Not a decision record + +**[../pipeline.md](../pipeline.md), how a document becomes a graph.** Records explain why; that page explains how things flow and where they get dropped, with five mermaid diagrams. Newcomers read it first, then come back here for the reasons. It is the "second kind of document" the conventions mention, kept at the `docs/` root beside `decisions/`. + +## What does not belong here + +The `docs/` root is a local scratch area (`/docs/*` is git-ignored except `/docs/decisions/`). Research notes, temporary checklists and test output live there and stay out of the repository. When a draft settles into a judgment worth keeping, it moves here as a record. diff --git a/docs/pipeline.md b/docs/pipeline.md index 355e7878..34cb2b9e 100644 --- a/docs/pipeline.md +++ b/docs/pipeline.md @@ -1,34 +1,31 @@ -# 一份文档如何变成图谱 +# How a Document Becomes a Graph -**这一篇不讲为什么，讲东西怎么流的。** 「为什么这样而不是那样」在 [decisions/](decisions/README.md)； -这里回答另一个问题：**我改的这一行，处在整条链的哪个位置，它上游给我什么、我不给下游什么会断在哪。** +**This page describes how a document flows through the pipeline.** The reasons behind each choice live in [decisions/](decisions/README.md). This page answers a different question: **where the line you are editing sits in the chain, what upstream hands it, and what breaks downstream if you hand nothing on.** -> 图上最值钱的不是箭头，是**箭头断掉的地方**。每一段末尾都有一节「这里会丢东西吗」， -> 列的是代码里真实存在的丢弃点与它们在库里的落点——不是"理论上可能失败"， -> 是**已经在 `extraction_drops` 里数得出来的那几种**。 +> The most valuable thing on the map is **where the arrows break**. Every stage ends with a "Where this stage drops things" section listing the drop points that actually exist in the code and where they land in the database: the kinds you can already count in `extraction_drops`, rather than theoretical failures. -## 全景 +## Overview ```mermaid flowchart TB - U[上传 / 来源同步] --> P[解析
parsers.rs] - P --> C[分块
1200 字符 · 重叠 150] - C --> E1[嵌入
chunks.embedding] - E1 --> RDY[(文档 ready
可搜可问)] - E1 --> X[抽取
每块一次 LLM] - X --> ENT[实体消解
这一条是谁] - X --> FCT[事实落库
双时态账本] - ENT --> ADJ[裁决
攒批一次 LLM] - ADJ --> MRG[合并 / 保持分开] - FCT --> TR[类型消解
这一条是什么] - FCT --> GROW[本体增长
词表外的说法回流成提案] - MRG --> G[(图谱)] + U[Upload / source sync] --> P[Parse
parsers.rs] + P --> C[Chunk
1200 chars · overlap 150] + C --> E1[Embed
chunks.embedding] + E1 --> RDY[(Document ready
searchable and askable)] + E1 --> X[Extract
one LLM call per chunk] + X --> ENT[Entity resolution
who this mention is] + X --> FCT[Facts to store
bitemporal ledger] + ENT --> ADJ[Adjudication
one batched LLM call] + ADJ --> MRG[Merge / keep apart] + FCT --> TR[Type resolution
what this entity is] + FCT --> GROW[Ontology growth
out-of-vocabulary terms become proposals] + MRG --> G[(Graph)] TR --> G - GROW --> ONT[(本体)] - ONT -.喂回.-> X - G --> R0[一致性检查
公理 vs 事实 · 不写库] + GROW --> ONT[(Ontology)] + ONT -.feeds back.-> X + G --> R0[Consistency check
axioms vs facts · writes nothing] ONT --> R0 - G --> R1[物化推导
开关 · 派生另存] + G --> R1[Materialized inference
opt-in · derived facts stored separately] ONT --> R1 R1 --> G @@ -37,274 +34,229 @@ flowchart TB style ONT fill:#2d4a5a,color:#fff ``` -**两段式是有意的**：嵌入完成即 `ready`，搜索与问答立刻可用，抽取在后台排队。 -一篇长文档的图谱要几分钟才长出来，但它在几十秒内就能被搜到。 +**The two-phase split is deliberate.** A document is `ready` as soon as embedding finishes; search and Q&A work immediately while extraction queues in the background. A long document takes minutes to grow its graph but is searchable within seconds. -**本体那条回流虚线是这套东西的循环**：抽取用本体，抽取遇到本体没有的说法就把原词记下来， -提案回流补进本体，下一批文档的抽取就用上了。见 [0003](decisions/0003-ontology-growth-loop.md)。 +**The dotted line back into extraction is the loop.** Extraction uses the ontology; when it meets a term the ontology lacks, it records the original wording; proposals flow back into the ontology; the next batch of documents is extracted with it. See [0003](decisions/0003-ontology-growth-loop.md). -**本体从哪来**：建库时什么都不种。起点是可选的预制包（schema.org 默认勾选，另有 W3C Org、PROV-O、FOAF、IOF Core）， -或者用户导入自己的 OWL，或者空着——空库照样能抽，实体没有类型就是没有类型。见 [0008](decisions/0008-ontology-packs-as-cold-start.md)、[0009](decisions/0009-no-type-is-a-type.md)。 +**Where the ontology comes from.** A new database seeds nothing. The starting point is an optional prebuilt pack (schema.org checked by default; also W3C Org, PROV-O, FOAF, IOF Core), a user-imported OWL file, or nothing at all. An empty ontology still extracts; an entity without a type simply has no type. See [0008](decisions/0008-ontology-packs-as-cold-start.md) and [0009](decisions/0009-no-type-is-a-type.md). -**最下面那两个框是本体的公理在干活**：一致性检查不写 `facts`，只把矛盾摆出来（Review 的 violations / defects 两档）； -物化推导默认关，打开后派生事实另存一张表、图上金色，永远不闭合任何断言事实。见第四节。 +**The bottom two boxes are the ontology's axioms at work.** The consistency check never writes `facts`; it only surfaces contradictions (the Review page's two tiers, violations and defects). Materialized inference is off by default; when on, derived facts go to a separate table, show as gold on the graph, and never close any asserted fact. See section 4. --- -## 一、抽取一个分块 +## 1. Extracting a Chunk ```mermaid flowchart TB - subgraph 提示词 - B{本体装得下预算吗?} - B -->|装得下| FULL[全量铺
小本体的老路] - B -->|装不下| RET[按这一块的向量检索
约 40 类 / 30 关系 / 30 属性
+ 命中类的祖先一起铺] + subgraph prompt [Prompt] + B{Does the ontology fit the budget?} + B -->|yes| FULL[Lay out the whole ontology
the small-ontology path] + B -->|no| RET[Retrieve by the chunk vector
about 40 classes / 30 relations / 30 attributes
+ ancestors of the hit classes] end FULL --> LLM[LLM] RET --> LLM - CHK[分块正文 + 本文档已认下的实体] --> LLM - LLM --> J{输出的每一条} - J -->|entities| EN[实体
type 从清单挑
specific_type 自由文本] - J -->|predicate 命中属性| AT[属性事实
值按 datatype 归一] - J -->|predicate 命中关系| RL[关系事实] - RL --> DIR{主宾类型
对得上签名?} - DIR -->|对| OK[落库] - DIR -->|主语违反且宾语符合| SWAP[按签名对调主宾
留 direction_corrected] - DIR -->|对调也不合法| NOP[谓词留空
主宾时间证据都留] - J -->|词表外 + 字面值| LIT[值落 object_value
原词落 proposed_predicate] - J -->|词表外 + 实体宾语| FB[谓词留空
原词落 proposed_predicate] + CHK[Chunk text + entities already accepted in this document] --> LLM + LLM --> J{Each item in the output} + J -->|entities| EN[Entity
type picked from the list
specific_type free text] + J -->|predicate matches an attribute| AT[Attribute fact
value normalized by datatype] + J -->|predicate matches a relation| RL[Relation fact] + RL --> DIR{Subject and object types
match the signature?} + DIR -->|yes| OK[Stored] + DIR -->|subject violates, object fits| SWAP[Swap subject and object per the signature
record direction_corrected] + DIR -->|swap is also invalid| NOP[Predicate left empty
subject, object, time and evidence kept] + J -->|out of vocabulary + literal value| LIT[Value into object_value
original term into proposed_predicate] + J -->|out of vocabulary + entity object| FB[Predicate left empty
original term into proposed_predicate] style LLM fill:#3a3a5a,color:#fff ``` -**`specific_type` 是这一步最容易被忽略的输出**：自由文本、不校验、不入本体，就是模型自己 -对这个实体的说法（"vector database software"）。类型消解靠它把任务从「读懂这是什么」 -换回「本体里哪个类叫这个名字」。没有它，实测 17 个实体的 `proposed_type` 全是空的—— -因为清单里总有个"差不多"的，模型选了它，心里那个更准的说法就此丢失。 +**`specific_type` is the easiest output to overlook.** Free text, unvalidated, never entered into the ontology: it is the model's own description of the entity ("vector database software"). Type resolution uses it to turn the task from "understand what this is" into "which class in the ontology has this name". Without it, a test run left `proposed_type` empty on all 17 entities: the list always has something close enough, the model picks it, and the more precise description is lost. -**词表外的两条路都不丢东西**：带字面值的落 `object_value`（而不是凭空造一个叫「2015」的实体）， -带实体宾语的**谓词留空**——不是降级成一个叫「有关联」的关系，那是断言不是含糊（[0010](decisions/0010-no-relation-is-no-relation.md)）。 -两者的原词都进 `fact_evidence.proposed_predicate`，那是这条事实身上唯一还留着原意的地方，显示时由 `fact_surface_predicate()` 取回。 +**Neither out-of-vocabulary path loses anything.** A literal value goes to `object_value` instead of becoming an entity named "2015". An entity object leaves the **predicate empty** instead of degrading into a relation called "related to", which would be an assertion rather than vagueness ([0010](decisions/0010-no-relation-is-no-relation.md)). In both cases the original term goes to `fact_evidence.proposed_predicate`, the one place on the fact that still carries the original meaning; `fact_surface_predicate()` reads it back for display. -**命中的关系还要过一道签名**：本体声明了 `employee (organization → person)`，模型照样会写 `Musk employee Microsoft`—— -提示词三轮都压不下去，英语的 "X is an employee of Y" 太强。所以写入时掰正：主语违反 domain 而宾语符合就对调，**绝不静默**，留一条 `direction_corrected`； -对调也不合法（`OpenAI affectedBy …`，schema.org 里那是医学检验用的）就丢掉谓词、留下主宾与证据。参数顺序是 key 的编码约定，不是关于世界的断言，所以这一处本体是执法的。 -见 [0012](decisions/0012-the-ontology-is-a-contract-not-a-suggestion.md)。 +**A matched relation still passes a signature check.** The ontology declares `employee (organization → person)`, and the model still writes `Musk employee Microsoft`; three rounds of prompt tuning could not suppress it, because English "X is an employee of Y" is too strong. So the write path corrects it: if the subject violates the domain and the object fits, swap them and record `direction_corrected`, never silently. If the swap is also invalid (`OpenAI affectedBy …`, a medical-test predicate in schema.org), drop the predicate and keep subject, object and evidence. Argument order is an encoding convention of the key, not a claim about the world, so here the ontology enforces. See [0012](decisions/0012-the-ontology-is-a-contract-not-a-suggestion.md). -### 这里会丢东西吗 +### Where this stage drops things -会。十二种原因码，全部记进 `extraction_drops`，界面上可见（其中一种不是丢弃，是留痕）： +Eleven reason codes, all recorded in `extraction_drops` and visible in the UI (one is a trace, not a drop): -| 原因 | 什么时候 | +| Reason | When | |---|---| -| `truncated_reply` | 模型的输出被截断，这一块整块作废 | -| `malformed_item` | 一条事实格式不对——**只丢这一条**，不丢整块（#127） | -| `not_an_entity_name` | 「实体名」是一整句话（按词数 + 限定动词判，#143） | -| `low_confidence` | 模型自报置信度低于阈值 | -| `subject_not_declared` | 主语没在 `entities` 里声明——关系与属性两条路径**都**记 | -| `attr_domain_mismatch` | 属性挂到了 domain 之外的类上（沿父类 DAG 上溯仍不匹配） | -| `attr_no_value` / `attr_datatype` | 属性事实没给值，或值换算不出声明的 datatype | -| `object_missing` | 关系事实没有宾语 | -| `direction_corrected` | **不是丢弃**：主宾按签名对调了，记下来是为了不静默 | -| `domain_mismatch` | 对调也不合法，谓词被丢掉（主宾与证据留下） | - -**`attr_domain_mismatch` 是最贵的一种**：它在落地当场丢弃，而事后改类救不回来—— -那条事实从没写入过，只能重抽。 - -从前还有一种 `fallback_relation_missing`（兜底关系被删了就整条消失）——随 `related_to` 一起没了。 +| `truncated_reply` | The model's output was cut off; the whole chunk is discarded | +| `malformed_item` | One fact is malformed; **only that item** is dropped, not the chunk (#127) | +| `not_an_entity_name` | The "entity name" is a whole sentence (judged by word count and finite verbs, #143) | +| `low_confidence` | The model's self-reported confidence is below the threshold | +| `subject_not_declared` | The subject is not declared in `entities`; recorded on **both** the relation and attribute paths | +| `attr_domain_mismatch` | The attribute is attached to a class outside its domain, even after walking up the parent DAG | +| `attr_no_value` / `attr_datatype` | The attribute fact has no value, or the value cannot be converted to the declared datatype | +| `object_missing` | The relation fact has no object | +| `direction_corrected` | **Not a drop**: subject and object were swapped per the signature; recorded so it is never silent | +| `domain_mismatch` | The swap is also invalid; the predicate is dropped (subject, object and evidence stay) | + +**`attr_domain_mismatch` is the most expensive.** It drops the fact at write time, and retyping the entity later does not recover it: the fact was never written and can only be re-extracted. --- -## 二、实体消解：这一条是谁 +## 2. Entity Resolution: Who This Mention Is ```mermaid flowchart TB - M[一次 mention
类型 + 名字 + 分块向量] --> EQ[等值召回
canonical_name 或 aliases 相等] - EQ --> S{画像相似度} - S -->|0.55 及以上| ATT[并进已有实体
更新画像] - S -->|0.35 到 0.55| NEW1[新建 + 入审阅队列] - S -->|低于 0.35| NEW2[新建，不打扰队列] - EQ -->|一个都没有| NEW3[新建] + M[One mention
type + name + chunk vector] --> EQ[Equality recall
canonical_name or aliases equal] + EQ --> S{Profile similarity} + S -->|0.55 and above| ATT[Attach to the existing entity
update its profile] + S -->|0.35 to 0.55| NEW1[Create + enqueue for review] + S -->|below 0.35| NEW2[Create, queue untouched] + EQ -->|no match| NEW3[Create] NEW1 --> CT NEW2 --> CT - NEW3 --> CT[包含关系召回
只在新建时跑一次] - CT --> Q[(审阅队列
pending)] - Q --> AD[裁决
攒批一次 LLM] - AD -->|同一个| ME[合并
名字进 aliases
事实搬过去] - AD -->|不是| KP[保持分开] - ME --> RD[把该 source 上其余 pending
改指到合并目标] + NEW3 --> CT[Containment recall
runs once, on creation only] + CT --> Q[(Review queue
pending)] + Q --> AD[Adjudication
one batched LLM call] + AD -->|same| ME[Merge
name into aliases
facts moved over] + AD -->|different| KP[Keep apart] + ME --> RD[Redirect the other pending pairs on that source
to the merge target] RD --> Q style AD fill:#3a3a5a,color:#fff style Q fill:#2d4a5a,color:#fff ``` -**三个阈值分三档**（`SIM_ATTACH = 0.55`、`SIM_NEW = 0.35`）：像得没话说就并， -像得可疑就新建但入队，不像就新建且不打扰队列。**宁分勿合**——错合的代价是两个实体的 -事实混在一起，比多一个实体贵得多。 +**Two thresholds, three tiers** (`SIM_ATTACH = 0.55`, `SIM_NEW = 0.35`): clearly the same attaches; suspiciously similar creates a new entity and queues it; dissimilar creates one without touching the queue. **Prefer splitting over merging.** A wrong merge mixes two entities' facts together, which costs far more than one extra entity. -**包含关系召回**（`Holmes` ⊂ `Sherlock Holmes`）补等值召回的盲区：前缀枚举不完， -简称会静默变成第二个实体。它有三条约束：较短那个名字至少 4 字符（低于此多是通名）、 -单次最多产出 4 对、SQL 侧多扫 16 行（硬互斥类型在 Rust 侧才筛得掉）。 +**Containment recall** (`Holmes` ⊂ `Sherlock Holmes`) covers the blind spot of equality recall: prefixes cannot be enumerated, so a short name would silently become a second entity. Three constraints: the shorter name is at least 4 characters (below that it is usually a generic word), at most 4 pairs per run, and SQL scans 16 extra rows because hard-disjoint types are only filtered on the Rust side. -**改指那一步**是整张图里最不直觉的一环，也是最容易被误删的：合并之后，涉及被合实体的 -其余待审阅对**不能关掉**，要改指到合并目标。理由见下。 +**The redirect step** is the least intuitive part of the graph and the easiest to delete by mistake. After a merge, the other pending pairs that involve the merged-away entity **must not be closed**; they are redirected to the merge target. The reason follows. -### 这一段修过三个洞，三个都是同一份语料照出来的 +### Three Holes, One Corpus -用《福尔摩斯冒险史》前六篇（`scripts/bench/corpora/holmes.json`）连跑四次，每次修一层： +Four runs over the first six stories of *The Adventures of Sherlock Holmes* (`scripts/bench/corpora/holmes.json`), each fixing one layer: -| | 原始 | 修同类型 | 加别名召回 | 加改指 | +| | Baseline | Same-type fix | Alias recall | Redirect | |---|---|---|---|---| -| 已合并实体 | 14 | 37 | 47 | **57** | -| `Holmes` 并入 | ✗ | ✓ | ✓ | ✓ | -| `Mr. Holmes` 并入 | ✗ | ✗ | ✗ | **✓** | +| Entities merged | 14 | 37 | 47 | **57** | +| `Holmes` merged | ✗ | ✓ | ✓ | ✓ | +| `Mr. Holmes` merged | ✗ | ✗ | ✗ | **✓** | -**第一层**：`classify_type_drift` 没有「两个类型相同」这一档，`person × person` 落进 -`Disjoint`——"永不可能是同一个"。那个函数生来服务「类型漂移」（同名被抽成两种类型）， -那里两边相同不会发生；后来被包含关系召回借去当相容性判据，**而那里两边相同才是常态**。 -全文最明显的同指关系一对都没进过队列，十二个既有单元测试全在测跨类型。 +**Layer one.** `classify_type_drift` had no tier for "both types equal", so `person × person` fell into `Disjoint`, "can never be the same". The function was written for type drift (one name extracted as two types), where equal sides never occur; containment recall later borrowed it as a compatibility test, where equal sides are the norm. The most obvious coreferences in the text never reached the queue, and all twelve existing unit tests covered cross-type cases. -**第二层**：召回只看 `canonical_name`。合并把名字搬进 `aliases`，于是**每成功合并一次 -就拆掉一条桥**——`Holmes` 并入之后，后来的 `Mr. Holmes` 跟 `Sherlock Holmes` 谁也不含谁， -本来正是靠 `Holmes` 桥接。修好第一层反而让第二层的漏显形了。 +**Layer two.** Recall looked only at `canonical_name`. A merge moves the name into `aliases`, so every successful merge removed a bridge: once `Holmes` was merged, the later `Mr. Holmes` and `Sherlock Holmes` contained neither the other, and `Holmes` had been the bridge. Fixing layer one exposed layer two. -**第三层**：合并会把涉及被合实体的其余 pending 审阅关成 `superseded by merge`， -代码注释里的理由是"疑点若仍在会由后续 mention 重新提起"。**那句是错的**：包含关系召回 -只在新建实体时跑，而这些实体早就存在、不会再被新建。关掉即永久关闭。现在改指到合并目标， -只有两类真正过时的才关——重定向后成自环的，和目标对已在队列里的。 +**Layer three.** A merge closed the other pending reviews involving the merged-away entity as `superseded by merge`; the code comment claimed a later mention would raise the doubt again. That was wrong: containment recall runs only when an entity is created, and these entities already exist, so closing was permanent. Now they are redirected to the merge target, and only two genuinely stale kinds are closed: pairs that become self-loops after the redirect, and pairs whose target pair is already queued. -**这三层是一层套一层的**：不修第一层看不见第二层，不修第二层看不见第三层。 -基准语料的价值不在第一次跑出的数字，在**每修一次就再照出下一层**。 +**Each layer hid the next.** The value of the benchmark corpus is the next layer it exposes after every fix, more than the first number it produces. -### 这里会丢东西吗 +### Where this stage drops things -不会丢事实，但会**留下不该分开的实体**。两个已知缺口： +No facts are lost, but **entities that belong together can stay apart**. Two known gaps: -- 两个名字既不互相包含、又没有共同别名做桥（`启明 X7 加速卡` vs `启明 X7 推理加速卡`）。 - 要三元组相似度，而 `CREATE EXTENSION pg_trgm` 需要超级权限，本仓库是受限角色连库。 -- 单次包含关系召回上限 4 对：一个通名可能被几十个实体包含，全放进去会淹掉队列。 +- Two names that neither contain each other nor share an alias as a bridge (`启明 X7 加速卡` vs `启明 X7 推理加速卡`). Trigram similarity would cover this, but `CREATE EXTENSION pg_trgm` needs superuser and this repository connects with a restricted role. +- Containment recall is capped at 4 pairs per run. A generic name may be contained by dozens of entities, and admitting them all would flood the queue. -合并本身**可撤销**（`entity_merges` 记着改之前的一切，`revert_merge` 放回去）。 +A merge itself is **reversible**: `entity_merges` records everything as it was, and `revert_merge` restores it. --- -## 三、本体消解：这一条是什么 +## 3. Type Resolution: What This Entity Is ```mermaid flowchart TB - subgraph 抽取留下的线索 - PT[proposed_type
词表外的类型名] - ST[specific_type
模型自己的说法] - PP[proposed_predicate
词表外的谓词原词] + subgraph clues [Clues left by extraction] + PT[proposed_type
out-of-vocabulary type name] + ST[specific_type
free-text description from the model] + PP[proposed_predicate
out-of-vocabulary predicate, original term] end PT --> TR ST --> TR - PP --> GP[本体提案
检索候选 + 裁决] - TR[类型消解] --> C1[候选一
画像 → 类的描述] - TR --> C2[候选二
语境近邻的类当票投] - C1 --> AD2{裁决} + PP --> GP[Ontology proposal
retrieve candidates + adjudicate] + TR[Type resolution] --> C1[Candidate one
profile → class descriptions] + TR --> C2[Candidate two
classes of context neighbors vote] + C1 --> AD2{Adjudication} C2 --> AD2 - AD2 -->|在原类子树里| AUTO[自动改类
entity_retypes] - AD2 -->|跨了分类轴| REV[待人工
类对认可一次即免问] - AD2 -->|都不是| NONE[不动
并记下理由] - GP -->|已有的| MAP[映射到已有类型
改写等待的事实] - GP -->|没有的| NEWT[新建类型 + 改写] + AD2 -->|within the original class subtree| AUTO[Retype automatically
entity_retypes] + AD2 -->|crosses a classification axis| REV[Human review
a class pair approved once is never asked again] + AD2 -->|neither| NONE[Leave alone
reason recorded] + GP -->|type exists| MAP[Map to the existing type
rewrite the waiting facts] + GP -->|type is new| NEWT[Create the type + rewrite] style AD2 fill:#3a3a5a,color:#fff ``` -**两路候选取并集，不合分数。** 距离在三处都不可比：跨实体不可比 -（`清华大学计算机系→computer_store` 0.46 比 `星云科技→corporation` 0.59 还近，而前者荒谬）、 -两路之间不可比（一个在类空间一个在实体空间）、同一路的两个查询之间也不可比 -（短查询"医药集团"产生的距离系统性小于一整段画像）。**一律交替取。** +**The two candidate routes are unioned, never scored together.** Distances are incomparable in three places: across entities (`清华大学计算机系→computer_store` at 0.46 is closer than `星云科技→corporation` at 0.59, and the former is absurd), between the two routes (one lives in class space, the other in entity space), and between two queries on the same route (a short query like "医药集团" yields systematically smaller distances than a full profile). Candidates are interleaved instead. -**分档不看模型自报的 confidence**——实测是双峰的（15 条全 ≥0.85、4 条 null，中间没有）， -自报置信度是语气不是概率。改用「选中的类在不在原类的子树里」：在 = 往下走一格，自动； -不在 = 换了分类轴，进人工。 +**Tiering ignores the model's self-reported confidence.** In practice it is bimodal (15 items all ≥ 0.85, 4 null, nothing in between): it is tone, not probability. The tier comes from whether the chosen class sits in the original class's subtree. Inside means one step down, applied automatically; outside means a different classification axis, sent to a human. -**纠正也走人工，而且天然如此**：抽取按块检索候选之后自己就会挑细类，也会挑错 -（`绍兴 → address`）；正确答案是错类的**兄弟**不是后代，所以必然判为跨轴。 -推翻抽取的判断比细化它风险大，不该自动发生。 +**Corrections go to a human as well, by construction.** Extraction already picks fine-grained classes after per-chunk retrieval, and sometimes picks wrong ones (`绍兴 → address`). The right answer is a **sibling** of the wrong class, so it always reads as crossing an axis. Overturning extraction's judgment is riskier than refining it and should not happen automatically. -### 这里会丢东西吗 +### Where this stage drops things -不丢事实，但**改类不进时间轴**——它是 `entities` 上一次 UPDATE 加一行 `entity_retypes`〔实体历史现在会显示 `retyped` / `retype_reverted` 两种事件，这一条已经补上〕。**可撤销不等于会被撤销**， -这是先做 preview 再做 apply 的理由。 +No facts are lost. A retype is one UPDATE on `entities` plus a row in `entity_retypes`, and entity history shows it as `retyped` / `retype_reverted` events. **Reversible does not mean it will be reversed**, which is why preview comes before apply. -**类型消解今天只能手动跑**——本体页 preview → apply，没有任何自动触发。抽取结束只入队本体扩展与实体裁决。 -所以大本体下新实体的细化依赖人记得去点一下，这是个真缺口（0001 P3a）。 +**Type resolution runs only by hand today**: preview → apply on the ontology page, with no automatic trigger. Finishing extraction only enqueues ontology growth and entity adjudication, so refining new entities under a large ontology depends on someone remembering to click. This is a real gap (0001 P3a). -**人拍过板的不再被引擎重判**：`entities.type_source` 是 `human` 的实体不进取材，包括「人判了，就是没有类型」（0001 P4a）。 +**Human decisions are never re-judged by the engine.** Entities whose `entities.type_source` is `human` are excluded from the candidate pool, including "a human decided it has no type" (0001 P4a). -**拒绝要给理由。** `left_alone` 曾经只是个数，而这一步的设计押在"选择都不是是个体面答案"上—— -最大的一档不透明。记上理由之后第一次跑就回答了此前答不出的问题：失败**全在检索一侧** -（`administrative_area`、`periodical` 从没被端上来过），不在裁决。 +**Rejections carry a reason.** The design bets on "none of these" being an honest answer, which made `left_alone` the largest and most opaque tier while it was only a count. With reasons recorded, the first run answered a question that had been unanswerable: the failures were **all on the retrieval side** (`administrative_area` and `periodical` were never offered), none in adjudication. --- -## 四、公理：检查与推导 +## 4. Axioms: Checking and Inference ```mermaid flowchart TB - ONT[(本体的公理
functional · symmetric · asymmetric
transitive · inverseOf · subPropertyOf · disjoint)] --> SELF[本体自检
八类缺陷] + ONT[(Ontology axioms
functional · symmetric · asymmetric
transitive · inverseOf · subPropertyOf · disjoint)] --> SELF[Ontology self-check
eight defect kinds] SELF --> DEF[(ontology_defects)] - ONT --> R0[事实层检查
自环 · 反对称 · 传递环 · 基数 · 签名] - G[(图谱)] --> R0 - R0 --> VIO[(axiom_violations
带完整路径)] - VIO --> DEC{人裁} - DEC -->|撤回事实| RET[事实作废] - DEC -->|放宽公理| RLX[改本体] - DEC -->|接受| ACC[两条都留] - ONT --> R1{materialize_inferences
开关，默认关} + ONT --> R0[Fact-level check
self-loop · asymmetry · transitive cycle · cardinality · signature] + G[(Graph)] --> R0 + R0 --> VIO[(axiom_violations
with the full path)] + VIO --> DEC{Human decision} + DEC -->|retract the fact| RET[Fact retracted] + DEC -->|relax the axiom| RLX[Edit the ontology] + DEC -->|accept| ACC[Both stay] + ONT --> R1{materialize_inferences
switch, off by default} G --> R1 - R1 -->|开| DER[(derived_facts
另一张表 · rule_id · 前提链)] - DER --> GV[图上金色边
实体面板「推出来的」] + R1 -->|on| DER[(derived_facts
separate table · rule_id · premise chain)] + DER --> GV[Gold edges on the graph
entity panel marks them inferred] style DEC fill:#3a3a5a,color:#fff ``` -**签名在三条写路径上都算数**（#190 / #196）：第一节那道「主宾对得上签名吗」的判断住在 store（`ontology::judge_direction`），抽取落新事实、采纳把谓词挂回旧事实都调它——两边都不合的采纳**不挂**，条数随 `facts_left_off` 上报；合并换了主宾之后不掰不改，只对搬动过的事实查一遍，违反的进 `axiom_violations`（`signature`）。检查里的第五类就是它的全量版，本体事后改了 domain 也逃不过。 +**Signatures count on all three write paths** (#190 / #196). The "do subject and object match the signature" check from section 1 lives in the store (`ontology::judge_direction`). Extraction calls it when writing new facts. Adoption calls it when reattaching a predicate to old facts; where both directions fail, the predicate is **not attached** and the count is reported as `facts_left_off`. A merge that changes subject or object neither swaps nor edits; it re-checks only the moved facts, and violations go to `axiom_violations` as `signature`. The `signature` check kind is the full-scan version of the same rule, so a domain changed in the ontology after the fact is caught too. -**检查不写库，推导写另一张表。** 一致性检查（R0）只指出问题，风险面为零；物化推导（R1）会往图里加东西，所以它的每条约束都是必要的： -规则**只从本体公理编译**，没有用户 DSL；**断言硬性优先于派生**——已经断言过的三元组不再派生，「这条是谁说的」有唯一答案； -深度上限加环检测，每谓词封顶两万条且被截掉的**要说出来**；有效时间取前提的交集，空交集不推。 -派生不进 `facts`：四十多处读 `facts` 的查询只有一处认得标记，分开之后忘了 UNION 的后果是**看不见**派生，而不是**混进去**。 +**The check writes nothing; inference writes to a separate table.** The consistency check (R0) only points at problems, with zero risk. Materialized inference (R1) adds to the graph, so each of its constraints is necessary: rules **compile only from ontology axioms**, with no user DSL; **asserted facts strictly override derived ones**, so a triple already asserted is never derived and "who said this" has one answer; a depth limit plus cycle detection, a cap of 20,000 per predicate, and truncation **is reported**; valid time is the intersection of the premises, and an empty intersection derives nothing. Derived facts never enter `facts`: of the forty-odd queries that read `facts`, only one recognizes a marker, so with separate tables a forgotten UNION makes derived facts **invisible** rather than **mixed in**. -**本体自检排在前面**：自相矛盾的本体（一个关系既 symmetric 又 asymmetric、子类成环、逆没指回来）会让事实层的结论全部可疑。 +**The ontology self-check runs first.** A self-contradictory ontology (a relation both symmetric and asymmetric, a subclass cycle, an inverse that does not point back) makes every fact-level conclusion suspect. -**没装本体包的库跑出来是零**，那是实情不是故障——没有公理就没有判据，不报矛盾比猜一个公理出来安全。 +**A database without an ontology pack reports zero.** That is the truth, not a fault: without axioms there is no criterion, and reporting no contradiction is safer than guessing an axiom. -**什么时候跑**：导入本体后自动跑一次检查（公理刚变，最该重算的时刻）；Review 页可手动跑；推导按 `inference_interval_minutes`（缺省 60）定时全量重推——增量维护还没做。 +**When it runs.** The check runs automatically after an ontology import (the axioms just changed, the best moment to recompute) and on demand from the Review page. Inference reruns in full on a timer, `inference_interval_minutes` (default 60); incremental maintenance is not built yet. -### 这里会丢东西吗 +### Where this stage drops things -不丢，但**有两处沉默**：派生 vs 断言矛盾时派生不落地，今天**不记信号**；同一三元组有多条推导路径只留第一条证明。两者都是 [0002](decisions/0002-reasoning-engine.md) 里写了而没做的。 +Nothing is dropped, but the engine keeps quiet in two places. A derivation that contradicts an assertion (or another derivation) is not written; since #238 it leaves an `axiom_violations` row of kind `derived_contradiction`, capped per predicate, with the overflow counted in the run report and rule-versus-rule disagreements recorded as `ontology_defects` (`rules_disagree`) — see [0017](decisions/0017-a-contradiction-points-upstream.md). When one triple has several derivation paths, only the first proof is kept; that one is still silent ([0002](decisions/0002-reasoning-engine.md)). -## 想自己跑一遍 +## Running the Benchmarks -`scripts/bench/` 是可重跑的测量台，**每一组一个新库**——复用一个库省几分钟， -换来的是一整段无效结论（那是踩过的坑，不是假设）。 +`scripts/bench/` is a rerunnable measurement bench, **one fresh database per run**. Reusing a database saves minutes and costs a whole run of invalid conclusions; that is a lesson learned, not a hypothesis. ```bash node scripts/bench/run.mjs --corpus pharma --label seeds-only node scripts/bench/run.mjs --corpus holmes --label holmes ``` -三份语料各测一件事，用途不同、要求也不同： +Three corpora measure three things, with different requirements: -| 语料 | 测什么 | 有答案键 | +| Corpus | Measures | Answer key | |---|---|---| -| `tech` / `pharma` | 类型准确性 | 有（弱：自己写的） | -| `holmes` | 实体消解 · demo 空镜 | **无，故意的** | +| `tech` / `pharma` | Type accuracy | Yes (weak: hand-written) | +| `holmes` | Entity resolution · demo footage | **None, by design** | -福尔摩斯那份**不该有**准确性答案键：模型早就读过它，量类型准确率量到的是记忆， -不是这条流水线。**编一份假答案比不打分更糟。** +The Holmes corpus **must not** have an accuracy key: the model has read the book, so measuring type accuracy would measure memory, not the pipeline. A fabricated answer key is worse than no score. -## 相关决策 +## Related Decisions -- [0001](decisions/0001-ontology-import-and-governance.md) 本体导入与治理，含 P3 的实测修订 -- [0003](decisions/0003-ontology-growth-loop.md) 本体从语料里长出来，人站在哪一环 -- [0006](decisions/0006-ontology-scale-and-the-prompt.md) 本体规模与抽取提示词，含曲线与一次撤回 -- [0002](decisions/0002-reasoning-engine.md) 推理机的顺序与安全边界；[0012](decisions/0012-the-ontology-is-a-contract-not-a-suggestion.md) 写入时的方向掰正 -- [0009](decisions/0009-no-type-is-a-type.md) / [0010](decisions/0010-no-relation-is-no-relation.md) 为什么「还没判出来」不是类、「说不出」不是关系 +- [0001](decisions/0001-ontology-import-and-governance.md) Ontology import and governance, with the measured revision of P3 +- [0003](decisions/0003-ontology-growth-loop.md) Growing the ontology from the corpus, and where the human stands in the loop +- [0006](decisions/0006-ontology-scale-and-the-prompt.md) Ontology scale and the extraction prompt, with the curve and one reversal +- [0002](decisions/0002-reasoning-engine.md) Reasoning engine order and safety boundary; [0012](decisions/0012-the-ontology-is-a-contract-not-a-suggestion.md) direction correction at write time +- [0017](decisions/0017-a-contradiction-points-upstream.md) A contradiction points at an error upstream: what happens when a derivation disagrees with the ledger +- [0009](decisions/0009-no-type-is-a-type.md) / [0010](decisions/0010-no-relation-is-no-relation.md) An undecided type and an unnamed relation both stay empty diff --git a/migrations/0020_a_contradiction_points_upstream.sql b/migrations/0020_a_contradiction_points_upstream.sql new file mode 100644 index 00000000..f893e556 --- /dev/null +++ b/migrations/0020_a_contradiction_points_upstream.sql @@ -0,0 +1,34 @@ +-- 派生撞上断言时，让路这件事从静默变成可见（docs/decisions/0017）。 +-- +-- 0002 定了 asserted > derived：推出来的事实撞上账本里的断言就不落地。此前那一步 +-- 什么都不留——`ceo_of ⊑ works_at` 推出的 works_at 没了，人不知道有过这回事，也就 +-- 不知道该去看看是抽取错了、旧断言该闭合、还是两个「Mira」其实是一个人。 +-- +-- 一致性检查多一种 `derived_contradiction`：left 是被撞的断言，right 是派生的最后一条 +-- 前提，path 是全部前提；推出来的三元组本身没有落库、没有 id 可指，放进 `detail`。 +-- 出路多一条 `fact_closed`——最常见的修法是给旧断言一个结束日期。 +-- +-- 派生之间互撞（两条规则加在一起产出互斥的结论）按规则对聚合进 `ontology_defects`， +-- 一种 `rules_disagree`，`detail` 记规则对与几个例子。逐对进 Review 只会淹掉队列。 + +ALTER TABLE axiom_violations + DROP CONSTRAINT axiom_violations_kind_check, + ADD CONSTRAINT axiom_violations_kind_check CHECK (kind IN ( + 'self_loop', 'asymmetry', 'cycle', 'functional', 'signature', + 'derived_contradiction' + )), + DROP CONSTRAINT axiom_violations_resolution_check, + ADD CONSTRAINT axiom_violations_resolution_check CHECK (resolution IN ( + 'fact_retracted', 'fact_closed', 'axiom_relaxed', 'accepted' + )), + ADD COLUMN detail JSONB NOT NULL DEFAULT '{}'::jsonb; + +ALTER TABLE ontology_defects + DROP CONSTRAINT ontology_defects_kind_check, + ADD CONSTRAINT ontology_defects_kind_check CHECK (kind IN ( + 'symmetric_and_asymmetric', 'transitive_and_functional', 'subclass_cycle', + 'disjoint_with_ancestor', 'inherits_disjoint', + 'inverse_of_itself', 'inverse_not_mutual', 'sub_property_cycle', + 'rules_disagree' + )), + ADD COLUMN detail JSONB NOT NULL DEFAULT '{}'::jsonb; diff --git a/migrations/0021_lakehouse_engines.sql b/migrations/0021_lakehouse_engines.sql new file mode 100644 index 00000000..3781a6a6 --- /dev/null +++ b/migrations/0021_lakehouse_engines.sql @@ -0,0 +1,7 @@ +-- 问数引擎扩到 HTTP 协议族：trino（Iceberg / Delta / Hive 都是它的 catalog）、 +-- databricks（SQL Statement API）、snowflake（SQL API v2）。 +-- 挂载模型与注册表引擎无关（0006 的判断仍成立），这里只放宽 engine 的取值； +-- 允许的名字与 `query_engine::ENGINES` 同一张表。 +ALTER TABLE data_sources DROP CONSTRAINT data_sources_engine_check; +ALTER TABLE data_sources ADD CONSTRAINT data_sources_engine_check + CHECK (engine IN ('postgres', 'trino', 'databricks', 'snowflake')); diff --git a/web/src/api.ts b/web/src/api.ts index 4b835cad..1ed58609 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -1,3 +1,4 @@ +import type { SourceKind } from "./sourceKinds"; import { S, lang } from "./i18n"; export class ApiError extends Error { @@ -179,21 +180,7 @@ export interface ExtractionDrop { export interface SourceView { id: string; - kind: - | "folder" - | "url" - | "rss" - | "api" - | "custom" - | "github_issues" - | "jira_issues" - | "s3" - | "azure_blob" - | "gcs" - | "webdav" - | "notion" - | "memory" - | "upload"; + kind: SourceKind; name: string; config: { urls?: string[]; @@ -324,6 +311,47 @@ export interface DerivedFact { premises: string[]; } +/** 一条**没有落地**的派生（0017 §3）：推出来了，撞上一条断言，拦在图外。 + * 没有自己的 id，用那条违规的 id 指它——面板、幽灵边、Review 卡片三处靠它对上 */ +export interface BlockedDerivation { + violation_id: string; + subject_id: string; + subject: string; + object_id: string; + object: string; + predicate: string; + rule: string; + via_label: string; + valid_from: string | null; + valid_to: string | null; + against_fact: string; + against_text: string; + premises: string[]; +} + +/** 证明的一步：一条断言前提，带它的证据。前提一律是断言，所以证明是链不是树 */ +export interface ProofStep { + seq: number; + fact_id: string; + subject_id: string; + subject: string; + predicate_id: string | null; + predicate: string | null; + object_id: string | null; + object: string | null; + valid_from: string | null; + valid_to: string | null; + confidence: number; + /** 这条前提后来被撤了；派生随之失效，证明仍要读得出当时靠的是什么 */ + retracted: boolean; + evidence: Evidence[]; +} + +export interface Proof { + derived: DerivedFact; + steps: ProofStep[]; +} + /** 审核页的分档。**与服务端的 queue 参数是同一组字面量**——拼错会拿到 * 一个明确的 unknown_queue 错误，而不是悄悄的空列表。 */ export type ReviewQueue = @@ -450,9 +478,33 @@ export interface MappingRevision { changed_at: string; } /** 一处公理违规（0002 R0）。判据来自本体自己声明的公理，没声明就不报 */ +/** derived_contradiction 独有（0017）：推出来的那条三元组——它没有落库， + * 只能在这里写出来。其它种类是 `{}` */ +export interface ViolationDetail { + axiom?: "functional" | "asymmetry" | "self_loop"; + rule?: "transitive" | "symmetric" | "inverse" | "sub_property"; + via_label?: string; + subject?: string; + predicate?: string; + object?: string; + valid_from?: string | null; + valid_to?: string | null; + premises?: string[]; +} +export type ViolationResolution = + | "fact_retracted" + | "fact_closed" + | "axiom_relaxed" + | "accepted"; export interface AxiomViolation { id: string; - kind: "self_loop" | "asymmetry" | "cycle" | "functional"; + kind: + | "self_loop" + | "asymmetry" + | "cycle" + | "functional" + | "signature" + | "derived_contradiction"; /** 判据来自哪条关系。判「公理写错了」时从这里进本体去改 */ predicate: string | null; left_fact: string; @@ -463,6 +515,10 @@ export interface AxiomViolation { /** 环的长度；其余三类为 0 */ path_len: number; detected_at: string; + detail: ViolationDetail; + /** 审核线索（0017 §2），一次只给一条：旧断言没写结束日期、有同名实体、 + * 抽取置信度低。没有就空 */ + hint: "stale" | "duplicate" | "unsure" | null; } /** 本体自己的一处自相矛盾。**与 AxiomViolation 不是一回事**：那个说 * 「事实与定义抵触」，这个说「定义自己站不住」，后者更根本 */ @@ -477,11 +533,27 @@ export interface OntologyDefect { // 0017 加的三类：都在谓词上，前两类关于逆，第三类是子属性成环 | "inverse_of_itself" | "inverse_not_mutual" - | "sub_property_cycle"; + | "sub_property_cycle" + // 0017：两条规则加在一起产出互斥的派生，按规则对聚合报一次 + | "rules_disagree"; subject_label: string | null; other_label: string | null; path_labels: string[]; detected_at: string; + detail: DefectDetail; +} +/** rules_disagree 独有：哪两条规则、撞在哪条公理上、几对、几个例子 */ +export interface DefectDetail { + count?: number; + rules?: { + rule_a: string; + via_a: string; + rule_b: string; + via_b: string; + axiom: string; + count: number; + examples: [string, string][]; + }[]; } export interface FactReviewItem { id: string; @@ -579,6 +651,11 @@ export interface GraphEdge { valid_from: string | null; valid_to: string | null; confidence: number; + /** 有争议（0017 §3）：有一条 open 的公理违规或时态冲突指着它。整条边画成警戒色 */ + contested: boolean; + /** 幽灵边（0017 §3）：没落地的派生。`id` 是那条 `derived_contradiction` 违规的 id； + * `derived` 同时为 true，跟着派生开关走。点它打开主语的面板 */ + blocked: boolean; } export interface EntityFact { @@ -606,6 +683,13 @@ export interface EntityFact { stale: boolean; /** 修正行：区间闭合来自引擎对账/人工裁决而非抽取原文 */ corrected: boolean; + /** 有争议（0017 §3）：哪一种、Review 里那一项的 id、派生撞断言时推出来的那句话。 + * 行不压暗——断言仍然活着 */ + contested: { + kind: string; + ref_id: string; + derived?: string | null; + } | null; /** 证据集合里最新的文档时间（开放事实的"最后确认时间"） */ last_evidence_time: string | null; } @@ -1294,6 +1378,8 @@ export const api = { /** 推出来的那些**单独一个键**，不掺进 facts：混在同一个列表里， * 用户看不出「文档里写的」和「引擎推的」的区别 */ derived: DerivedFact[]; + /** 没落地的派生（0017 §3）：连 `derived_facts` 都不在，所以也单独一个键 */ + blocked: BlockedDerivation[]; /** 同名的其他实体。**打开面板就给**——合并入口要长在能看见同名的地方， * 而不是藏在「改一次名」之后 */ same_name: GraphNode[]; @@ -1318,6 +1404,17 @@ export const api = { request<{ evidence: Evidence[] }>( `/api/v1/kbs/${kbId}/facts/${factId}/evidence`, ), + /** 一条派生事实的证明（0002 R2）：前提按推导顺序，每条带证据，一路到原句。 + * 派生已失效时回 null——不是错误 */ + derivedProof: (kbId: string, derivedId: string) => + request<{ proof: Proof | null }>( + `/api/v1/kbs/${kbId}/derived/${derivedId}/proof`, + ), + /** 没落地的派生的证明链（0017 §3）：前提在违规的 path 里 */ + blockedProof: (kbId: string, violationId: string) => + request<{ steps: ProofStep[] | null }>( + `/api/v1/kbs/${kbId}/violations/${violationId}/proof`, + ), documentDetail: (id: string) => request<{ document: Doc; chunks: ChunkFull[] }>(`/api/v1/documents/${id}`), extractDocument: (id: string) => @@ -1691,11 +1788,15 @@ export const api = { decideViolation: ( kbId: string, violationId: string, - resolution: "fact_retracted" | "axiom_relaxed" | "accepted", + resolution: ViolationResolution, + closeAt?: string, ) => request<{ ok: boolean }>( `/api/v1/kbs/${kbId}/review/violations/${violationId}`, - { method: "POST", body: JSON.stringify({ resolution }) }, + { + method: "POST", + body: JSON.stringify({ resolution, close_at: closeAt ?? null }), + }, ), confirmFact: (kbId: string, factId: string) => request<{ ok: boolean }>(`/api/v1/kbs/${kbId}/facts/${factId}/confirm`, { diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 613c9ad5..e5fb9403 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -732,6 +732,23 @@ export const en = { derivedHint: "Edges no one asserted — the engine worked them out from axioms your ontology declares. Each one shows the premises it came from.", derivedNoProof: "The premises are gone.", + /* 争议（0017 §3） */ + contestedChip: "disputed", + contestedHint: (kind: string, derived: string | null) => + kind === "derived_contradiction" + ? `A derivation contradicts this assertion${derived ? `: ${derived}` : ""}. Open it under Review.` + : kind === "temporal_conflict" + ? "A newer assertion conflicts with this one in time. Open it under Review." + : "This assertion breaks an axiom the ontology declares. Open it under Review.", + blockedTitle: "Did not land", + blockedHint: + "The engine could draw these edges; an assertion stood in the way. They show on the graph as ghost edges.", + blockedBy: (t: string) => `blocked by ${t}`, + blockedReview: "Review", + /** 证明链（0002 R2）：每一步是一条断言前提，展开到原句 */ + proofStep: (n: number) => `Step ${n}`, + proofRetracted: "since retracted", + proofLoading: "Tracing the proof…", derivedPanel: "Inference", derivedRunAsk: "Re-run inference for the whole base?", derivedRunGo: "Run", @@ -872,7 +889,13 @@ export const en = { "Read-only database connections for asking questions about your data in Chat. " + "Register connections here; each knowledge base mounts the ones it may query.", name: "Name", - connString: "Connection string (postgres://user:pass@host:5432/db)", + connString: "Connection string — the scheme picks the engine", + // 四种写法各一行；令牌放 password 位，Databricks 的路径就是控制台里的 httpPath + connSchemes: + "postgres://user:pass@host:5432/db\n" + + "trino://user[:pass]@host:8080/catalog[/schema] (Iceberg, Delta Lake, Hive)\n" + + "databricks://:TOKEN@host/sql/1.0/warehouses/ID?catalog=main\n" + + "snowflake://:TOKEN@account.snowflakecomputing.com/DB/SCHEMA?warehouse=WH", add: "Add data source", test: "Test", testOk: "Connected", @@ -1299,6 +1322,16 @@ export const en = { defectInverseSelf: "Its own inverse — say symmetric instead", defectInverseNotMutual: "The inverse does not point back", defectSubPropertyCycle: "subPropertyOf runs in a circle", + defectRulesDisagree: "Two rules produce contradicting derivations", + rulesDisagreeCount: (n: number) => + `${n} pair(s) of derivations held back until this is settled`, + rulesDisagreeRule: ( + a: string, + va: string, + b: string, + vb: string, + axiom: string, + ) => `${a} on ${va} with ${b} on ${vb}, against ${axiom}`, defectNeverInstantiable: "no instance can ever satisfy it", defectFixed: "I fixed the ontology", defectAccepted: "Leave it", @@ -1321,6 +1354,24 @@ export const en = { /** 签名违规（#190 / #196）：一条事实的主语或宾语落在谓词声明的类型之外—— * 抽取时会掰正，采纳与合并这两条路从前绕过了检查 */ violationSignature: "Subject or object outside the declared types", + /** 0017：派生撞上断言。卡片是一次审核，线索指向上游的错 */ + violationDerived: "A derivation contradicts an assertion", + derivedLine: (s: string, p: string, o: string) => + `Derived: ${s} · ${p} · ${o}`, + derivedBy: (rule: string, via: string) => `by ${rule} on ${via}`, + assertedLine: (t: string) => `Asserted: ${t}`, + hintStale: + "The assertion has no end date and the derivation starts later. It may simply have ended.", + hintDuplicate: + "Two entities share this name. They may be the same one.", + hintUnsure: + "The assertion was extracted with low confidence. Read its sentence.", + hintReadBoth: "Read both sentences and decide which one is wrong.", + closeAssertion: "Give the assertion an end date", + retractAssertion: "Retract the assertion", + seeDuplicates: "See duplicates", + openOntology: "Open the ontology", + letBothStand: "Let both stand", violationVia: (p: string) => `via ${p}`, violationPath: (n: number) => `${n} facts in the cycle`, retractFact: "Data is wrong", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index 14c01d8e..73a027d6 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -674,6 +674,20 @@ export const zh: Strings = { derivedHint: "没有人断言过的边——引擎按本体声明的公理推出来的。每一条都附着它用到的前提。", derivedNoProof: "前提已经不在了。", + contestedChip: "有争议", + contestedHint: (kind: string, derived: string | null) => + kind === "derived_contradiction" + ? `有一条推出来的事实与它抵触${derived ? `：${derived}` : ""}。去 Review 看。` + : kind === "temporal_conflict" + ? "有一条更新的断言在时间上与它冲突。去 Review 看。" + : "这条断言违反了本体声明的公理。去 Review 看。", + blockedTitle: "没落地的", + blockedHint: "引擎本可以画出这些边，被一条断言挡住了。图上画成幽灵边。", + blockedBy: (t: string) => `挡住它的：${t}`, + blockedReview: "去 Review", + proofStep: (n: number) => `第 ${n} 步`, + proofRetracted: "后来撤了", + proofLoading: "正在展开证明…", derivedPanel: "推理", derivedRunAsk: "对整个库重跑一遍推理？", derivedRunGo: "跑", @@ -797,7 +811,12 @@ export const zh: Strings = { "只读的数据库连接，用于在「对话」里就你的数据提问。" + "在这里登记连接；每个知识库各自挂载允许查询的那些。", name: "名称", - connString: "连接串（postgres://user:pass@host:5432/db）", + connString: "连接串，前缀决定引擎", + connSchemes: + "postgres://user:pass@host:5432/db\n" + + "trino://user[:pass]@host:8080/catalog[/schema] （Iceberg、Delta Lake、Hive）\n" + + "databricks://:TOKEN@host/sql/1.0/warehouses/ID?catalog=main\n" + + "snowflake://:TOKEN@account.snowflakecomputing.com/DB/SCHEMA?warehouse=WH", add: "添加数据源", test: "测试", testOk: "已连接", @@ -1174,6 +1193,15 @@ export const zh: Strings = { defectInverseSelf: "自己是自己的逆——写成「对称」更直白", defectInverseNotMutual: "逆关系没有指回来", defectSubPropertyCycle: "子属性绕成了环", + defectRulesDisagree: "两条规则推出互相抵触的结论", + rulesDisagreeCount: (n: number) => `${n} 对派生等这里定了再落地`, + rulesDisagreeRule: ( + a: string, + va: string, + b: string, + vb: string, + axiom: string, + ) => `${va} 上的 ${a} 与 ${vb} 上的 ${b}，撞在 ${axiom} 上`, defectNeverInstantiable: "这个类永远不可能有实例", defectFixed: "已去本体里改了", defectAccepted: "先放着", @@ -1192,6 +1220,20 @@ export const zh: Strings = { violationCycle: "传递链绕成了环", violationFunctional: "该只有一个值，却有两个", violationSignature: "主语或宾语不在关系声明的类型里", + violationDerived: "推出来的与断言相抵触", + derivedLine: (s: string, p: string, o: string) => + `推出：${s} · ${p} · ${o}`, + derivedBy: (rule: string, via: string) => `由 ${via} 上的 ${rule}`, + assertedLine: (t: string) => `断言：${t}`, + hintStale: "这条断言没写结束日期，而推出来的那条起得更晚——它可能只是结束了。", + hintDuplicate: "有两个同名实体，它们可能是同一个。", + hintUnsure: "这条断言抽取时把握不大，去读一下原句。", + hintReadBoth: "读一下两边的原句，判断哪条错了。", + closeAssertion: "给断言一个结束日期", + retractAssertion: "撤掉断言", + seeDuplicates: "去看重复实体", + openOntology: "去本体页", + letBothStand: "两边都成立", violationVia: (p: string) => `依据 ${p}`, violationPath: (n: number) => `环上 ${n} 条事实`, retractFact: "数据错了", diff --git a/web/src/pages/Graph.tsx b/web/src/pages/Graph.tsx index 1ec96c23..e69a0cea 100644 --- a/web/src/pages/Graph.tsx +++ b/web/src/pages/Graph.tsx @@ -1,4 +1,11 @@ -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + type MutableRefObject, +} from "react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { Link, useNavigate, useSearch } from "@tanstack/react-router"; import Graphology from "graphology"; @@ -34,6 +41,8 @@ import { type Evidence, type GraphEdge, type GraphNode, + type BlockedDerivation, + type ProofStep, } from "../api"; import { S } from "../i18n"; import { usePopoverFlip } from "../ui/popoverFlip"; @@ -69,6 +78,10 @@ const EDGE_COLOR_INFERRED = "rgba(163,163,163,0.1)"; // 用户要在余光里就分得出「文档里写的」和「推出来的」 const EDGE_COLOR_DERIVED = "rgba(231,197,124,0.42)"; const EDGE_COLOR_DERIVED_DIM = "rgba(231,197,124,0.14)"; +/* 争议（0017 §3）：珊瑚橙 `--u-contest`。金是派生、琥珀是警告、粉是危险， + 它得跟三个都拉开。**整条边换色**——环在节点上、边还是灰的，余光分不出来 */ +const EDGE_COLOR_CONTEST = "rgba(255,106,61,0.55)"; +const EDGE_FOCUS_CONTEST = "rgba(255,106,61,1)"; /** 相邻两条弧之间的曲率差。太小仍然糊，太大在长边上会甩得离节点很远 */ const EDGE_CURVATURE_STEP = 0.18; @@ -187,6 +200,10 @@ const NODE_BUDGETS: number[] = [150, 300, 600, 1000]; // 注意：sigma 边着色器在预乘混合(ONE, ONE_MINUS_SRC_ALPHA)下不预乘 RGB， // alpha 无法压暗边——暗度必须编码进 RGB（不透明近背景色） const EDGE_DIM = "#141414"; +/* 幽灵边：没落地的派生。同一个色相往 EDGE_DIM 混（预乘混合下 alpha 压不暗边）， + 更细。sigma 默认的边程序画不了虚线，也不为此另写一个 */ +const EDGE_GHOST = lerpColor("rgba(255,106,61,1)", EDGE_DIM, 0.55); +const EDGE_GHOST_FOCUS = lerpColor("rgba(255,106,61,1)", EDGE_DIM, 0.2); const EDGE_FOCUS = "rgba(255,255,255,0.55)"; // 选中/悬停时的派生边。**不能跟着走白**：选中恰恰是看得最仔细的时候， // 而这时候「这条边是推出来的、没人写过」比任何时候都该说清楚。 @@ -439,6 +456,8 @@ export function Graph() { 先留在原地演完退场，再真的移除。用 selectedRef 取当前值而不是把 setState 写成带副作用的 updater：那种写法在 StrictMode 下会跑两遍 */ const [exiting, setExiting] = useState(null); + // 打开面板时想停在哪一档、展开哪一行——幽灵边点进来时用 + const panelIntentRef = useRef<{ view: "derived"; open: string } | null>(null); const deselect = useCallback(() => { const cur = selectedRef.current; if (!cur) return; @@ -843,13 +862,20 @@ export function Graph() { ); for (const { edge: e, curvature, alsoLabels } of placed.edges) { g.addEdgeWithKey(e.id, e.source, e.target, { - label: e.label?.toUpperCase() ?? "", - size: 1, - color: e.derived - ? EDGE_COLOR_DERIVED - : e.inferred - ? EDGE_COLOR_INFERRED - : EDGE_COLOR, + // 争议的边标签前置 ⚠：颜色之外再给一个不靠色觉的记号 + label: (e.contested ? "⚠ " : "") + (e.label?.toUpperCase() ?? ""), + size: e.blocked ? 0.7 : 1, + color: e.blocked + ? EDGE_GHOST + : e.contested + ? EDGE_COLOR_CONTEST + : e.derived + ? EDGE_COLOR_DERIVED + : e.inferred + ? EDGE_COLOR_INFERRED + : EDGE_COLOR, + contested: e.contested, + blocked: e.blocked, // 独一条就走直线：曲线是为了把重叠分开，没有重叠就不必弯 type: curvature === 0 ? "line" : "curved", curvature, @@ -1170,16 +1196,24 @@ export function Graph() { : null; const hovNow = hoverRef.current; const focused = selNow ?? hovNow; + const ghost = attrs.blocked === true; const from = !focused - ? EDGE_COLOR_DERIVED + ? ghost + ? EDGE_GHOST + : EDGE_COLOR_DERIVED : s === focused || t === focused - ? EDGE_FOCUS_DERIVED + ? ghost + ? EDGE_GHOST_FOCUS + : EDGE_FOCUS_DERIVED : EDGE_DIM; res.color = lerpColor(from, EDGE_DIM, k); res.label = ""; return res; } - const pulse = lerpColor( + // 幽灵边不呼吸：它不是知识，是一条没走通的路 + const pulse = attrs.blocked === true + ? EDGE_GHOST + : lerpColor( EDGE_COLOR_DERIVED_DIM, EDGE_COLOR_DERIVED, // 三角波而不是正弦：两端各停一瞬，看起来是「呼吸」不是「闪」 @@ -1199,7 +1233,14 @@ export function Graph() { ? selectedRef.current : null; const boost = () => { - res.color = isDerived ? EDGE_FOCUS_DERIVED : EDGE_FOCUS; + res.color = + attrs.blocked === true + ? EDGE_GHOST_FOCUS + : attrs.contested === true + ? EDGE_FOCUS_CONTEST + : isDerived + ? EDGE_FOCUS_DERIVED + : EDGE_FOCUS; res.size = Math.max((attrs.size as number) * 1.42, 1.85); res.zIndex = 5; }; @@ -1245,6 +1286,13 @@ export function Graph() { }, }); sigma.on("clickNode", ({ node }) => setSelected(node)); + // 幽灵边点一下：打开它主语的面板，停在「推出来的」那一档、展开那一行（0017 §3） + sigma.on("clickEdge", ({ edge }) => { + if (g.getEdgeAttribute(edge, "blocked") !== true) return; + const [s] = g.extremities(edge); + panelIntentRef.current = { view: "derived", open: edge }; + setSelected(s); + }); sigma.on("doubleClickNode", ({ node, event }) => { event.preventSigmaDefault(); setFocusEntity(node); @@ -1840,6 +1888,7 @@ export function Graph() { kbId={kb.id} entityId={(selected ?? exiting)!} exiting={!selected} + intent={panelIntentRef} onClose={deselect} onNavigate={(id) => { // 跳转目标可能不在当前画布：同时把图 refocus 到它的邻域（与搜索选择一致） @@ -2443,6 +2492,7 @@ function DerivedPanel({ * Relations/Timeline/History 的紧凑行里显得是另一个产品的东西，而且十几条 * 推导堆起来是一面墙。证明是「问了才看」的东西，收进展开区正合适。 */ function DerivedRow({ + kbId, d, otherId, otherName, @@ -2450,6 +2500,7 @@ function DerivedRow({ onToggle, onNavigate, }: { + kbId: string; d: DerivedFact; otherId: string; otherName: string; @@ -2490,21 +2541,188 @@ function DerivedRow({ {d.premises.length} - {/* 证明：前提按推导顺序。**边框与 EvidenceList 同一档**—— - 两者是同一件事的两种形态：一个给出处，一个给推理链 */} + {/* 证明：前提按推导顺序，每条展开到原句（0002 R2）。**边框与 EvidenceList + 同一档**——两者是同一件事的两种形态：一个给出处，一个给推理链 */} + {open && } + + ); +} + +/** 一条派生的证明链。展开时才取——证明是「问了才看」的东西。 + * + * 每一步是一条断言前提：三元组在上，它的原句在下，原句可点进文档。 + * 前提被撤过的打标记但不藏：派生随之失效，而「当时靠的是什么」正是记录轴要答的。 + * 取不到（派生已失效）就退回列表里带来的那几行文本，不空着。 */ +function ProofChain({ kbId, d }: { kbId: string; d: DerivedFact }) { + const proof = useQuery({ + queryKey: ["proof", d.id], + queryFn: () => api.derivedProof(kbId, d.id), + }); + const steps = proof.data?.proof?.steps; + return ( +
+ {proof.isPending && ( +

{S.graph.proofLoading}

+ )} + {steps && } + {/* 派生已失效、证明取不到：退回列表里带来的那几行文本 */} + {!proof.isPending && !steps && ( +
    + {d.premises.map((p, i) => ( +
  1. + {p} +
    2. + ))} + {d.premises.length === 0 && ( +
  2. {S.graph.derivedNoProof}
    3. + )} +
+ )} +
+ ); +} + +/** 证明的步，落了地的与没落地的派生共用：前提是同一种东西 */ +function ProofSteps({ kbId, steps }: { kbId: string; steps: ProofStep[] }) { + return ( +
    + {steps.map((st) => ( +
  1. +
    + + {S.graph.proofStep(st.seq + 1)} + + + {st.subject} + — {st.predicate ?? "?"} → + {st.object ?? "?"} + + {st.retracted && ( + {S.graph.proofRetracted} + )} +
    +
    + {st.evidence.map((ev) => ( + +
    + {ev.quote ? `“${ev.quote}”` : S.graph.noQuote} +
    +
    + {S.graph.sectionRef(ev.filename, ev.seq + 1)} + {ev.stale && ( + + {S.graph.fromVersion(ev.doc_version)} + + )} +
    + + ))} + {st.evidence.length === 0 && ( +

    {S.graph.noEvidence}

    + )} +
    +
    2. + ))} +
+ ); +} + +/** 没落地的派生（0017 §3）：像 DerivedRow 一样的一行，多一句「挡住它的是谁」， + * 展开是它的证明链——人在这里看到「引擎本可以画这条边，是什么拦住了它」 */ +function BlockedRow({ + kbId, + b, + entityId, + open, + onToggle, + onNavigate, +}: { + kbId: string; + b: BlockedDerivation; + entityId: string; + open: boolean; + onToggle: () => void; + onNavigate: (entityId: string) => void; +}) { + const navigate = useNavigate(); + const out = b.subject_id === entityId; + const otherId = out ? b.object_id : b.subject_id; + const otherName = out ? b.object : b.subject; + const proof = useQuery({ + queryKey: ["blocked-proof", b.violation_id], + queryFn: () => api.blockedProof(kbId, b.violation_id), + enabled: open, + }); + return ( +
+ +
+ + {S.graph.blockedBy(b.against_text)} + + + navigate({ + to: "/kb/$kbId/review", + params: { kbId }, + search: { queue: "violations", item: b.violation_id }, + }) + } + className="ml-auto shrink-0 text-neutral-500 hover:text-neutral-300 hover:underline underline-offset-2" + > + {S.graph.blockedReview} → + +
{open && (
-
    - {d.premises.map((p, i) => ( -
  1. - {p} -
    2. - ))} -
- {d.premises.length === 0 && ( -

- {S.graph.derivedNoProof} -

+ {proof.isPending && ( +

{S.graph.proofLoading}

+ )} + {proof.data?.steps && ( + )}
)} @@ -2512,10 +2730,52 @@ function DerivedRow({ ); } +/** 争议 chip（0017 §3）：有一条 open 的违规或冲突指着这条断言。行不压暗—— + * 它仍然活着。点它去 Review 对应那一档，并把那张卡点亮 */ +function ContestedChip({ + kbId, + c, +}: { + kbId: string; + c: NonNullable; +}) { + const navigate = useNavigate(); + const queue = c.kind === "temporal_conflict" ? "conflicts" : "violations"; + return ( + { + ev.stopPropagation(); + navigate({ + to: "/kb/$kbId/review", + params: { kbId }, + search: { queue, item: c.ref_id }, + }); + }} + onKeyDown={(ev) => { + if (ev.key === "Enter") { + ev.stopPropagation(); + navigate({ + to: "/kb/$kbId/review", + params: { kbId }, + search: { queue, item: c.ref_id }, + }); + } + }} + className="u-chip u-chip-contest shrink-0 !text-[10px] !px-1.5 cursor-pointer" + title={S.graph.contestedHint(c.kind, c.derived ?? null)} + > + {S.graph.contestedChip} + + ); +} + function EntityPanel({ kbId, entityId, exiting, + intent, onClose, onNavigate, }: { @@ -2523,6 +2783,8 @@ function EntityPanel({ entityId: string; /** 正在演退场：还挂在 DOM 上，但已经不接受点击 */ exiting: boolean; + /** 打开时停在哪一档、展开哪一行；读一次就清掉 */ + intent?: MutableRefObject<{ view: "derived"; open: string } | null>; onClose: () => void; onNavigate: (entityId: string) => void; }) { @@ -2534,6 +2796,8 @@ function EntityPanel({ // 推出来的那些。**单独一个键，不掺进 facts**——混在一个列表里，用户看不出 // 「文档里写的」和「引擎推的」的区别 const derived = detail.data?.derived ?? []; + // 没落地的（0017 §3）：推出来了，撞上一条断言 + const blocked = detail.data?.blocked ?? []; /* 按「方向 + 谓词 + 规则」分组，骨架与 Relations 的 groups 一致。 规则挂在组上而不是每一行：它对整组都成立，逐行重复既冗余， 那个琥珀色小字还会跟派生边抢色相 */ @@ -2565,6 +2829,13 @@ function EntityPanel({ const [view, setView] = useState< "relations" | "timeline" | "history" | "derived" >("relations"); + useEffect(() => { + const it = intent?.current; + if (!it) return; + intent.current = null; + setView(it.view); + setOpenFact(it.open); + }, [entityId, intent]); const e: GraphNode | undefined = detail.data?.entity; @@ -2849,8 +3120,10 @@ function EntityPanel({
{(["relations", "timeline", "history", "derived"] as const) // 推出来的那一档：**没有派生就不出现**。一个没开推理的库不该看到 - // 一个永远是空的标签页 - .filter((v) => v !== "derived" || derived.length > 0) + // 一个永远是空的标签页。没落地的也算——那正是这一档要说的事 + .filter( + (v) => v !== "derived" || derived.length > 0 || blocked.length > 0, + ) .map((v) => (
))} + {blocked.length > 0 && ( +
+
+ {S.graph.blockedTitle} + + {blocked.length} + +
+

+ {S.graph.blockedHint} +

+ {blocked.map((b) => ( + + setOpenFact( + openFact === b.violation_id ? null : b.violation_id, + ) + } + onNavigate={onNavigate} + /> + ))} +
+ )} )} {view !== "history" && @@ -3141,6 +3443,9 @@ function TimelineRow({ {S.graph.staleFactChip} )} + {fact.contested && ( + + )}
{open && } @@ -3225,6 +3530,7 @@ function FactRow({ {S.graph.staleFactChip} )} + {fact.contested && } {interval && ( {interval} diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsx index e5764ca8..5a379737 100644 --- a/web/src/pages/Library.tsx +++ b/web/src/pages/Library.tsx @@ -14,6 +14,10 @@ import { } from "lucide-react"; import { api, type Doc, type ExtractionDrop, type SourceView } from "../api"; import { S } from "../i18n"; +import { + CREATABLE_SOURCE_KINDS, + type CreatableSourceKind, +} from "../sourceKinds"; import { useKb, useKbId } from "../kb"; import { toast } from "../toast"; import { Chip, type ChipTone, DangerConfirm, Loading, Pager } from "../ui"; @@ -1267,20 +1271,7 @@ function SourceModal({ /** isApi=true 时父级紧接着打开密钥弹窗 */ onDone: (id?: string, isApi?: boolean) => void; }) { - const [kind, setKind] = useState< - | "folder" - | "url" - | "rss" - | "custom" - | "api" - | "github_issues" - | "jira_issues" - | "s3" - | "azure_blob" - | "gcs" - | "webdav" - | "notion" - >("folder"); + const [kind, setKind] = useState("folder"); const [name, setName] = useState(""); const [icon, setIcon] = useState(null); const [urls, setUrls] = useState(""); @@ -1447,22 +1438,7 @@ function SourceModal({
{/* 类型 */}
- {( - [ - "folder", - "url", - "rss", - "github_issues", - "jira_issues", - "s3", - "azure_blob", - "gcs", - "webdav", - "notion", - "api", - "custom", - ] as const - ).map((k) => { + {CREATABLE_SOURCE_KINDS.map((k) => { const Icon = KIND_ICON[k]; return ( + + + + +
+
+ ); +} + /* ---------- 页面：左栏分类 + 单类内容区 ---------- */ type Sel = @@ -701,7 +830,11 @@ export function Review() { const { kb } = useKb(); const queryClient = useQueryClient(); const navigate = useNavigate(); - const [sel, setSel] = useState(null); + // 面板的争议 chip 带着 queue / item 跳过来：先落到那一档，再把那张卡点亮 + const search = useSearch({ from: "/app/kb/$kbId/review" }); + const [sel, setSel] = useState( + QUEUE_ORDER.includes(search.queue as Sel) ? (search.queue as Sel) : null, + ); const [page, setPage] = useState(0); // 队列变化经 SSE 事件流推送（useKbEvents 挂在 Shell），无需轮询。 @@ -788,10 +921,12 @@ export function Review() { mutationFn: ({ id, resolution, + closeAt, }: { id: string; - resolution: "fact_retracted" | "axiom_relaxed" | "accepted"; - }) => api.decideViolation(kb!.id, id, resolution), + resolution: ViolationResolution; + closeAt?: string; + }) => api.decideViolation(kb!.id, id, resolution, closeAt), onSettled: invalidate, }); // 检查是同步的纯计算,所以直接 mutate 不排队。跑完把报告留在按钮旁边—— @@ -1170,17 +1305,27 @@ export function Review() { )} {asViolations().map((v) => ( - + - violationAction.mutate({ id: v.id, resolution }) + onDecide={(resolution, closeAt) => + violationAction.mutate({ id: v.id, resolution, closeAt }) } + onDuplicates={() => select("duplicates")} + onOntology={() => navigate({ to: "/ontology" })} /> + ))} )} diff --git a/web/src/pages/Settings.tsx b/web/src/pages/Settings.tsx index 4a4aa9c0..f61040aa 100644 --- a/web/src/pages/Settings.tsx +++ b/web/src/pages/Settings.tsx @@ -639,7 +639,12 @@ function DataSourcesAdmin() {
-
{d.name}
+
+ {d.name} + + {d.engine} + +
{d.summary}
@@ -695,6 +700,9 @@ function DataSourcesAdmin() { value={conn} onChange={(e) => setConn(e.target.value)} /> +

+ {S.settings.datasources.connSchemes} +