From 039dfed0ded6dd06e05c1a28ac3507b801fd9310 Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Thu, 3 Sep 2026 14:04:00 +0800 Subject: [PATCH 1/2] Source credentials stay on the server Co-Authored-By: Claude Fable 5.1 --- crates/utopia-core/src/models.rs | 28 ++++ .../utopia-server/src/api/sources_routes.rs | 141 ++++++++++++++---- crates/utopia-store/src/sources.rs | 9 +- .../tests/a_viewer_never_sees_a_credential.rs | 139 +++++++++++++++++ ...3-a-source-should-hand-over-its-history.md | 4 + 5 files changed, 291 insertions(+), 30 deletions(-) create mode 100644 crates/utopia-store/tests/a_viewer_never_sees_a_credential.rs diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index 091a5cf8..4085d6ae 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -137,6 +137,34 @@ pub struct Source { pub created_at: DateTime, } +/// 来源配置里**用来鉴权**的那几个键。凭据只进不出：列表与创建 / 更新的响应都剔掉， +/// 更新时客户端没传或传空串就保留库里的原值，审计里也不落。 +/// +/// **一张表，四处共用。** 此前那条规矩只对 `auth_header` 一个键成立，而对象存储、 +/// WebDAV、Notion 各自的密钥原样发给了每一个 Viewer（#246）。加连接器时**先加这里**， +/// 再写读它的代码。`username` / `account_name` / `access_key_id` 这类是身份标识， +/// 单独拿到鉴不了权，留着让界面显示得出「这是哪个账号」。 +pub const SOURCE_SECRET_KEYS: &[&str] = &[ + "auth_header", + "token", + "password", + "secret_access_key", + "account_key", + "service_account_key", +]; + +impl Source { + /// 剔掉凭据后的这条来源——任何要回给客户端的 `Source` 都从这里过 + pub fn without_secrets(mut self) -> Self { + if let Some(obj) = self.config.as_object_mut() { + for key in SOURCE_SECRET_KEYS { + obj.remove(*key); + } + } + self + } +} + /// 来源同步运行记录（渠道审计历史）。 #[derive(Debug, Clone, Serialize, sqlx::FromRow)] pub struct SyncRun { diff --git a/crates/utopia-server/src/api/sources_routes.rs b/crates/utopia-server/src/api/sources_routes.rs index baaae99f..3889a69e 100644 --- a/crates/utopia-server/src/api/sources_routes.rs +++ b/crates/utopia-server/src/api/sources_routes.rs @@ -6,7 +6,7 @@ use axum::Json; use chrono::{DateTime, Utc}; use serde::Deserialize; use serde_json::json; -use utopia_core::models::Role; +use utopia_core::models::{Role, SOURCE_SECRET_KEYS}; use uuid::Uuid; use super::graph_routes::require_kb; @@ -19,6 +19,23 @@ fn new_ingest_token() -> String { format!("utp_{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()) } +/// 取一条来源，并确认它属于路径上的这个库。 +/// +/// `require_kb` 只查人对库的权限；来源 id 是另一个维度——不比对的话，A 库的 +/// Editor 拿着 B 库来源的 id 就能同步、清理、删除它。不属于就当不存在（404）， +/// 与 `get_token` 一直以来的做法一致 +async fn source_in_kb( + state: &AppState, + kb_id: Uuid, + source_id: Uuid, +) -> ApiResult { + let source = utopia_store::sources::get(&state.pool, source_id).await?; + if source.kb_id != kb_id { + return Err(utopia_core::AppError::NotFound.into()); + } + Ok(source) +} + pub async fn list( State(state): State, AuthUser(user): AuthUser, @@ -102,8 +119,8 @@ pub async fn get_token( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; - let source = utopia_store::sources::get(&state.pool, source_id).await?; - if source.kb_id != kb_id || source.kind != "api" { + let source = source_in_kb(&state, kb_id, source_id).await?; + if source.kind != "api" { return Err(utopia_core::AppError::NotFound.into()); } Ok(Json(json!({ "ingest_token": source.ingest_token }))) @@ -116,8 +133,8 @@ pub async fn rotate_token( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; - let source = utopia_store::sources::get(&state.pool, source_id).await?; - if source.kb_id != kb_id || source.kind != "api" { + let source = source_in_kb(&state, kb_id, source_id).await?; + if source.kind != "api" { return Err(utopia_core::AppError::NotFound.into()); } let token = new_ingest_token(); @@ -125,12 +142,34 @@ pub async fn rotate_token( Ok(Json(json!({ "ingest_token": token }))) } -/// 响应前剔除凭据（auth_header 只进不出）。 -fn mask_secrets(mut source: utopia_core::models::Source) -> utopia_core::models::Source { - if let Some(obj) = source.config.as_object_mut() { - obj.remove("auth_header"); +/// 响应前剔除凭据（只进不出；键见 `SOURCE_SECRET_KEYS`）。 +fn mask_secrets(source: utopia_core::models::Source) -> utopia_core::models::Source { + source.without_secrets() +} + +/// 更新时凭据的合并规则，每个 `SOURCE_SECRET_KEYS` 里的键一样：新配置里**没有**这个键 +/// 或值是空串 → 保留库里的原值（表单留空就是「别动」）；显式 `null` → 删掉； +/// 其余照新值。响应从不回显，所以客户端没有办法把旧值原样送回来，规则只能长在这里 +fn keep_secrets(next: &mut serde_json::Value, existing: &serde_json::Value) { + let Some(obj) = next.as_object_mut() else { + return; + }; + for key in SOURCE_SECRET_KEYS { + let keep = match obj.get(*key) { + None => true, + Some(serde_json::Value::Null) => { + obj.remove(*key); + false + } + Some(v) => v.as_str().is_some_and(|s| s.trim().is_empty()), + }; + if keep { + obj.remove(*key); + if let Some(prev) = existing.get(*key) { + obj.insert((*key).to_string(), prev.clone()); + } + } } - source } #[derive(Deserialize)] @@ -158,23 +197,11 @@ pub async fn update( Json(body): Json, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; - // 凭据只进不出：响应从不回显 auth_header，表单留空 = 保留库里原值 + let existing = source_in_kb(&state, kb_id, source_id).await?; + // 凭据只进不出：响应从不回显，表单留空 / 没传 = 保留库里原值 let mut config = body.config; if let Some(cfg) = config.as_mut() { - let blank = cfg - .get("auth_header") - .and_then(|v| v.as_str()) - .map(str::trim) - .is_none_or(|s| s.is_empty()); - if blank { - if let Some(obj) = cfg.as_object_mut() { - obj.remove("auth_header"); - let existing = utopia_store::sources::get(&state.pool, source_id).await?; - if let Some(prev) = existing.config.get("auth_header").and_then(|v| v.as_str()) { - obj.insert("auth_header".into(), json!(prev)); - } - } - } + keep_secrets(cfg, &existing.config); } let source = utopia_store::sources::update( &state.pool, @@ -187,7 +214,7 @@ pub async fn update( ) .await?; state.emit_source(kb_id); - // 审计不落凭据：config 只记除 auth_header 外的键 + // 审计不落凭据：config 只记「改没改」 let _ = utopia_store::audit::record( &state.pool, Some(kb_id), @@ -208,6 +235,7 @@ pub async fn cleanup_missing( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; + source_in_kb(&state, kb_id, source_id).await?; let ids = utopia_store::documents::list_missing(&state.pool, source_id).await?; for id in &ids { utopia_store::documents::delete(&state.pool, *id).await?; @@ -229,7 +257,7 @@ pub async fn delete( ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; // Memory 来源常驻：记忆空间不因来源整理而蒸发（记忆文档本身可在 Library 删除） - let source = utopia_store::sources::get(&state.pool, source_id).await?; + let source = source_in_kb(&state, kb_id, source_id).await?; if source.kind == utopia_store::memory::MEMORY_SOURCE_KIND { return Err(utopia_core::AppError::invalid( "memory_source_permanent", @@ -259,6 +287,7 @@ pub async fn runs( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Viewer).await?; + source_in_kb(&state, kb_id, source_id).await?; let runs = utopia_store::sources::list_runs(&state.pool, source_id, 20).await?; Ok(Json(json!({ "runs": runs }))) } @@ -269,6 +298,7 @@ pub async fn sync_now( Path((kb_id, source_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; + source_in_kb(&state, kb_id, source_id).await?; let queued = utopia_store::sources::mark_queued(&state.pool, source_id).await?; if queued { utopia_store::jobs::enqueue( @@ -505,3 +535,60 @@ pub async fn re_extract( .await; Ok(Json(json!({ "queued": ids.len() }))) } + +#[cfg(test)] +mod tests { + use super::keep_secrets; + use serde_json::json; + + #[test] + fn a_blank_or_missing_secret_keeps_the_stored_one() { + let existing = json!({ "bucket": "old", "secret_access_key": "s", "password": "p" }); + // 没传 → 留；空串 → 留；有值 → 换；null → 删 + let mut next = json!({ "bucket": "new", "password": " ", "token": null }); + keep_secrets(&mut next, &existing); + assert_eq!(next["bucket"], "new"); + assert_eq!( + next["secret_access_key"], "s", + "missing keeps the stored value" + ); + assert_eq!(next["password"], "p", "blank keeps the stored value"); + assert!(next.get("token").is_none(), "an explicit null removes it"); + let mut next = json!({ "secret_access_key": "fresh" }); + keep_secrets(&mut next, &existing); + assert_eq!(next["secret_access_key"], "fresh"); + assert_eq!(next["password"], "p"); + } + + #[test] + fn no_secret_reaches_a_response() { + let source = utopia_core::models::Source { + id: uuid::Uuid::nil(), + kb_id: uuid::Uuid::nil(), + kind: "s3".into(), + name: "s".into(), + config: json!({ "bucket": "b", "access_key_id": "AKIA", "secret_access_key": "x", + "account_key": "y", "service_account_key": "z", "password": "w", + "token": "t", "auth_header": "h" }), + icon: None, + sync_interval_minutes: None, + sync_cron: None, + last_sync_at: None, + last_sync_status: "never".into(), + last_sync_error: None, + last_sync_added: 0, + ingest_token: Some("utp_x".into()), + created_at: chrono::Utc::now(), + }; + let masked = super::mask_secrets(source); + let obj = masked.config.as_object().unwrap(); + for key in utopia_core::models::SOURCE_SECRET_KEYS { + assert!(!obj.contains_key(*key), "{key} leaked"); + } + assert_eq!(obj["bucket"], "b"); + assert_eq!( + obj["access_key_id"], "AKIA", + "an identifier is not a secret" + ); + } +} diff --git a/crates/utopia-store/src/sources.rs b/crates/utopia-store/src/sources.rs index f0ca22f7..e4d6f381 100644 --- a/crates/utopia-store/src/sources.rs +++ b/crates/utopia-store/src/sources.rs @@ -2,7 +2,7 @@ use chrono::{DateTime, Utc}; use sqlx::PgPool; -use utopia_core::models::{Role, Source, SourceView, SyncRun}; +use utopia_core::models::{Role, Source, SourceView, SyncRun, SOURCE_SECRET_KEYS}; use utopia_core::{AppError, AppResult}; use uuid::Uuid; @@ -55,9 +55,11 @@ fn cron_next_after(expr: &str, after: DateTime) -> Option> { } pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult> { - // config 剔除 auth_header：自定义拉取器的凭据不下发给任何客户端 + // config 剔掉凭据：列表给 Viewer 看，哪一种连接器的密钥都不下发。 + // 键在 `SOURCE_SECRET_KEYS` 一张表上——从前这里只减 `auth_header`，五种连接器 + // 的密钥就这么漏出去的（#246） let rows: Vec = sqlx::query_as( - "SELECT s.id, s.kind, s.name, s.config - 'auth_header' AS config, s.icon, + "SELECT s.id, s.kind, s.name, s.config - $2::text[] AS config, s.icon, s.sync_interval_minutes, s.sync_cron, s.last_sync_at, s.last_sync_status, s.last_sync_error, s.last_sync_added, (SELECT count(*) FROM documents d WHERE d.source_id = s.id) AS doc_count, @@ -66,6 +68,7 @@ pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult> { FROM sources s WHERE s.kb_id = $1 ORDER BY s.created_at", ) .bind(kb_id) + .bind(SOURCE_SECRET_KEYS) .fetch_all(pool) .await?; Ok(rows) diff --git a/crates/utopia-store/tests/a_viewer_never_sees_a_credential.rs b/crates/utopia-store/tests/a_viewer_never_sees_a_credential.rs new file mode 100644 index 00000000..5efcab44 --- /dev/null +++ b/crates/utopia-store/tests/a_viewer_never_sees_a_credential.rs @@ -0,0 +1,139 @@ +//! #246：来源列表不带任何凭据。 +//! +//! 列表接口给 Viewer 看，此前只剔了 `auth_header`；对象存储、WebDAV、Notion 各自的 +//! 密钥原样下发。现在凭据键列在 `SOURCE_SECRET_KEYS` 一张表上，列表 SQL 按表剔。 +//! 这里守两件事： +//! +//! 1. **列表里一个凭据键都没有**，每一种连接器都试一遍。 +//! 2. **身份标识留着**（bucket、username、account_name），界面要显示得出「这是哪个账号」； +//! 而同步那条路（`sources::get`）拿到的仍是完整配置——凭据只是不出去，不是没了。 +//! +//! 直接插表而不走 `sources::create`：`KINDS` 少了五种（#247），那是另一个修复。 +//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆，绝不碰已有的库。 + +use sqlx::PgPool; +use utopia_core::models::SOURCE_SECRET_KEYS; +use utopia_store::sources; +use uuid::Uuid; + +async fn seed(pool: &PgPool) -> anyhow::Result<(Uuid, Uuid)> { + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'secret-test')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'secret-test')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'secret-test')", + ) + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + Ok((org, kb)) +} + +#[tokio::test] +async fn a_viewer_never_sees_a_credential() -> anyhow::Result<()> { + let Ok(url) = std::env::var("UTOPIA_DATABASE_URL") else { + eprintln!("跳过：未设 UTOPIA_DATABASE_URL"); + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let (org, kb) = seed(&pool).await?; + + let run = async { + // 每种连接器一条，配置按各自界面真正会写的键 + let fixtures: Vec<(&str, serde_json::Value)> = vec![ + ( + "custom", + serde_json::json!({ "endpoint": "https://x.test/items", "auth_header": "Bearer c" }), + ), + ( + "github_issues", + serde_json::json!({ "repo": "o/r", "auth_header": "Bearer g" }), + ), + ( + "jira_issues", + serde_json::json!({ "base_url": "https://j.test", "project": "P", "auth_header": "Basic j" }), + ), + ( + "s3", + serde_json::json!({ "bucket": "b", "region": "r", "access_key_id": "AKIA", + "secret_access_key": "s3-secret" }), + ), + ( + "azure_blob", + serde_json::json!({ "bucket": "c", "account_name": "acct", "account_key": "az-key" }), + ), + ( + "gcs", + serde_json::json!({ "bucket": "g", "service_account_key": "{\"private_key\":\"x\"}" }), + ), + ( + "webdav", + serde_json::json!({ "base_url": "https://d.test", "path": "/", "username": "u", + "password": "dav-pass" }), + ), + ("notion", serde_json::json!({ "token": "secret_n", "query": "q" })), + ]; + for (kind, config) in &fixtures { + sqlx::query( + "INSERT INTO sources (id, kb_id, kind, name, config) VALUES ($1, $2, $3, $3, $4)", + ) + .bind(Uuid::now_v7()) + .bind(kb) + .bind(kind) + .bind(config) + .execute(&pool) + .await?; + } + + let listed = sources::list(&pool, kb).await?; + assert_eq!(listed.len(), fixtures.len()); + for s in &listed { + let obj = s.config.as_object().expect("config is an object"); + for key in SOURCE_SECRET_KEYS { + assert!( + !obj.contains_key(*key), + "{}: `{key}` must not reach a viewer, got {:?}", + s.kind, + obj + ); + } + } + // 身份标识留着 + let by_kind = |k: &str| { + listed + .iter() + .find(|s| s.kind == k) + .map(|s| s.config.clone()) + .expect("listed") + }; + assert_eq!(by_kind("s3")["bucket"], "b"); + assert_eq!(by_kind("s3")["access_key_id"], "AKIA"); + assert_eq!(by_kind("azure_blob")["account_name"], "acct"); + assert_eq!(by_kind("webdav")["username"], "u"); + assert_eq!(by_kind("custom")["endpoint"], "https://x.test/items"); + assert_eq!(by_kind("notion")["query"], "q"); + + // 同步那条路仍拿完整配置：凭据只是不出去，不是没了 + for s in &listed { + let full = sources::get(&pool, s.id).await?; + let want = &fixtures.iter().find(|(k, _)| *k == s.kind).unwrap().1; + assert_eq!(&full.config, want, "{}: sync still sees the credentials", s.kind); + } + Ok::<_, anyhow::Error>(()) + } + .await; + + let _ = sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await; + run +} diff --git a/docs/decisions/0013-a-source-should-hand-over-its-history.md b/docs/decisions/0013-a-source-should-hand-over-its-history.md index 1e6ccc1f..c84d1f66 100644 --- a/docs/decisions/0013-a-source-should-hand-over-its-history.md +++ b/docs/decisions/0013-a-source-should-hand-over-its-history.md @@ -107,3 +107,7 @@ shape — a class with an IRI drawn as a circle is a picture that lies. `sync_custom`. Acceptance: unit tests on `render()`, fixture tests where a real response is obtainable, create → sync → versions present → second sync adds 0, and clean `cargo clippy --workspace --all-targets` and `npm run typecheck`. + +## Revisions + +- 2026-09-03: every connector's credentials stay on the server (#246). Until now only `auth_header` was stripped from responses; the object-storage, WebDAV and Notion keys went out to every viewer. The keys now live in one list, `SOURCE_SECRET_KEYS`, shared by the listing, the create / update responses and the update merge (blank or missing keeps the stored value, an explicit `null` removes it). Adding a connector means adding its keys there first. From db2ca3cbe2040d59c53d5a1104439bebe2bb4a86 Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Thu, 3 Sep 2026 14:34:27 +0800 Subject: [PATCH 2/2] A security contact for the repository Co-Authored-By: Claude Fable 5.1 --- SECURITY.md | 48 ++---------------------------------------------- 1 file changed, 2 insertions(+), 46 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 30815ea4..64517446 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,49 +1,5 @@ # Security -*[中文版](SECURITY.zh-CN.md)* +If you find a vulnerability in Utopia, please email **security@deeplethe.com** rather than opening a public issue. Include the affected version or commit, the endpoint or component, and steps to reproduce. -Utopia is at v0.1. Below are the **known, unresolved** limits — not a vulnerability report, -but the places the design has not reached yet. - -## Before you put this on a public network - -**Credentials are stored in the clear.** LLM API keys and Ask-the-Data connection strings -are plain text in Postgres (`llm_settings.chat_api_key`, `data_sources.conn_string`). Anyone -who can read the database can read them. Encryption at rest is a 1.0 item; until then, keep -the system and its database inside a trusted network. - -**The default database password is `utopia`.** By default the port binds to loopback -(`127.0.0.1:1517`), so nothing outside the host can connect. If you change `UTOPIA_DB_BIND` -to expose it, change `UTOPIA_DB_PASSWORD` in `.env` first. - -**A data source is only as safe as its grants.** Registering one is a deployment-level -action, but the connection string reaches every workspace the source is granted to. Grant it -only where that database should be visible, and use a read-only database role in the string -itself — the SQL gate below is defence in depth, not a substitute for least privilege at the -source. - -## What is in place - -- **JWT signing key generated on first start** — 32 bytes from a CSPRNG, stored in the - database. No deployment shares a default key. -- **`Secure` on session cookies behind TLS** — decided from `X-Forwarded-Proto`, so local - HTTP development still works. Force it with `UTOPIA_COOKIE_SECURE=true` if your proxy - omits the header. -- **Database port bound to loopback** — `127.0.0.1:1517`; the app reaches the database over - the compose network. -- **Optional least-privilege runtime role** — set `UTOPIA_APP_DB_PASSWORD` and - `UTOPIA_MIGRATION_URL`, and the app connects as a role that can only read and write - business tables and append to the ledger, while migrations run as the owner. -- **Data sources reach only granted workspaces** — a registered database is mounted into a - knowledge base only where an explicit grant exists. Before this, any base admin could - mount any registered source, which crossed tenants. -- **Read-only gate on Ask-the-Data** — parser allowlist, read-only transaction, enforced row - limit; three layers, so a statement past the parser still cannot write. -- **Accounts are deactivated, not deleted** — `users.deactivated_at` blocks sign-in while the - ledger keeps that person's decisions attributable. -- **Passwords hashed with argon2.** - -## Reporting a vulnerability - -Open an issue. If it involves exploitable detail, start with the minimum needed to reproduce -and we will follow up privately. +You will get an acknowledgement within a few days. Once a fix is released, the advisory names the reporter unless you ask otherwise.