Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ dotenvy = "0.15"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
flate2 = "1"
# 枚举的字符串化与遍历(来源种类 `SourceKind`)——手写两份清单的漂移就是 #247
strum = { version = "0.27", features = ["derive"] }

# Phase 1: 摄入/检索/LLM
utopia-ingest = { path = "crates/utopia-ingest" }
Expand Down
1 change: 1 addition & 0 deletions crates/utopia-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,4 @@ chrono.workspace = true
sqlx.workspace = true
figment.workspace = true
pgvector.workspace = true
strum.workspace = true
61 changes: 61 additions & 0 deletions crates/utopia-core/src/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,67 @@ impl Source {
}
}

/// 来源的种类。**一处定义,三处消费**:创建时的白名单、同步时的分派(按枚举穷举匹配,
/// 加一种就得决定它怎么同步)、前端的下拉框(`web/src/sourceKinds.ts`,由
/// `utopia-store` 的测试对表)。
///
/// 此前后端两张手写清单各自演进:五种连接器加了同步分支、进了界面,却没进创建的
/// 白名单,界面上选得到、建的时候报「kind must be one of…」(#247)。变体顺序就是
/// 对话框里的顺序;字符串形式由 strum 按 snake_case 生成,不再手写
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
Hash,
strum::EnumIter,
strum::IntoStaticStr,
strum::EnumString,
)]
#[strum(serialize_all = "snake_case")]
pub enum SourceKind {
Folder,
Url,
Rss,
GithubIssues,
JiraIssues,
S3,
AzureBlob,
Gcs,
Webdav,
Notion,
Api,
Custom,
/// 每个库自带的记忆来源,不可建不可删(0015)
Memory,
/// 老数据里 `sources.kind` 的默认值,没有对应的界面
Upload,
}

impl SourceKind {
pub fn as_str(self) -> &'static str {
self.into()
}

pub fn parse(s: &str) -> Option<Self> {
s.parse().ok()
}

pub fn all() -> impl Iterator<Item = Self> {
<Self as strum::IntoEnumIterator>::iter()
}

/// 人能从界面建的:`memory` 与 `upload` 之外的全部
pub fn creatable_by_hand(self) -> bool {
!matches!(self, Self::Memory | Self::Upload)
}

pub fn creatable() -> impl Iterator<Item = Self> {
Self::all().filter(|k| k.creatable_by_hand())
}
}

/// 来源同步运行记录(渠道审计历史)。
#[derive(Debug, Clone, Serialize, sqlx::FromRow)]
pub struct SyncRun {
Expand Down
29 changes: 18 additions & 11 deletions crates/utopia-server/src/ingest_sources.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use crate::state::AppState;
use chrono::{DateTime, Utc};
use sha2::{Digest, Sha256};
use utopia_core::models::Source;
use utopia_core::models::SourceKind;
use uuid::Uuid;

/// 单次同步的新文档上限(防超长 feed/URL 列表拖垮任务)
Expand Down Expand Up @@ -48,17 +49,23 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<()
let run_id = utopia_store::sources::start_run(&state.pool, source_id).await?;
state.emit_source(source.kb_id);

let outcome = match source.kind.as_str() {
"url" => sync_urls(state, &source).await,
"rss" => sync_rss(state, &source).await,
"custom" => sync_custom(state, &source).await,
"github_issues" => sync_github_issues(state, &source).await,
"jira_issues" => sync_jira_issues(state, &source).await,
"s3" | "azure_blob" | "gcs" => sync_object_storage(state, &source).await,
"webdav" => sync_webdav(state, &source).await,
"notion" => sync_notion(state, &source).await,
// folder / api 无拉取语义
_ => Ok(SyncStats::default()),
// 按枚举穷举:加一种来源就得在这里决定它怎么同步,编译器不放过漏掉的那一支
let outcome = match SourceKind::parse(&source.kind) {
Some(SourceKind::Url) => sync_urls(state, &source).await,
Some(SourceKind::Rss) => sync_rss(state, &source).await,
Some(SourceKind::Custom) => sync_custom(state, &source).await,
Some(SourceKind::GithubIssues) => sync_github_issues(state, &source).await,
Some(SourceKind::JiraIssues) => sync_jira_issues(state, &source).await,
Some(SourceKind::S3 | SourceKind::AzureBlob | SourceKind::Gcs) => {
sync_object_storage(state, &source).await
}
Some(SourceKind::Webdav) => sync_webdav(state, &source).await,
Some(SourceKind::Notion) => sync_notion(state, &source).await,
// 被动容器:folder / api / memory / upload 没有拉取语义
Some(SourceKind::Folder | SourceKind::Api | SourceKind::Memory | SourceKind::Upload) => {
Ok(SyncStats::default())
}
None => Err(anyhow::anyhow!("unknown source kind `{}`", source.kind)),
};

match outcome {
Expand Down
26 changes: 10 additions & 16 deletions crates/utopia-store/src/sources.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,22 @@

use chrono::{DateTime, Utc};
use sqlx::PgPool;
use utopia_core::models::{Role, Source, SourceView, SyncRun, SOURCE_SECRET_KEYS};
use utopia_core::models::{Role, Source, SourceKind, SourceView, SyncRun, SOURCE_SECRET_KEYS};
use utopia_core::{AppError, AppResult};
use uuid::Uuid;

/// folder = 纯容器(上传/拖拽入内,无同步语义);url/rss = 拉取型;api = 推送型。
/// 本机目录监听(watch_folder)已否决——自部署用户看不到服务器磁盘;
/// 未来的 watch 形态是对象存储/网盘(P5 连接器,与 BlobStore 接缝配套)。
/// 对象存储 / WebDAV / Notion 是它的替代形态(0013)。
/// custom = 自定义拉取器:任何实现 Utopia ingest 接口的 URL(返回 items JSON)即可定时摄取。
/// github_issues / jira_issues = 工单:一张工单连同它的状态变更史成为一篇文档。
///
/// **改这里就得改前端那份清单**(`Library.tsx` 的建来源对话框与 `api.ts` 的
/// `SourceView["kind"]`)。两处对不上时的症状是:界面上选得到、建的时候报
/// 「kind must be one of…」——单元测试与 tsc 都看不见,只有端到端会撞上。
pub const KINDS: &[&str] = &[
"folder",
"url",
"rss",
"api",
"custom",
"github_issues",
"jira_issues",
];
/// 种类的清单**不在这里写**:`SourceKind`(utopia-core)一个枚举出全部——创建的白名单、
/// 同步的分派、前端的下拉框(有测试对表)。从前这里有一张手写的 `KINDS`,五种连接器
/// 加了同步却没进这张表,界面上选得到、建不出来(#247)
pub fn creatable_kinds() -> Vec<&'static str> {
SourceKind::creatable().map(|k| k.as_str()).collect()
}

/// 校验并规范化标准 5 段 cron 表达式(内部用 cron crate 的 6 段:补秒位)。
pub fn validate_cron(expr: &str) -> AppResult<String> {
Expand Down Expand Up @@ -93,10 +87,10 @@ pub async fn create(
sync_interval_minutes: Option<i32>,
sync_cron: Option<&str>,
) -> AppResult<Source> {
if !KINDS.contains(&kind) {
if !SourceKind::parse(kind).is_some_and(|k| k.creatable_by_hand()) {
return Err(AppError::Validation(format!(
"kind must be one of: {}",
KINDS.join(", ")
creatable_kinds().join(", ")
)));
}
if name.trim().is_empty() {
Expand Down
53 changes: 53 additions & 0 deletions crates/utopia-store/tests/a_source_kind_is_listed_once.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
//! #247:来源的种类只在一处定义,前后端对表。
//!
//! 后端 `SourceKind`(utopia-core)一个枚举出两份清单:创建时的白名单、同步时的分派
//! (后者按枚举穷举匹配,编译器保证加了种类就得决定它怎么同步)。前端那一份在
//! `web/src/sourceKinds.ts`,这个测试把它读出来跟枚举比——此前两边各自手写,五种
//! 连接器进了界面、进了同步,却没进创建白名单,界面上选得到、建的时候报
//! 「kind must be one of…」。单元测试与 tsc 都看不见的那种漂移,这里看得见。
//!
//! 不需要数据库。

use std::path::Path;
use utopia_core::models::SourceKind;

/// 从 `CREATABLE_SOURCE_KINDS = [ "…", … ] as const` 里把引号里的字面量按顺序读出来
fn frontend_kinds(src: &str) -> Vec<String> {
let start = src
.find("CREATABLE_SOURCE_KINDS = [")
.expect("web/src/sourceKinds.ts declares CREATABLE_SOURCE_KINDS");
let body = &src[start..];
let end = body.find(']').expect("the array closes");
body[..end]
.split('"')
.skip(1)
.step_by(2)
.map(str::to_string)
.collect()
}

#[test]
fn the_frontend_list_matches_the_backend_enum() {
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../web/src/sourceKinds.ts");
let src =
std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()));
let frontend = frontend_kinds(&src);
let backend: Vec<String> = SourceKind::creatable()
.map(|k| k.as_str().to_string())
.collect();
assert_eq!(
frontend, backend,
"web/src/sourceKinds.ts and utopia_core::models::SourceKind list different kinds (order matters: it is the dialog's order)"
);
}

#[test]
fn every_kind_round_trips_through_its_string() {
for k in SourceKind::all() {
assert_eq!(SourceKind::parse(k.as_str()), Some(k), "{k:?}");
}
assert_eq!(SourceKind::parse("watch_folder"), None);
assert!(!SourceKind::Memory.creatable_by_hand());
assert!(!SourceKind::Upload.creatable_by_hand());
assert!(SourceKind::S3.creatable_by_hand());
}
Original file line number Diff line number Diff line change
Expand Up @@ -111,3 +111,4 @@ shape — a class with an IRI drawn as a circle is a picture that lies.
## Revisions

- 2026-09-03: every connector's credentials stay on the server (#246). Until now only `auth_header` was stripped from responses; the object-storage, WebDAV and Notion keys went out to every viewer. The keys now live in one list, `SOURCE_SECRET_KEYS`, shared by the listing, the create / update responses and the update merge (blank or missing keeps the stored value, an explicit `null` removes it). Adding a connector means adding its keys there first.
- 2026-09-03: the five connectors added under this record could not be created (#247): the store's hand-written `KINDS` allowlist stopped at seven kinds while the sync dispatcher and the UI knew twelve. The kinds now come from one enum, `SourceKind` in `utopia-core`; the allowlist is derived from it, the dispatcher matches it exhaustively, and a test compares the frontend's `web/src/sourceKinds.ts` against it, so the three can no longer drift apart.
17 changes: 2 additions & 15 deletions web/src/api.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { SourceKind } from "./sourceKinds";
import { S, lang } from "./i18n";

export class ApiError extends Error {
Expand Down Expand Up @@ -179,21 +180,7 @@ export interface ExtractionDrop {

export interface SourceView {
id: string;
kind:
| "folder"
| "url"
| "rss"
| "api"
| "custom"
| "github_issues"
| "jira_issues"
| "s3"
| "azure_blob"
| "gcs"
| "webdav"
| "notion"
| "memory"
| "upload";
kind: SourceKind;
name: string;
config: {
urls?: string[];
Expand Down
36 changes: 6 additions & 30 deletions web/src/pages/Library.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ import {
} from "lucide-react";
import { api, type Doc, type ExtractionDrop, type SourceView } from "../api";
import { S } from "../i18n";
import {
CREATABLE_SOURCE_KINDS,
type CreatableSourceKind,
} from "../sourceKinds";
import { useKb, useKbId } from "../kb";
import { toast } from "../toast";
import { Chip, type ChipTone, DangerConfirm, Loading, Pager } from "../ui";
Expand Down Expand Up @@ -1267,20 +1271,7 @@ function SourceModal({
/** isApi=true 时父级紧接着打开密钥弹窗 */
onDone: (id?: string, isApi?: boolean) => void;
}) {
const [kind, setKind] = useState<
| "folder"
| "url"
| "rss"
| "custom"
| "api"
| "github_issues"
| "jira_issues"
| "s3"
| "azure_blob"
| "gcs"
| "webdav"
| "notion"
>("folder");
const [kind, setKind] = useState<CreatableSourceKind>("folder");
const [name, setName] = useState("");
const [icon, setIcon] = useState<string | null>(null);
const [urls, setUrls] = useState("");
Expand Down Expand Up @@ -1447,22 +1438,7 @@ function SourceModal({
<div className="px-5 py-4">
{/* 类型 */}
<div className="flex gap-2 mb-2">
{(
[
"folder",
"url",
"rss",
"github_issues",
"jira_issues",
"s3",
"azure_blob",
"gcs",
"webdav",
"notion",
"api",
"custom",
] as const
).map((k) => {
{CREATABLE_SOURCE_KINDS.map((k) => {
const Icon = KIND_ICON[k];
return (
<button
Expand Down
6 changes: 4 additions & 2 deletions web/src/pages/SourcesRail.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
/* 来源栏("来源即文件夹"):Library 与 DocViewer 共用的左侧导航。 */
import type { SourceKind } from "../sourceKinds";
import { useQuery } from "@tanstack/react-query";
import {
Archive,
Expand Down Expand Up @@ -59,7 +60,8 @@ export const SOURCE_ICONS: Record<string, LucideIcon> = {
users: Users,
};

export const KIND_ICON = {
// 按 SourceKind 键全:加一种来源没配图标,tsc 就红
export const KIND_ICON: Record<SourceKind, LucideIcon> = {
folder: FolderOpen,
url: Globe,
rss: Rss,
Expand All @@ -74,7 +76,7 @@ export const KIND_ICON = {
notion: Notebook,
memory: Brain,
upload: Upload,
} as const;
};

/** 有拉取/同步语义的来源类型(folder/api 无同步概念) */
export const SYNCING_KINDS = new Set([
Expand Down
Loading
Loading