Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ on:
workflow_dispatch:
inputs:
tag:
description: "Image tag to publish (e.g. 0.1.0-rc1)"
description: "Image tag to publish (e.g. 0.1.0-rc2)"
required: true

permissions:
Expand Down
3 changes: 3 additions & 0 deletions crates/utopia-core/src/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,9 @@ pub struct AuditEventView {
pub target_kind: String,
pub target_id: Option<Uuid>,
pub detail: serde_json::Value,
/// NULL = 引擎自动(裁决器合并、一致性检查、推理物化……)。界面靠它把
/// 「没有人」和「人已被移除」分开:后者 actor_id 还在,只是查不到显示名
pub actor_id: Option<Uuid>,
pub actor_name: Option<String>,
pub created_at: DateTime<Utc>,
}
Expand Down
20 changes: 18 additions & 2 deletions crates/utopia-server/src/api/kbs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -359,14 +359,30 @@ async fn install_packs(
if pack_ids.is_empty() {
return Ok(());
}
let mut packs = Vec::with_capacity(pack_ids.len());
for id in pack_ids {
let pack = crate::ontology_packs::get(id)
.ok_or_else(|| AppError::invalid("unknown_pack", format!("未知的本体包:{id}")))?;
let bytes = crate::ontology_packs::bytes(pack)?;
crate::owl_import::apply(state, kb_id, actor, pack.filename, &bytes)
packs.push((pack, crate::ontology_packs::bytes(pack)?));
}
for (pack, bytes) in &packs {
crate::owl_import::apply(state, kb_id, actor, pack.filename, bytes)
.await
.map_err(|e| AppError::Other(anyhow::anyhow!("装本体包 {} 失败:{e}", pack.id)))?;
}
// 第二遍:跨包的 domain / range。包是挨个装的,先装的看不见后装的类——
// W3C Org 的 headOf 要等 FOAF 的 Agent(#222)。只装一个包时没有"别的包"
if packs.len() > 1 {
for (pack, bytes) in &packs {
let (d, r) =
crate::owl_import::relink_domains_ranges(state, kb_id, pack.filename, bytes)
.await
.map_err(|e| {
AppError::Other(anyhow::anyhow!("补本体包 {} 的签名失败:{e}", pack.id))
})?;
tracing::debug!(%kb_id, pack = pack.id, domains = d, ranges = r, "跨包签名补链");
}
}
Ok(())
}

Expand Down
51 changes: 41 additions & 10 deletions crates/utopia-server/src/api/sources_routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,10 @@ pub async fn sync_now(
#[derive(Deserialize)]
pub struct IngestBody {
pub filename: String,
/// 墓碑推送(`deleted: true`)可以不带 content——指南一直这么写,而字段
/// 从前是必填,缺了就在反序列化那一步被拒。其余情况仍然必填:空串在
/// 下面的校验里挡
#[serde(default)]
pub content: String,
#[serde(default)]
pub doc_time: Option<DateTime<Utc>>,
Expand Down Expand Up @@ -341,14 +345,33 @@ pub async fn ingest(
Ok(Json(json!({ "action": action_str(action) })))
}

/// 推送失败的两种性质。**调用方发错了**和**我们这边没接住**得分开:前者记进
/// run 供集成调试,但不算来源同步失败——来源没坏,是那一次请求不合格;
/// 后者才该把来源标成 failed 并进告警中心。从前两者都走 `finish_sync(error)`,
/// 一次格式错误就让铃铛说"来源同步失败,没有新内容进来"
enum PushError {
/// 4xx:负载不合格(JSON 解析、缺字段)
Rejected(String),
/// 摄入本身失败
Failed(String),
}

impl PushError {
fn message(&self) -> &str {
match self {
PushError::Rejected(m) | PushError::Failed(m) => m,
}
}
}

/// 认证之后的推送处理:解析 + 校验 + 摄入/墓碑。错误一律返回文字(记进 run)。
async fn handle_push(
state: &AppState,
source: &utopia_core::models::Source,
bytes: &[u8],
) -> Result<crate::ingest_sources::IngestAction, String> {
let body: IngestBody =
serde_json::from_slice(bytes).map_err(|e| format!("Invalid JSON payload: {e}"))?;
) -> Result<crate::ingest_sources::IngestAction, PushError> {
let body: IngestBody = serde_json::from_slice(bytes)
.map_err(|e| PushError::Rejected(format!("Invalid JSON payload: {e}")))?;
let identity = body
.external_id
.as_deref()
Expand All @@ -357,20 +380,24 @@ async fn handle_push(
.unwrap_or_else(|| body.filename.trim())
.to_string();
if identity.is_empty() {
return Err("external_id or filename is required".into());
return Err(PushError::Rejected(
"external_id or filename is required".into(),
));
}
let key = format!("api:{identity}");

// 墓碑:标记"不在来源中"(与 custom 的 deleted[] 同一条路径),content 可省略
if body.deleted {
utopia_store::documents::mark_missing_keys(&state.pool, source.id, &[key])
.await
.map_err(|e| e.to_string())?;
.map_err(|e| PushError::Failed(e.to_string()))?;
return Ok(crate::ingest_sources::IngestAction::Tombstoned);
}

if body.filename.trim().is_empty() || body.content.trim().is_empty() {
return Err("filename and content are required".into());
return Err(PushError::Rejected(
"filename and content are required".into(),
));
}
let action = crate::ingest_sources::ingest_item(
state,
Expand All @@ -383,11 +410,11 @@ async fn handle_push(
body.doc_time,
)
.await
.map_err(|e| e.to_string())?;
.map_err(|e| PushError::Failed(e.to_string()))?;
// 失而复得:曾被墓碑标记的身份再次正常推送,摘掉 missing 标记
utopia_store::documents::clear_missing_keys(&state.pool, source.id, &[key])
.await
.map_err(|e| e.to_string())?;
.map_err(|e| PushError::Failed(e.to_string()))?;
Ok(action)
}

Expand Down Expand Up @@ -434,10 +461,14 @@ pub async fn push(
state.emit_source(source.kb_id);
Ok(Json(json!({ "action": action_str(action) })))
}
Err(msg) => {
Err(err) => {
let msg = err.message().to_string();
utopia_store::sources::finish_run(&state.pool, run, source.id, Some(&msg), 0, 0)
.await?;
utopia_store::sources::finish_sync(&state.pool, source.id, Some(&msg), 0).await?;
// 只有我们这边没接住才算来源失败;调用方发错了留在 run 历史里就够
if let PushError::Failed(_) = err {
utopia_store::sources::finish_sync(&state.pool, source.id, Some(&msg), 0).await?;
}
state.emit_source(source.kb_id);
Err(utopia_core::AppError::Validation(msg).into())
}
Expand Down
35 changes: 35 additions & 0 deletions crates/utopia-server/src/extraction.rs
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,8 @@ async fn run(state: &AppState, document_id: Uuid, proposed_by: Option<Uuid>) ->
utopia_store::documents::set_graph_status(&state.pool, document_id, "extracting").await?;
state.emit_document(doc.kb_id, document_id);
let etypes = utopia_store::graph::entity_types(&state.pool, doc.kb_id).await?;
// 这一轮落过的事实(新建或重复观察):结尾对它们跑一遍签名检查
let mut touched_facts: Vec<Uuid> = Vec::new();
let rtypes = utopia_store::graph::relation_types(&state.pool, doc.kb_id).await?;
// 关系与属性分道:属性走字面值通道,不进关系清单。
//
Expand Down Expand Up @@ -762,6 +764,7 @@ async fn run(state: &AppState, document_id: Uuid, proposed_by: Option<Uuid>) ->
confidence,
)
.await?;
touched_facts.push(fact_id);
// 属性谓词也留原词:模型偶尔照抄 "person.salary" 全限定名,
// 命中的是剥掉前缀后的 key,原样是什么值得留着
utopia_store::graph::add_evidence(
Expand Down Expand Up @@ -882,6 +885,7 @@ async fn run(state: &AppState, document_id: Uuid, proposed_by: Option<Uuid>) ->
confidence,
)
.await?;
touched_facts.push(fact_id);
utopia_store::graph::add_evidence(
&state.pool,
fact_id,
Expand Down Expand Up @@ -1126,6 +1130,7 @@ async fn run(state: &AppState, document_id: Uuid, proposed_by: Option<Uuid>) ->
confidence,
)
.await?;
touched_facts.push(fact_id);
// 重复观察也要挂证据:多来源相互印证,任一来源删除后事实不孤儿化。
// 表层谓词随每次观察落笔——甲块说 "runs on"、乙块说 "optimized for"
// 会并进同一条事实,放事实上就是先写者胜,放证据上两个都留着
Expand Down Expand Up @@ -1221,6 +1226,36 @@ async fn run(state: &AppState, document_id: Uuid, proposed_by: Option<Uuid>) ->
if let Some(msg) = incomplete_reason(&unextracted, chunks.len()) {
return Err(anyhow::anyhow!(msg));
}
// 刚落的事实立刻过一遍签名。写入时只掰方向(judge_direction);掰不动的
// ——两个方向都对不上、或宾语没类型判不了——从前要等人按 Review 里的
// Run check 才露面,Axioms 一直是 0,图里却躺着反向事实(#222)。
// 检查失败不影响抽取本身:事实已经在库里,下一次 Run check 仍然查得到
if !touched_facts.is_empty() {
match utopia_store::reasoning::signature_breaks(
&state.pool,
doc.kb_id,
Some(&touched_facts),
)
.await
{
Ok(broken) if !broken.is_empty() => {
match utopia_store::reasoning::record_signature_breaks(
&state.pool,
doc.kb_id,
&broken,
)
.await
{
Ok(_) => state.emit_review(doc.kb_id),
Err(e) => {
tracing::warn!(%document_id, error = %e, "抽取后的签名违规没记进队列")
}
}
}
Ok(_) => {}
Err(e) => tracing::warn!(%document_id, error = %e, "抽取后的签名检查失败"),
}
}
utopia_store::documents::set_graph_status(&state.pool, document_id, "done").await?;
state.emit_document(doc.kb_id, document_id);

Expand Down
55 changes: 55 additions & 0 deletions crates/utopia-server/src/mappings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,39 @@ use uuid::Uuid;

const MAX_SCHEMA_CHARS: usize = 12_000;

/// 探索把 schema 里的量与维度落成 Metric / Dimension 实体,而这两个类不在任何
/// 内置本体包里——0009 之后建库不再自带类。没有它们,下面的 `type_id` 查不到,
/// 每条提议都被 `continue` 吞掉,页面只说"已排队"就再无下文(#223)。
/// 所以探索前把两个类补上:builtin,描述给抽取提示词,本体页可以改
async fn ensure_concept_types(pool: &sqlx::PgPool, kb_id: Uuid) -> anyhow::Result<()> {
for (key, label, description) in [
(
"metric",
"Metric",
"An aggregatable business quantity (revenue, order count, average ticket) that maps to a definition in a mounted database.",
),
(
"dimension",
"Dimension",
"A group-by attribute (region, month, product line) that maps to a column in a mounted database.",
),
] {
sqlx::query(
"INSERT INTO entity_types (id, kb_id, key, label, builtin, description)
SELECT $1, $2, $3, $4, TRUE, $5
WHERE NOT EXISTS (SELECT 1 FROM entity_types WHERE kb_id = $2 AND key = $3)",
)
.bind(Uuid::now_v7())
.bind(kb_id)
.bind(key)
.bind(label)
.bind(description)
.execute(pool)
.await?;
}
Ok(())
}

pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> {
let kb = utopia_store::kbs::get(&state.pool, kb_id).await?;
let settings = utopia_store::settings::get(&state.pool, kb.workspace_id)
Expand All @@ -27,6 +60,7 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<(
if sources.is_empty() {
anyhow::bail!("No data sources mounted");
}
ensure_concept_types(&state.pool, kb_id).await?;

// 各源 schema(引擎直读,保证新鲜;限量防 prompt 爆炸)
let mut schema_txt = String::new();
Expand Down Expand Up @@ -168,6 +202,27 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<(
}

tracing::info!(%kb_id, proposals = accepted, "映射探索完成,提议已入审核队列");
// 一条都没提出来时页面上什么都不会变——Pending 还是 0,而"已排队"那句
// 早就翻篇了。走告警中心说一声,人才知道该去刷新结构或给列加注释
if accepted == 0 {
if let Err(e) = utopia_store::alerts::raise(
&state.pool,
utopia_store::alerts::NewAlert {
kb_id: Some(kb_id),
severity: "info",
kind: utopia_store::alerts::kind::MAPPING_EXPLORATION_EMPTY,
min_role: utopia_core::models::Role::Editor,
subject_type: None,
subject_id: None,
detail: serde_json::json!({ "proposals": 0, "sources": source_names }),
},
)
.await
{
tracing::warn!(%kb_id, error = %e, "映射探索空结果的告警没写进去");
}
state.emit_alert();
}
state.emit_review(kb_id);
Ok(())
}
73 changes: 71 additions & 2 deletions crates/utopia-server/src/owl_import.rs
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,64 @@ pub async fn plan(
Ok((plan, proj, format))
}

/// 装完一组包之后再过一遍:把 domain / range 指向**别的包里的类**的那些关系接上。
///
/// 单次导入认本文件里的类和库里已有的类(见 [`apply`] 里的 resolve),但包是挨个
/// 装的:装 W3C Org 时 FOAF 还没来,`headOf` 的 `rdfs:domain foaf:Agent` 就落了空;
/// 等 FOAF 装好,没人回头补。没有 domain 的谓词 `judge_direction` 不判方向,
/// 反向的 `Project Aurora head_of Li Ting` 就原样进图(#222)。
///
/// 只补不删,关联表 ON CONFLICT DO NOTHING,重复跑无害。属性不在这里:属性的
/// 去向在计划阶段就定了(没有 domain 的根本建不出来),事后补 domain 改不了它
/// 已经是不是一列的事实
pub async fn relink_domains_ranges(
state: &AppState,
kb_id: Uuid,
filename: &str,
bytes: &[u8],
) -> AppResult<(usize, usize)> {
let format = RdfFormat::detect(filename, bytes);
let proj = ontology_rdf::project(bytes, format).map_err(|e| {
utopia_core::AppError::invalid_detail(
"bad_ontology_file",
"Could not parse this ontology file",
e.to_string(),
)
})?;
let classes: HashMap<String, Uuid> = utopia_store::graph::entity_types(&state.pool, kb_id)
.await?
.into_iter()
.filter_map(|t| t.iri.clone().map(|i| (i, t.id)))
.collect();
let relations: HashMap<String, Uuid> = utopia_store::graph::relation_types(&state.pool, kb_id)
.await?
.into_iter()
.filter_map(|r| r.iri.clone().map(|i| (i, r.id)))
.collect();
let mut link_d: Vec<(Uuid, Uuid)> = Vec::new();
let mut link_r: Vec<(Uuid, Uuid)> = Vec::new();
for p in &proj.properties {
if p.is_datatype {
continue;
}
let Some(&rid) = relations.get(&p.iri) else {
continue;
};
link_d.extend(
p.domains
.iter()
.filter_map(|d| classes.get(d).map(|&t| (rid, t))),
);
link_r.extend(
p.ranges
.iter()
.filter_map(|r| classes.get(r).map(|&t| (rid, t))),
);
}
utopia_store::ontology::link_domains_ranges_bulk(&state.pool, &link_d, &link_r).await?;
Ok((link_d.len(), link_r.len()))
}

/// 执行计划。属性在类之后落库——它们要挂在 domain 上,而 domain 要等
/// 类先建好并解析 IRI → id(就是下面那个 `id_of`)。
pub async fn apply(
Expand Down Expand Up @@ -500,9 +558,20 @@ pub async fn apply(
continue;
};
// domain/range 指向没被建出来的类时只丢那一个,不丢整条关系:
// 关系不像属性那样必须挂在类上,没有 domain 就是"不限主语类型"
// 关系不像属性那样必须挂在类上,没有 domain 就是"不限主语类型"。
//
// **库里已有的类也算数。** 从前只认本文件里的类,于是 W3C Org 的
// `headOf rdfs:domain foaf:Agent` 在 FOAF 已经装好的库里照样丢 domain,
// 而没有 domain 的谓词 `judge_direction` 根本不判方向(#222)
let resolve = |iris: &[String]| -> Vec<Uuid> {
iris.iter().filter_map(|i| id_of.get(i).copied()).collect()
iris.iter()
.filter_map(|i| {
id_of
.get(i)
.copied()
.or_else(|| existing_by_iri.get(i).copied())
})
.collect()
};
let domains = resolve(&p.domains);
let ranges = resolve(&p.ranges);
Expand Down
Loading
Loading