diff --git a/Cargo.lock b/Cargo.lock index eca2967d3..b9be850b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -720,7 +720,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "codegraph" -version = "2.1.7" +version = "2.1.8" dependencies = [ "anyhow", "camino", @@ -743,7 +743,7 @@ dependencies = [ [[package]] name = "codegraph-api" -version = "2.1.7" +version = "2.1.8" dependencies = [ "anyhow", "camino", @@ -760,7 +760,7 @@ dependencies = [ [[package]] name = "codegraph-bench" -version = "2.1.7" +version = "2.1.8" dependencies = [ "anyhow", "camino", @@ -778,7 +778,7 @@ dependencies = [ [[package]] name = "codegraph-binary" -version = "2.1.7" +version = "2.1.8" dependencies = [ "camino", "codegraph-core", @@ -795,7 +795,7 @@ dependencies = [ [[package]] name = "codegraph-context" -version = "2.1.7" +version = "2.1.8" dependencies = [ "codegraph-core", "codegraph-graph", @@ -807,7 +807,7 @@ dependencies = [ [[package]] name = "codegraph-core" -version = "2.1.7" +version = "2.1.8" dependencies = [ "async-graphql", "camino", @@ -818,7 +818,7 @@ dependencies = [ [[package]] name = "codegraph-docs" -version = "2.1.7" +version = "2.1.8" dependencies = [ "anyhow", "codegraph-core", @@ -835,7 +835,7 @@ dependencies = [ [[package]] name = "codegraph-extract" -version = "2.1.7" +version = "2.1.8" dependencies = [ "camino", "codegraph-binary", @@ -874,7 +874,7 @@ dependencies = [ [[package]] name = "codegraph-graph" -version = "2.1.7" +version = "2.1.8" dependencies = [ "async-trait", "bincode", @@ -904,7 +904,7 @@ dependencies = [ [[package]] name = "codegraph-graphql" -version = "2.1.7" +version = "2.1.8" dependencies = [ "anyhow", "async-graphql", @@ -927,7 +927,7 @@ dependencies = [ [[package]] name = "codegraph-installer" -version = "2.1.7" +version = "2.1.8" dependencies = [ "anyhow", "camino", @@ -943,7 +943,7 @@ dependencies = [ [[package]] name = "codegraph-mcp" -version = "2.1.7" +version = "2.1.8" dependencies = [ "anyhow", "axum", @@ -966,7 +966,7 @@ dependencies = [ [[package]] name = "codegraph-sboxes" -version = "2.1.7" +version = "2.1.8" dependencies = [ "camino", "codegraph-core", diff --git a/Cargo.toml b/Cargo.toml index a3f168eaf..ab9c478ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ members = [ ] [workspace.package] -version = "2.1.7" +version = "2.1.8" edition = "2021" rust-version = "1.80" license = "MIT" diff --git a/crates/codegraph-docs/src/graph.rs b/crates/codegraph-docs/src/graph.rs index 86560c796..f5076c668 100644 --- a/crates/codegraph-docs/src/graph.rs +++ b/crates/codegraph-docs/src/graph.rs @@ -7,6 +7,8 @@ use codegraph_graph::Search; use codegraph_graph::Storage; use serde::Serialize; use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use tokio::sync::RwLock as TokioRwLock; @@ -17,6 +19,10 @@ const TYPE_RECORD_BASE: u64 = 200_000_000_000; const VALUE_RECORD_BASE: u64 = 300_000_000_000; const STRUCT_RECORD_BASE: u64 = 400_000_000_000; const PATTERN_RECORD_BASE: u64 = 500_000_000_000; +/// Dải riêng cho doc id — node id và doc id phải không đè nhau (hydrate/ +/// storage key dùng chung namespace `set_node_meta`). Node id ≥ `doc_base` +/// (~1e9), pattern id ở dải 5e11, doc id ở dải này. +const DOC_ID_BASE: u64 = 600_000_000_000; /// Sentinel storage keys for persisted node/doc lists (node ids are u64 /// that never reach these small constants because real ids start at @@ -24,6 +30,9 @@ const PATTERN_RECORD_BASE: u64 = 500_000_000_000; const DOC_NODE_LIST_RECORD: u64 = 0; const DOC_LIST_RECORD: u64 = 1; const DOC_META_BASE: u64 = 10_000_000_000; +/// Sentinel cho interner (mảng string JSON theo thứ tự id). Doc id nằm ở +/// dải ≥ DOC_ID_BASE nên các slot nhỏ này không đụng doc metadata. +const DOC_INTERNER_RECORD: u64 = DOC_META_BASE + 1; /// Default sharding for document tries (mirrors code graph). const DEFAULT_SHARDING: usize = 64; @@ -65,13 +74,17 @@ impl DocumentGraph { docs: HashMap::new(), nodes: std::sync::Mutex::new(HashMap::new()), intern: Interner::new(), - path_trie: Search::new(sharding, storage.clone()), - type_trie: Search::new(sharding, storage.clone()), - value_trie: Search::new(sharding, storage.clone()), - struct_trie: Search::new(sharding, storage.clone()), - pattern_trie: Search::new(sharding, storage.clone()), + // 4 trie projection dùng CHUNG một storage — radix lưu root/shortcut + // theo shard index (0..sharding) nên mỗi trie phải ở một dải shard + // riêng (bias * sharding), nếu không root pointer ghi đè lẫn nhau + // và search chỉ thấy trie insert sau cùng. + path_trie: Search::with_shard_bias(sharding, storage.clone(), 0), + type_trie: Search::with_shard_bias(sharding, storage.clone(), 1), + value_trie: Search::with_shard_bias(sharding, storage.clone(), 2), + struct_trie: Search::with_shard_bias(sharding, storage.clone(), 3), + pattern_trie: Search::with_shard_bias(sharding, storage.clone(), 4), doc_base, - next_doc_id: doc_base, + next_doc_id: DOC_ID_BASE, next_node_id: doc_base, } } @@ -111,9 +124,78 @@ impl DocumentGraph { }; graph.next_doc_id = graph.next_doc_id.max(max_doc + 1); graph.next_node_id = graph.next_node_id.max(max_node + 1); + // Interner chỉ sống trong RAM — persist kèm mỗi upsert, restore tại + // đây để id trong tries persist vẫn resolve được. Thiếu blob (graph + // cũ) → dựng lại từ doc metadata theo thứ tự doc id (không clear trie + // — Search::clear xoá cả namespace dùng chung của storage). + if !graph.restore_interner().await? { + graph.rebuild_interner_from_docs(); + } + graph.materialize_node_cache(); Ok(graph) } + /// Load interner đã persist. Trả `false` nếu chưa có blob (graph cũ). + async fn restore_interner(&mut self) -> Result { + let bytes = { + let guard = self.storage.read().await; + guard.get_node_meta(DOC_INTERNER_RECORD as usize).await? + }; + let Some(bytes) = bytes else { + return Ok(false); + }; + if bytes.is_empty() { + return Ok(false); + } + let strings: Vec = serde_json::from_slice(&bytes) + .map_err(|e| anyhow::anyhow!("corrupt interner blob: {e}"))?; + self.intern = Interner::with_strings(strings); + Ok(true) + } + + /// Dựng interner từ keys + scalar values của các doc đã load (theo thứ tự + /// doc id — trùng thứ tự intern lúc ingest tuần tự). + fn rebuild_interner_from_docs(&mut self) { + self.intern = Interner::new(); + let mut docs: Vec<&Document> = self.docs.values().collect(); + docs.sort_by_key(|d| d.id); + for doc in docs { + for node in &doc.nodes { + if let Some(k) = &node.key { + self.intern.intern(k.clone()); + } + if let Some(Scalar::String(s)) = &node.value { + self.intern.intern(s.clone()); + } + } + } + } + + /// Persist interner — gọi sau mỗi upsert để lần `open()` sau vẫn khớp + /// token payload đã ghi vào tries. + async fn persist_interner(&self) -> Result<()> { + let blob = + serde_json::to_vec(&self.intern.strings()).map_err(|e| anyhow::anyhow!("{e}"))?; + self.storage + .write() + .await + .set_node_meta(DOC_INTERNER_RECORD as usize, &blob) + .await + .map_err(|e| anyhow::anyhow!(e.to_string())) + } + + /// Materialize node cache từ doc metadata (Document serialize đủ nodes) — + /// hydrate/path_tokens/search_value đọc từ cache trước storage. + fn materialize_node_cache(&self) { + let mut cache = self.nodes.lock().unwrap(); + cache.clear(); + for doc in self.docs.values() { + for node in &doc.nodes { + cache.insert(node.id, node.clone()); + } + } + } + /// Ingest một file từ disk: đọc, detect format theo extension (override /// bằng `format`), parse rồi upsert. Trùng `path` với doc đã có → thay thế /// tại chỗ (re-ingest khi chạy lại `codegraph init` là idempotent). @@ -166,19 +248,21 @@ impl DocumentGraph { self.add_doc_id(doc_id).await?; let node_ids: Vec = doc.nodes.iter().map(|n| n.id).collect(); self.add_node_ids(&node_ids).await?; - // Insert into tries. - for node in &doc.nodes { - self.insert_node_into_tries(node).await?; - } - // Materialize nodes vào cache in-memory (hydrate/collect_path đọc từ - // đây trước, thiếu thì mới xuống storage). + // Materialize nodes vào cache TRƯỚC khi tokenize — `path_tokens`/ + // `type_tokens` đi lên tổ tiên qua cache, cache thiếu thì path chain + // chỉ còn `[root]` (bug gốc: tries insert trước, cache sau). { let mut cache = self.nodes.lock().unwrap(); for node in &doc.nodes { cache.insert(node.id, node.clone()); } } + // Insert into tries. + for node in &doc.nodes { + self.insert_node_into_tries(node).await?; + } self.docs.insert(doc_id, doc.clone()); + self.persist_interner().await?; Ok(doc_id) } @@ -223,22 +307,47 @@ impl DocumentGraph { /// Hydrate a node into a small payload suitable for LLM reasoning. /// Đọc node + tổ tiên (cho path) + con theo nhu cầu từ storage. pub async fn hydrate(&self, node_id: u64) -> Option { - let node = self.node(node_id).await?; - let path = self.collect_path(node_id).await; - let mut children = Vec::new(); - for c in &node.children { - if let Some(payload) = Box::pin(self.hydrate(*c)).await { - children.push(payload); + self.hydrate_depth(node_id, None).await + } + + /// Như `hydrate` nhưng giới hạn số tầng con đi xuống (`max_depth = Some(2)` + /// là payload 2 tầng — giữ payload nhỏ cho LLM trên doc lớn). + pub async fn hydrate_depth( + &self, + node_id: u64, + max_depth: Option, + ) -> Option { + self.hydrate_inner(node_id, max_depth, 0).await + } + + fn hydrate_inner( + &self, + node_id: u64, + max_depth: Option, + level: usize, + ) -> Pin> + Send + '_>> { + Box::pin(async move { + let node = self.node(node_id).await?; + let path = self.collect_path(node_id).await; + let mut children = Vec::new(); + let descend = max_depth.is_none_or(|d| level < d); + if descend { + for c in &node.children { + if let Some(payload) = self.hydrate_inner(*c, max_depth, level + 1).await { + children.push(payload); + } + } } - } - Some(NodePayload { - id: node.id, - path, - kind: node.kind, - value: node.value.clone(), - key: node.key.clone(), - doc: node.doc, - children, + Some(NodePayload { + id: node.id, + path, + kind: node.kind, + value: node.value.clone(), + key: node.key.clone(), + index: node.index, + doc: node.doc, + children, + }) }) } @@ -289,6 +398,78 @@ impl DocumentGraph { Ok(ids) } + // ── Doc listing / value lookup (dùng bởi MCP + CLI) ─────────────── + + /// Liệt kê các document đã ingest kèm metadata (path, format, root, số node). + pub fn list_docs(&self) -> Vec { + let mut infos: Vec = self + .docs + .values() + .map(|d| DocInfo { + doc_id: d.id, + path: d.path.clone(), + format: d.format.clone(), + root_node_id: d.root, + nodes: d.nodes.len(), + }) + .collect(); + infos.sort_by_key(|i| i.doc_id); + infos + } + + /// Tra id đã intern cho một key — cầu nối query text → `DocToken::field`. + pub fn intern_id(&self, s: &str) -> Option { + self.intern.get(s) + } + + /// Giải ngược id interned thành chuỗi (resolve kết quả search). + pub fn intern_str(&self, id: u64) -> Option { + self.intern.resolve(id).map(str::to_string) + } + + /// Tìm node scalar chứa `query` (case-insensitive) — quét node cache, + /// không cần trie. `limit` chặn kết quả cho payload LLM. + pub fn search_value_substring(&self, query: &str, limit: usize) -> Vec { + let q = query.to_lowercase(); + let cache = self.nodes.lock().unwrap(); + let mut hits = Vec::new(); + for node in cache.values() { + let matched = match &node.value { + Some(Scalar::String(s)) => s.to_lowercase().contains(&q), + Some(Scalar::Number(n)) => format!("{n}").contains(&q), + _ => false, + }; + if matched { + hits.push(node.clone()); + if hits.len() >= limit { + break; + } + } + } + hits.sort_by_key(|n| n.id); + hits + } + + /// Tìm node có key chứa `query` (case-insensitive) — fallback cho + /// `search_path` khi pattern không phải full path từ root (chain radix + /// luôn bắt đầu từ root nên tên key đơn lẻ không match được). + pub fn search_key_substring(&self, query: &str, limit: usize) -> Vec { + let q = query.to_lowercase(); + let cache = self.nodes.lock().unwrap(); + let mut hits: Vec = cache + .values() + .filter(|n| { + n.key + .as_deref() + .is_some_and(|k| k.to_lowercase().contains(&q)) + }) + .cloned() + .collect(); + hits.sort_by_key(|n| n.id); + hits.truncate(limit); + hits + } + // ── Stats ───────────────────────────────────────────────────────── pub async fn stats(&self) -> Result { @@ -507,27 +688,76 @@ impl DocumentGraph { } fn path_tokens(&mut self, node: &Node) -> Vec { - let mut tokens = vec![DocToken::root()]; + // Thu keys từ node đi lên (node → ancestor), đảo lại thành + // ancestor → node rồi MỚI gắn root ở đầu: chain phải là + // [root, field(top), ..., field(node)] để khớp query full path. + let mut keys = Vec::new(); let mut cur = node.id; let cache = self.nodes.lock().unwrap(); while let Some(n) = cache.get(&cur) { if let Some(key) = &n.key { let key_id = self.intern.intern(key.clone()); - tokens.push(DocToken::field(key_id)); + keys.push(DocToken::field(key_id)); } cur = n.parent.unwrap_or(0); } + keys.reverse(); + let mut tokens = vec![DocToken::root()]; + tokens.extend(keys); + tokens + } + /// Token hóa loại node theo chuỗi tổ tiên: MAP → FIELD → NUMBER ... + /// Payload để 0 — đây là token "kind", không mang id. + fn type_tokens(&mut self, node: &Node) -> Vec { + let mut tokens = Vec::new(); + let mut cur = Some(node.clone()); + let cache = self.nodes.lock().unwrap(); + while let Some(n) = cur { + tokens.push(kind_token(&n.kind)); + cur = n.parent.and_then(|p| cache.get(&p).cloned()); + } tokens.reverse(); tokens } - fn type_tokens(&self, _node: &Node) -> Vec { - vec![DocToken::map(), DocToken::field(0)] // simplified + /// Token hóa giá trị scalar: Str/Num/Bool intern theo giá trị. + fn value_tokens(&mut self, node: &Node) -> Vec { + match &node.value { + Some(Scalar::String(s)) => vec![DocToken::str(self.intern.intern(s.clone()))], + Some(Scalar::Number(n)) => vec![DocToken::num(self.intern.intern(format!("{n}")))], + Some(Scalar::Bool(b)) => { + vec![DocToken::bool(self.intern.intern(b.to_string()))] + } + Some(Scalar::Null) => vec![DocToken::null()], + None => vec![], + } } - fn value_tokens(&self, _node: &Node) -> Vec { - vec![] + /// Token hóa cấu trúc: cửa sổ 2 tầng [kind cha, kind node] — ví + /// "FIELD NUMBER" là hình dạng điển hình của một field mang scalar. + fn struct_tokens(&mut self, node: &Node) -> Vec { + let parent_kind = node + .parent + .and_then(|p| self.nodes.lock().unwrap().get(&p).cloned()) + .map(|p| kind_token(&p.kind)); + let mut tokens = Vec::new(); + if let Some(t) = parent_kind { + tokens.push(t); + } + tokens.push(kind_token(&node.kind)); + tokens } - fn struct_tokens(&self, _node: &Node) -> Vec { - vec![] +} + +fn kind_token(kind: &Kind) -> DocToken { + match kind { + Kind::Root => DocToken::root(), + Kind::Map => DocToken::map(), + Kind::Array => DocToken::arr(), + Kind::Index => DocToken::idx(0), + Kind::Field => DocToken::field(0), + Kind::String => DocToken::str(0), + Kind::Number => DocToken::num(0), + Kind::Bool => DocToken::bool(0), + Kind::Null | Kind::Reference => DocToken::null(), } } @@ -539,10 +769,21 @@ pub struct NodePayload { pub kind: Kind, pub value: Option, pub key: Option, + pub index: Option, pub doc: u64, pub children: Vec, } +/// Thông tin tóm tắt một document — trả về cho `doc list`. +#[derive(Debug, Clone, Serialize)] +pub struct DocInfo { + pub doc_id: u64, + pub path: String, + pub format: String, + pub root_node_id: u64, + pub nodes: usize, +} + /// Summary returned by `codegraph doc stats`. #[derive(Debug, Default, Serialize)] pub struct DocStats { @@ -605,4 +846,87 @@ mod tests { .unwrap(); assert!(d3 > d1 && d3 > d2, "d3={d3} phải sau d1={d1}, d2={d2}"); } + + /// Children phải được persist: hydrate root đi xuống được, và + /// `hydrate_depth` giới hạn số tầng trả về. + #[tokio::test] + async fn hydrate_descends_children_with_depth_limit() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("a.yaml"); + std::fs::write(&p, "service:\n name: api\n replicas: 3\n").unwrap(); + let storage = Arc::new(TokioRwLock::new(InMemoryStorage::default())); + let mut graph = DocumentGraph::new(storage, DocConfig::default()); + let _doc_id = graph.ingest_file(p.to_str().unwrap(), None).await.unwrap(); + let root = graph.list_docs()[0].root_node_id; + + let full = graph.hydrate(root).await.unwrap(); + // root → service → {name, replicas} + assert_eq!(full.children.len(), 1, "root phải có 1 con `service`"); + let service = &full.children[0]; + assert_eq!(service.key.as_deref(), Some("service")); + assert_eq!(service.children.len(), 2, "service phải có name + replicas"); + + let shallow = graph.hydrate_depth(root, Some(1)).await.unwrap(); + assert_eq!(shallow.children.len(), 1); + assert!( + shallow.children[0].children.is_empty(), + "max_depth=1 không đi xuống tầng service" + ); + } + + /// Sau reopen, interner phải khớp token đã ghi trong tries — search theo + /// key name vẫn trả kết quả. + #[tokio::test] + async fn reopen_preserves_interner_and_search() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("a.yaml"); + std::fs::write(&p, "service:\n replicas: 3\n").unwrap(); + let storage = Arc::new(TokioRwLock::new(InMemoryStorage::default())); + let mut graph = DocumentGraph::new(storage.clone(), DocConfig::default()); + graph.ingest_file(p.to_str().unwrap(), None).await.unwrap(); + drop(graph); + + let reopened = DocumentGraph::open(storage, DocConfig::default()) + .await + .unwrap(); + let key_id = reopened + .intern_id("replicas") + .expect("key `replicas` phải còn trong interner sau reopen"); + let tokens = vec![ + DocToken::root(), + DocToken::field(reopened.intern_id("service").unwrap()), + DocToken::field(key_id), + ]; + let ids = reopened.search_path(&tokens, None).await.unwrap(); + assert!(!ids.is_empty(), "search `replicas` sau reopen phải match"); + } + + #[tokio::test] + async fn doc_ids_do_not_collide_with_node_ids() { + let dir = tempfile::tempdir().unwrap(); + let files: Vec<_> = (0..3) + .map(|i| { + let p = dir.path().join(format!("d{i}.yaml")); + std::fs::write(&p, format!("svc{i}:\n name: a{i}\n")).unwrap(); + p + }) + .collect(); + let storage = Arc::new(TokioRwLock::new(InMemoryStorage::default())); + let mut graph = DocumentGraph::new(storage, DocConfig::default()); + let mut doc_ids = Vec::new(); + for f in &files { + doc_ids.push(graph.ingest_file(f.to_str().unwrap(), None).await.unwrap()); + } + let node_ids: Vec = graph.list_docs().iter().map(|i| i.root_node_id).collect(); + for d in &doc_ids { + assert!( + !node_ids.contains(d), + "doc id {d} không được trùng node id nào" + ); + assert!( + *d >= DOC_ID_BASE, + "doc id {d} phải nằm trong dải riêng ≥ DOC_ID_BASE" + ); + } + } } diff --git a/crates/codegraph-docs/src/intern.rs b/crates/codegraph-docs/src/intern.rs index 8a209f026..1eb237652 100644 --- a/crates/codegraph-docs/src/intern.rs +++ b/crates/codegraph-docs/src/intern.rs @@ -20,6 +20,20 @@ impl Interner { } } + /// Khôi phục interner từ danh sách string đã persist (id = vị trí + 1). + pub fn with_strings(strings: Vec) -> Self { + let mut this = Self::new(); + for s in strings { + this.intern(s); + } + this + } + + /// Toàn bộ string theo thứ tự id — dùng để persist. + pub fn strings(&self) -> Vec { + self.reverse.clone() + } + /// Return the interned id for `s`, inserting if absent. pub fn intern(&mut self, s: String) -> u64 { if let Some(&id) = self.strings.get(&s) { diff --git a/crates/codegraph-docs/src/parsers/mod.rs b/crates/codegraph-docs/src/parsers/mod.rs index a9a96675e..e6ba658a5 100644 --- a/crates/codegraph-docs/src/parsers/mod.rs +++ b/crates/codegraph-docs/src/parsers/mod.rs @@ -88,6 +88,7 @@ impl DocBuilder { doc: self.doc_id, }; self.order.push(built_node.clone()); + let order_idx = self.order.len() - 1; self.nodes.insert(id, built_node.clone()); // Link parent → child. if let Some(pid) = parent @@ -123,6 +124,11 @@ impl DocBuilder { } _ => {} } + // Sync children vào `order` — lúc push ban đầu `children` còn rỗng, + // các link parent→child chỉ xuất hiện sau khi đệ quy xong. + if let Some(p) = self.nodes.get(&id) { + self.order[order_idx].children = p.children.clone(); + } // Return a clone of the built node (children already filled in `order`). self.nodes.get(&id).cloned().unwrap_or(built_node) } diff --git a/crates/codegraph-graph/src/radix.rs b/crates/codegraph-graph/src/radix.rs index fc0f3f676..5adf2bc8e 100644 --- a/crates/codegraph-graph/src/radix.rs +++ b/crates/codegraph-graph/src/radix.rs @@ -172,6 +172,9 @@ pub fn shard_of(elem: T, sharding: usize) -> usize { pub struct Radix { sharding: usize, + /// Shard bias — dịch dải shard của trie này sang một vùng khác để nhiều + /// trie dùng chung một storage không đè root/shortcut của nhau. + shard_bias: usize, /// Storage handle. `Radix` chỉ gọi method của `CategoryStorage` + một vài /// method của `NodeMetaStorage` / `ShortcutsStorage` / `BloomStorage`; nhưng /// cùng một `Arc` được `Search` dùng cho 5 trait phụ — nhận `Storage` (umbrella) @@ -182,9 +185,15 @@ pub struct Radix { } impl Radix { + /// Gán shard bias (phải gọi trước khi insert/search bất kỳ). + pub fn set_shard_bias(&mut self, bias: usize) { + self.shard_bias = bias; + } + pub fn new(sharding: usize, storage: Arc>) -> Self { Self { sharding: sharding.max(1), + shard_bias: 0, storage, on_node: None, on_split: None, @@ -253,7 +262,7 @@ impl Radix { .storage .read() .await - .get_root(shard_of(prefix[0], self.sharding)) + .get_root(shard_of(prefix[0], self.sharding) + self.shard_bias) .await?; while node_id != storage::EMPTY { @@ -326,7 +335,7 @@ impl Radix { .await .new_node(Self::from_vec(&prefix[..1]), storage::EMPTY) .await?; - let si = shard_of(prefix[0], self.sharding); + let si = shard_of(prefix[0], self.sharding) + self.shard_bias; self.storage.write().await.set_root(si, root).await?; let leaf = self.extend(root, &prefix[1..], index).await?; self.storage @@ -343,7 +352,7 @@ impl Radix { .await .new_node(Self::from_vec(prefix), index) .await?; - let si = shard_of(prefix[0], self.sharding); + let si = shard_of(prefix[0], self.sharding) + self.shard_bias; self.storage.write().await.set_root(si, id).await?; self.maintain_bloom(prefix).await?; Ok((id, 0)) @@ -403,7 +412,7 @@ impl Radix { self.storage .read() .await - .get_root(shard_of(prefix[0], self.sharding)) + .get_root(shard_of(prefix[0], self.sharding) + self.shard_bias) .await? } else { begin @@ -466,7 +475,7 @@ impl Radix { .storage .read() .await - .get_root(shard_of(key[0], self.sharding)) + .get_root(shard_of(key[0], self.sharding) + self.shard_bias) .await?; if node_id == storage::EMPTY { return Ok(Vec::new()); @@ -518,7 +527,7 @@ impl Radix { self.storage .read() .await - .get_root(shard_of(prefix[0], self.sharding)) + .get_root(shard_of(prefix[0], self.sharding) + self.shard_bias) .await? } else { begin @@ -628,7 +637,7 @@ impl Radix { self.storage .read() .await - .get_root(shard_of(pattern[0], self.sharding)) + .get_root(shard_of(pattern[0], self.sharding) + self.shard_bias) .await? } else { begin diff --git a/crates/codegraph-graph/src/search.rs b/crates/codegraph-graph/src/search.rs index 4eebc3773..253c049aa 100644 --- a/crates/codegraph-graph/src/search.rs +++ b/crates/codegraph-graph/src/search.rs @@ -260,6 +260,10 @@ type PendingSplitElems = Vec<(usize, Vec)>; /// trait object. pub struct Search { sharding: usize, + /// Shard bias — đồng bộ với `Radix::shard_bias`, dùng khi tính shard cho + /// shortcut lookup. Nhiều `Search` dùng chung một storage phải có bias + /// khác nhau (khác nhau ≥ sharding) để không đè root/shortcut của nhau. + shard_bias: usize, trie: Radix, storage: Arc>, @@ -274,10 +278,21 @@ pub struct Search { impl Search { pub fn new(sharding: usize, storage: Arc>) -> Self { + Self::with_shard_bias(sharding, storage, 0) + } + + /// Như `new` nhưng dịch dải shard của trie sang `bias * sharding` — + /// dùng khi nhiều trie chia sẻ cùng một storage (document projections). + pub fn with_shard_bias( + sharding: usize, + storage: Arc>, + bias: usize, + ) -> Self { let sharding = sharding.max(1); let pending_split_elems = Arc::new(Mutex::new(Vec::new())); let mut trie = Radix::new(sharding, storage.clone()); + trie.set_shard_bias(bias * sharding); // Mặc định: mọi element có meta khi insert_chain được lưu vào node // stream keyed theo chính element id (chain model: element id = node @@ -306,6 +321,7 @@ impl Search { Self { sharding, + shard_bias: bias * sharding, trie, storage, pending_split_elems, @@ -398,7 +414,7 @@ impl Search { let mut storage = self.storage.write().await; for (leg_id, elem_bytes) in pending { let elem = T::decode(&elem_bytes); - let si = radix::shard_of(elem, self.sharding); + let si = radix::shard_of(elem, self.sharding) + self.shard_bias; storage.add_shortcut_node(si, &elem_bytes, leg_id).await?; } storage.set_key_len(index, key.len()).await?; @@ -431,7 +447,7 @@ impl Search { let mut storage = self.storage.write().await; for elem in key.iter().skip(breakpoint) { - let si = radix::shard_of(*elem, self.sharding); + let si = radix::shard_of(*elem, self.sharding) + self.shard_bias; storage .add_shortcut_node(si, &elem.encode(), node_id) .await?; @@ -504,7 +520,7 @@ impl Search { } let first_elem = pattern[0]; - let si = radix::shard_of(first_elem, self.sharding); + let si = radix::shard_of(first_elem, self.sharding) + self.shard_bias; // Query candidates trực tiếp từ storage (deterministic per snapshot — // resume chỉ cần cand_idx, không cần lưu candidates). diff --git a/crates/codegraph-mcp/src/lib.rs b/crates/codegraph-mcp/src/lib.rs index 21cf12067..7df78379e 100644 --- a/crates/codegraph-mcp/src/lib.rs +++ b/crates/codegraph-mcp/src/lib.rs @@ -300,7 +300,52 @@ impl CodegraphServer { None, ) })?; - tools::dispatch_doc_hydrate(doc_graph, node_id) + let max_depth = args + .get("max_depth") + .and_then(|v| v.as_u64()) + .map(|v| v as usize); + tools::dispatch_doc_hydrate(doc_graph, node_id, max_depth) + .await + .map_err(|e| McpError::internal_error(e.to_string(), None)) + .map(|text| ToolOutput::Text { + text, + source_bytes: 0, + }) + } + "codegraph_doc_search_value" => { + let query = args.get("query").and_then(|v| v.as_str()).ok_or_else(|| { + McpError::invalid_params( + "codegraph_doc_search_value requires `query`", + None, + ) + })?; + let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(20) as usize; + tools::dispatch_doc_search_value(doc_graph, query, limit) + .await + .map_err(|e| McpError::internal_error(e.to_string(), None)) + .map(|text| ToolOutput::Text { + text, + source_bytes: 0, + }) + } + "codegraph_doc_ingest_dir" => { + let path = args.get("path").and_then(|v| v.as_str()).ok_or_else(|| { + McpError::invalid_params("codegraph_doc_ingest_dir requires `path`", None) + })?; + let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(500) as usize; + tools::dispatch_doc_ingest_dir(doc_graph, path, limit) + .await + .map_err(|e| McpError::internal_error(e.to_string(), None)) + .map(|text| ToolOutput::Text { + text, + source_bytes: 0, + }) + } + "codegraph_doc_remove" => { + let doc_id = args.get("doc_id").and_then(|v| v.as_u64()).ok_or_else(|| { + McpError::invalid_params("codegraph_doc_remove requires `doc_id`", None) + })?; + tools::dispatch_doc_remove(doc_graph, doc_id) .await .map_err(|e| McpError::internal_error(e.to_string(), None)) .map(|text| ToolOutput::Text { diff --git a/crates/codegraph-mcp/src/tools.rs b/crates/codegraph-mcp/src/tools.rs index 193e9968f..167be2cfd 100644 --- a/crates/codegraph-mcp/src/tools.rs +++ b/crates/codegraph-mcp/src/tools.rs @@ -288,22 +288,31 @@ fn tool_defs() -> Vec { ), tool( "codegraph_doc_search", - "Search document nodes by path pattern. Returns matching node IDs and their hydrated payloads.", + "Search document nodes by dotted key path (e.g. `spec.replicas` matches nodes under any `spec` → `replicas` chain across all ingested documents). Returns matching node IDs with path, key and value.", json!({ "type": "object", "properties": { - "pattern": { "type": "string", "description": "Search pattern (substring match on path tokens)." }, - "depth": { "type": "integer", "default": 1, "description": "Search depth." } + "pattern": { "type": "string", "description": "Dotted key path, e.g. `spec.replicas`. Only the last segments need to match at increasing depth." }, + "depth": { "type": "integer", "default": 1, "description": "Search depth (extra levels below the pattern where the chain may still match)." } }, "required": ["pattern"] }), ), + tool( + "codegraph_doc_search_value", + "Search document nodes whose scalar value (string/number) contains the query substring, case-insensitive. Good for finding images, hosts, ports across Kubernetes manifests / Terraform files.", + json!({ "type": "object", "properties": { + "query": { "type": "string", "description": "Value substring to search, e.g. `nginx`." }, + "limit": { "type": "integer", "default": 20, "description": "Max results." } + }, "required": ["query"] }), + ), tool( "codegraph_doc_hydrate", "Hydrate a document node into a small payload suitable for LLM reasoning (path, kind, value, key, children).", json!({ "type": "object", "properties": { - "node_id": { "type": "integer", "description": "Node id to hydrate." } + "node_id": { "type": "integer", "description": "Node id to hydrate." }, + "max_depth": { "type": "integer", "description": "Max child levels to include (omit = unlimited). Use a small value (1-3) to keep payloads small on large documents." } }, "required": ["node_id"] }), ), tool( "codegraph_doc_list", - "List all ingested documents with their paths and formats.", + "List all ingested documents with doc id, path, format, root node id and node count.", json!({ "type": "object", "properties": {} }), ), tool( @@ -311,6 +320,21 @@ fn tool_defs() -> Vec { "Show document graph statistics (number of documents and nodes).", json!({ "type": "object", "properties": {} }), ), + tool( + "codegraph_doc_ingest_dir", + "Bulk ingest every document file (.yaml/.yml/.json/.toml/.tf/.hcl) under a directory, recursively. Use `limit` to cap the number of files on large repos.", + json!({ "type": "object", "properties": { + "path": { "type": "string", "description": "Directory to walk recursively." }, + "limit": { "type": "integer", "default": 500, "description": "Max files to ingest." } + }, "required": ["path"] }), + ), + tool( + "codegraph_doc_remove", + "Remove an ingested document (by doc id, see codegraph_doc_list) and its nodes from the graph and indexes.", + json!({ "type": "object", "properties": { + "doc_id": { "type": "integer", "description": "Doc id returned by codegraph_doc_ingest / codegraph_doc_list." } + }, "required": ["doc_id"] }), + ), // ── Binary tools (dataset riêng .codegraph/binary.sqlite — lazy SQL) ── tool( "codegraph_binary_list", @@ -1086,36 +1110,111 @@ pub async fn dispatch_doc_ingest( pub async fn dispatch_doc_search( doc_graph: Arc, - _pattern: &str, + pattern: &str, depth: usize, ) -> Result { - let tokens = vec![DocToken::root()]; - let ids = doc_graph - .graph() - .await - .read() - .await - .search_path(&tokens, Some(depth)) - .await - .map_err(|e| Error::Other(e.to_string()))?; + let graph = doc_graph.graph().await; + let graph = graph.read().await; + // Pattern "spec.replicas" → [root, FIELD(spec), FIELD(replicas)]. + // Chain radix luôn bắt đầu từ root nên pattern phải là full path; + // không match → fallback quét key chứa segment cuối (case-insensitive). + let mut tokens = vec![DocToken::root()]; + let mut unknown_seg = None; + for seg in pattern.split('.') { + match graph.intern_id(seg) { + Some(id) => tokens.push(DocToken::field(id)), + None => { + unknown_seg = Some(seg.to_string()); + break; + } + } + } + let ids = if unknown_seg.is_none() { + graph + .search_path(&tokens, Some(depth)) + .await + .unwrap_or_default() + } else { + Vec::new() + }; if ids.is_empty() { - return Ok("no nodes matched".to_string()); + // Fallback: quét key theo segment cuối của pattern. + let last = pattern.rsplit('.').next().unwrap_or(pattern); + let hits = graph.search_key_substring(last, 100); + if hits.is_empty() { + return Ok(format!( + "no nodes matched — key `{last}` not seen in any ingested document" + )); + } + let results: Vec = hits + .iter() + .map(|n| { + json!({ + "id": n.id, + "doc": n.doc, + "key": n.key, + "kind": format!("{:?}", n.kind), + "value": n.value, + }) + }) + .collect(); + return serde_json::to_string_pretty(&results).map_err(|e| Error::Other(e.to_string())); } - let graph = doc_graph.graph().await; let mut results = Vec::new(); - for id in &ids { - if let Some(payload) = graph.read().await.hydrate(*id).await { - results.push(json!({ "id": payload.id, "path": payload.path, "kind": format!("{:?}", payload.kind), "value": payload.value })); + for id in ids.iter().take(100) { + if let Some(payload) = graph.hydrate_depth(*id, Some(1)).await { + results.push(json!({ + "id": payload.id, + "doc": payload.doc, + "path": payload.path, + "key": payload.key, + "index": payload.index, + "kind": format!("{:?}", payload.kind), + "value": payload.value, + })); } } serde_json::to_string_pretty(&results).map_err(|e| Error::Other(e.to_string())) } +pub async fn dispatch_doc_search_value( + doc_graph: Arc, + query: &str, + limit: usize, +) -> Result { + let graph = doc_graph.graph().await; + let graph = graph.read().await; + let hits = graph.search_value_substring(query, limit); + if hits.is_empty() { + return Ok(format!("no scalar values matched `{query}`")); + } + let results: Vec = hits + .iter() + .map(|n| { + json!({ + "id": n.id, + "doc": n.doc, + "key": n.key, + "kind": format!("{:?}", n.kind), + "value": n.value, + }) + }) + .collect(); + serde_json::to_string_pretty(&results).map_err(|e| Error::Other(e.to_string())) +} + pub async fn dispatch_doc_hydrate( doc_graph: Arc, node_id: u64, + max_depth: Option, ) -> Result { - let payload = doc_graph.graph().await.read().await.hydrate(node_id).await; + let payload = doc_graph + .graph() + .await + .read() + .await + .hydrate_depth(node_id, max_depth) + .await; match payload { Some(p) => { let json = serde_json::to_string_pretty(&p).map_err(|e| Error::Other(e.to_string()))?; @@ -1126,15 +1225,82 @@ pub async fn dispatch_doc_hydrate( } pub async fn dispatch_doc_list(doc_graph: Arc) -> Result { - let stats = doc_graph + let graph = doc_graph.graph().await; + let graph = graph.read().await; + let infos = graph.list_docs(); + serde_json::to_string_pretty(&infos).map_err(|e| Error::Other(e.to_string())) +} + +/// Ingest hàng loạt mọi file document (theo extension) trong thư mục +/// `path` (đệ quy). `limit` chặn số file — tránh nghẹn graph khi trỏ vào +/// repo lớn; trả về tổng kết. +pub async fn dispatch_doc_ingest_dir( + doc_graph: Arc, + path: &str, + limit: usize, +) -> Result { + const EXTS: [&str; 6] = ["yaml", "yml", "json", "toml", "tf", "hcl"]; + let mut files = Vec::new(); + let mut stack = vec![std::path::PathBuf::from(path)]; + while let Some(dir) = stack.pop() { + let entries = match std::fs::read_dir(&dir) { + Ok(e) => e, + Err(_) => continue, + }; + for entry in entries.flatten() { + let p = entry.path(); + if p.is_dir() { + stack.push(p); + } else if p + .extension() + .and_then(|e| e.to_str()) + .is_some_and(|ext| EXTS.contains(&ext.to_ascii_lowercase().as_str())) + { + files.push(p); + } + } + } + files.sort(); + if files.len() > limit { + files.truncate(limit); + } + let total = files.len(); + let mut ingested = 0usize; + let mut failed = Vec::new(); + for f in &files { + let Some(p) = f.to_str() else { continue }; + match doc_graph + .graph() + .await + .write() + .await + .ingest_file(p, None) + .await + { + Ok(_) => ingested += 1, + Err(e) => failed.push(format!("{}: {e}", f.display())), + } + } + let mut summary = json!({ "requested": total, "ingested": ingested, "failed": failed.len() }); + if !failed.is_empty() { + summary["errors"] = json!(failed.iter().take(10).collect::>()); + } + serde_json::to_string_pretty(&summary).map_err(|e| Error::Other(e.to_string())) +} + +pub async fn dispatch_doc_remove( + doc_graph: Arc, + doc_id: u64, +) -> Result { + doc_graph .graph() .await - .read() + .write() .await - .stats() + .remove_document(doc_id) .await .map_err(|e| Error::Other(e.to_string()))?; - Ok(format!("documents: {}, nodes: {}", stats.docs, stats.nodes)) + Ok(format!("removed doc {doc_id}")) } pub async fn dispatch_doc_stats(doc_graph: Arc) -> Result { diff --git a/crates/codegraph/src/main.rs b/crates/codegraph/src/main.rs index 3a461e1a8..7c7ac5161 100644 --- a/crates/codegraph/src/main.rs +++ b/crates/codegraph/src/main.rs @@ -158,9 +158,10 @@ impl OutputFormat { enum DocCmd { /// Parse and ingest a document file (HCL, YAML, JSON, TOML, XML). Ingest { - /// Path to the document file. + /// Path to the document file. Đặt tên `file` — positional `path` đụng + /// global `--path` (Utf8PathBuf parser) làm clap panic khi parse args. #[arg()] - path: String, + file: String, /// Override auto-detected format (hcl, yaml, json, toml, nginx). #[arg(long)] format: Option, @@ -730,21 +731,43 @@ async fn cmd_doc(root: &Utf8Path, cmd: DocCmd) -> Result<()> { let mut graph = open_doc_graph(root).await?; match cmd { - DocCmd::Ingest { path, format } => { - let inserted = graph.ingest_file(&path, format.as_deref()).await?; - println!("ingested {path} → doc_id={inserted}"); + DocCmd::Ingest { file, format } => { + let inserted = graph.ingest_file(&file, format.as_deref()).await?; + println!("ingested {file} → doc_id={inserted}"); } - DocCmd::Search { pattern: _, depth } => { + DocCmd::Search { pattern, depth } => { use codegraph_docs::DocToken; - let tokens = vec![DocToken::root(), DocToken::field(0)]; // simplified - let ids = graph.search_path(&tokens, Some(depth)).await?; + // Full path search qua trie; không match → fallback quét key. + let mut tokens = vec![DocToken::root()]; + for seg in pattern.split('.') { + match graph.intern_id(seg) { + Some(id) => tokens.push(DocToken::field(id)), + None => break, + } + } + let ids = graph + .search_path(&tokens, Some(depth)) + .await + .unwrap_or_default(); if ids.is_empty() { - println!("no nodes matched"); - } else { - for id in &ids { - if let Some(payload) = graph.hydrate(*id).await { - println!("{}: {:?}", id, payload); - } + let last = pattern.rsplit('.').next().unwrap_or(&pattern); + let hits = graph.search_key_substring(last, 100); + if hits.is_empty() { + println!("no nodes matched — key `{last}` not seen in any ingested document"); + return Ok(()); + } + for n in hits { + println!( + "node {} doc={} key={:?} kind={:?} value={:?}", + n.id, n.doc, n.key, n.kind, n.value + ); + } + return Ok(()); + } + for id in ids.iter().take(100) { + if let Some(payload) = graph.hydrate_depth(*id, Some(1)).await { + let json = serde_json::to_string_pretty(&payload)?; + println!("{json}"); } } } diff --git a/packaging/aur/codegraph-rs-bin/PKGBUILD b/packaging/aur/codegraph-rs-bin/PKGBUILD index 657cf7b45..84d03b872 100644 --- a/packaging/aur/codegraph-rs-bin/PKGBUILD +++ b/packaging/aur/codegraph-rs-bin/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: Hung Pham pkgname=codegraph-rs-bin -pkgver=2.1.7 +pkgver=2.1.8 pkgrel=1 pkgdesc="Local-first code intelligence: tree-sitter knowledge graph + MCP server (prebuilt binary)" arch=('x86_64' 'aarch64') diff --git a/packaging/choco/codegraph.nuspec b/packaging/choco/codegraph.nuspec index a9821eb6d..ea24a2f41 100644 --- a/packaging/choco/codegraph.nuspec +++ b/packaging/choco/codegraph.nuspec @@ -2,7 +2,7 @@ codegraph - 2.1.7 + 2.1.8 codegraph Hung Pham https://github.com/hungpham10/codegraph-rs diff --git a/packaging/winget/codegraph.yaml b/packaging/winget/codegraph.yaml index ecfd8f4e8..fbebf44b0 100644 --- a/packaging/winget/codegraph.yaml +++ b/packaging/winget/codegraph.yaml @@ -6,7 +6,7 @@ # release time (or automate it in the release pipeline before submitting to # microsoft/winget-pkgs). PackageIdentifier: hungpham10.codegraph -PackageVersion: 2.1.7 +PackageVersion: 2.1.8 PackageName: codegraph Publisher: Hung Pham PublisherUrl: https://github.com/hungpham10/codegraph-rs @@ -18,7 +18,7 @@ PackageUrl: https://github.com/hungpham10/codegraph-rs InstallerType: zip Installers: - Architecture: x64 - InstallerUrl: https://github.com/hungpham10/codegraph-rs/releases/download/v2.1.7/codegraph-x86_64-pc-windows-msvc.zip + InstallerUrl: https://github.com/hungpham10/codegraph-rs/releases/download/v2.1.8/codegraph-x86_64-pc-windows-msvc.zip InstallerSha256: 0000000000000000000000000000000000000000000000000000000000000000 InstallerType: zip ManifestType: singleton diff --git a/scripts/install.ps1 b/scripts/install.ps1 index 75cc24b8b..5e0f98ffb 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -5,11 +5,11 @@ # # Usage (pin a version / download the script first): # irm https://raw.githubusercontent.com/hungpham10/codegraph-rs/main/scripts/install.ps1 -OutFile install.ps1 -# .\install.ps1 -Version 2.1.7 +# .\install.ps1 -Version 2.1.8 [CmdletBinding()] param( - # Pin a specific version, e.g. "2.1.7". Empty = latest release. + # Pin a specific version, e.g. "2.1.8". Empty = latest release. [string]$Version )