From 0ddd30c59e1359f2de5dd57a5db1e56bfc9bac17 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Sat, 12 Sep 2026 06:23:11 +0700 Subject: [PATCH 1/6] Persistent document graph into disk --- Cargo.lock | 3 + crates/codegraph-binary/src/extract.rs | 125 +++++++++- crates/codegraph-binary/src/model.rs | 12 + crates/codegraph-docs/Cargo.toml | 4 + crates/codegraph-docs/src/graph.rs | 198 +++++++++++++--- crates/codegraph-docs/src/lib.rs | 1 + crates/codegraph-docs/src/parsers/mod.rs | 30 +++ crates/codegraph-extract/Cargo.toml | 3 + crates/codegraph-extract/src/config.rs | 276 ++++++++++++++++++++++- crates/codegraph-extract/src/docgraph.rs | 41 ++++ crates/codegraph-extract/src/lib.rs | 4 +- crates/codegraph-graph/src/lib.rs | 60 +++++ crates/codegraph-mcp/src/lib.rs | 19 +- crates/codegraph-mcp/src/tools.rs | 34 +-- crates/codegraph/src/main.rs | 76 +++---- 15 files changed, 770 insertions(+), 116 deletions(-) create mode 100644 crates/codegraph-extract/src/docgraph.rs diff --git a/Cargo.lock b/Cargo.lock index ea77da5f5..03ec8a20b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -827,6 +827,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "tempfile", "tokio", "toml", "toml_edit 0.22.27", @@ -839,8 +840,10 @@ dependencies = [ "camino", "codegraph-binary", "codegraph-core", + "codegraph-docs", "codegraph-graph", "getrandom 0.2.17", + "glob", "ignore", "indicatif", "rayon", diff --git a/crates/codegraph-binary/src/extract.rs b/crates/codegraph-binary/src/extract.rs index 04a34ee1c..f5ba3b008 100644 --- a/crates/codegraph-binary/src/extract.rs +++ b/crates/codegraph-binary/src/extract.rs @@ -70,6 +70,13 @@ fn do_extract( // 1. Functions (`aflj`) let functions = parse_aflj(session)?; + // Parse exports (`iEj`) for JNI address-based detection (catches stripped binaries). + let exports = parse_iej(session)?; + let jni_export_map: HashMap = exports + .iter() + .filter(|e| is_jni_name(e.name.as_deref().unwrap_or(""))) + .filter_map(|e| e.vaddr.map(|v| (v, e.name.clone().unwrap_or_default()))) + .collect(); let mut symbols: Vec = Vec::new(); let mut chains: HashMap> = HashMap::new(); let mut calls: Vec = Vec::new(); @@ -96,6 +103,15 @@ fn do_extract( fn_id_to_name.insert(id, name.clone()); // r2 6.x tự sinh symbol C++: class.X, method.Class.foo, namespace.X, enum.X let (kind, name) = classify_symbol(&raw_name, &name); + // JNI enrichment: name-based + address-based (via iEj export table). + let mut annotations = Vec::new(); + if is_jni_name(&name) || jni_export_map.contains_key(&addr) { + annotations.push(Annotation { + name: "jni".to_string(), + args: HashMap::new(), + line: 0, + }); + } symbols.push(Symbol { id, name, @@ -109,7 +125,7 @@ fn do_extract( end_line: addr.saturating_add(size).try_into().unwrap_or(u32::MAX), signature: Some(sig), doc: None, - annotations: Vec::new(), + annotations, language: "binary".to_string(), }); } @@ -289,6 +305,19 @@ fn classify_symbol(raw_name: &str, name: &str) -> (SymbolKind, String) { (SymbolKind::Function, name.to_string()) } +/// Kiểm tra tên có phải là JNI symbol không. +/// Java_* — JNI native method naming convention. +/// JNI_* — JNI runtime functions. +fn is_jni_name(name: &str) -> bool { + name.starts_with("Java_") || name.starts_with("JNI_") +} + +/// Parse exported symbols từ `iEj` để phát hiện JNI symbol qua address matching. +/// Trả về danh sách symbol xuất khẩu có tên bắt đầu bằng Java_ hoặc JNI_. +fn parse_iej(session: &mut dyn R2Client) -> Result, Error> { + parse_array(session.cmdj("iEj")?) +} + /// Bản đồ tra cứu từ address/name sang symbol id — gom parameter cho chain builder. struct FnMaps<'a> { fn_by_addr: &'a HashMap, @@ -471,6 +500,7 @@ fn resolve_call_target(target: Option, maps: &FnMaps) -> (u64, String) { mod tests { use super::*; use codegraph_core::SymbolKind; + use serde_json::json; #[test] fn test_classify_symbol_class() { @@ -575,4 +605,97 @@ mod tests { assert_eq!(kind, SymbolKind::Function); assert_eq!(cleaned_name, name); } + + // === JNI tests === + + #[test] + fn test_is_jni_name_java_prefix() { + assert!(is_jni_name("Java_com_example_Foo_bar")); + assert!(is_jni_name("Java_org_example_Baz_qux")); + } + + #[test] + fn test_is_jni_name_jni_prefix() { + assert!(is_jni_name("JNI_OnLoad")); + assert!(is_jni_name("JNI_OnUnload")); + assert!(is_jni_name("JNI_RegisterNatives")); + assert!(is_jni_name("JNI_CreateJavaVM")); + } + + #[test] + fn test_is_jni_name_not_jni() { + assert!(!is_jni_name("fcn.00401000")); + assert!(!is_jni_name("sub_1234")); + assert!(!is_jni_name("main")); + assert!(!is_jni_name("sym.imp.puts")); + } + + #[test] + fn test_parse_iej_jni_detection() { + // Mock R2Client returning iEj with JNI exports + struct MockR2 { + responses: HashMap, + } + impl R2Client for MockR2 { + fn cmd(&mut self, _cmd: &str) -> Result { Ok(String::new()) } + fn cmdj(&mut self, cmd: &str) -> Result { + Ok(self.responses.get(cmd).cloned().unwrap_or(json!([]))) + } + } + let mut mock = MockR2 { + responses: HashMap::new(), + }; + mock.responses.insert( + "iEj".to_string(), + json!([ + {"name": "JNI_OnLoad", "vaddr": 4194304, "bind": "GLOBAL", "type": "FUNC"}, + {"name": "Java_com_example_Foo_bar", "vaddr": 4194368, "bind": "GLOBAL", "type": "FUNC"}, + {"name": "free", "vaddr": 4194432, "bind": "GLOBAL", "type": "FUNC"} + ]), + ); + let exports = parse_iej(&mut mock).unwrap(); + assert_eq!(exports.len(), 3); + assert!(exports.iter().any(|e| e.name.as_deref() == Some("JNI_OnLoad"))); + assert!(exports.iter().any(|e| e.name.as_deref() == Some("Java_com_example_Foo_bar"))); + } + + #[test] + fn test_parse_iej_empty() { + struct MockR2 { + responses: HashMap, + } + impl R2Client for MockR2 { + fn cmd(&mut self, _cmd: &str) -> Result { Ok(String::new()) } + fn cmdj(&mut self, cmd: &str) -> Result { + Ok(self.responses.get(cmd).cloned().unwrap_or(json!([]))) + } + } + let mut mock = MockR2 { + responses: HashMap::new(), + }; + mock.responses.insert("iEj".to_string(), json!([])); + let exports = parse_iej(&mut mock).unwrap(); + assert!(exports.is_empty()); + } + + #[test] + fn test_jni_annotation_name_based() { + // Java_com_* name should produce jni annotation via is_jni_name + let raw = "Java_com_example_Foo_bar"; + let name = "Java_com_example_Foo_bar"; + let (kind, cleaned_name) = classify_symbol(raw, name); + assert_eq!(kind, SymbolKind::Function); + assert!(is_jni_name(&cleaned_name)); + } + + #[test] + fn test_jni_annotation_address_based() { + // JNI_OnLoad in iEj export table should match by address + use std::collections::HashMap; + let mut jni_export_map: HashMap = HashMap::new(); + jni_export_map.insert(4194304, "JNI_OnLoad".to_string()); + let addr = 4194304u64; + assert!(jni_export_map.contains_key(&addr)); + // The function with this addr would get jni annotation even if r2 renamed it + } } diff --git a/crates/codegraph-binary/src/model.rs b/crates/codegraph-binary/src/model.rs index 3eb5e92b2..644d563aa 100644 --- a/crates/codegraph-binary/src/model.rs +++ b/crates/codegraph-binary/src/model.rs @@ -95,6 +95,18 @@ pub struct SymEntry { pub is_imported: Option, } +/// Symbol xuất khẩu từ `iEj`. +#[derive(Debug, Deserialize)] +pub struct ExportEntry { + pub name: Option, + pub vaddr: Option, + pub paddr: Option, + pub size: Option, + pub bind: Option, + #[serde(rename = "type")] + pub type_: Option, +} + /// String từ `izj` / `izzj`. #[derive(Debug, Deserialize)] pub struct StrEntry { diff --git a/crates/codegraph-docs/Cargo.toml b/crates/codegraph-docs/Cargo.toml index 3f7d2b3fb..535e615fd 100644 --- a/crates/codegraph-docs/Cargo.toml +++ b/crates/codegraph-docs/Cargo.toml @@ -20,3 +20,7 @@ toml_edit = { workspace = true } hcl-rs = "0.19.8" tokio = { workspace = true, features = ["sync"] } anyhow = { workspace = true } + +[dev-dependencies] +tempfile = "3" +tokio = { workspace = true, features = ["sync", "macros", "rt"] } diff --git a/crates/codegraph-docs/src/graph.rs b/crates/codegraph-docs/src/graph.rs index 4897fcf44..be65e67db 100644 --- a/crates/codegraph-docs/src/graph.rs +++ b/crates/codegraph-docs/src/graph.rs @@ -42,7 +42,12 @@ pub struct DocumentGraph { type_trie: Search, value_trie: Search, struct_trie: Search, + /// Pattern-mining trie — reserve cho tính năng mined patterns, chưa có + /// reader (trước đây chỉ được clear trong rebuild). + #[allow(dead_code)] pattern_trie: Search, + /// Base id global cho node/doc — id nhỏ hơn đây là id local của parser. + doc_base: u64, next_doc_id: u64, next_node_id: u64, } @@ -63,6 +68,7 @@ impl DocumentGraph { value_trie: Search::new(sharding, storage.clone()), struct_trie: Search::new(sharding, storage.clone()), pattern_trie: Search::new(sharding, storage.clone()), + doc_base, next_doc_id: doc_base, next_node_id: doc_base, } @@ -72,9 +78,32 @@ impl DocumentGraph { pub async fn open(storage: Arc>, config: DocConfig) -> Result { let mut graph = Self::new(storage, config); graph.rebuild().await?; + // Resume id counters từ trạng thái đã persist — reset về `doc_base` + // sẽ đè lên id cũ khi ingest tiếp. + let max_doc = graph.docs.keys().copied().max().unwrap_or(0); + let max_node = graph.nodes.keys().copied().max().unwrap_or(0); + graph.next_doc_id = graph.next_doc_id.max(max_doc + 1); + graph.next_node_id = graph.next_node_id.max(max_node + 1); Ok(graph) } + /// Ingest một file từ disk: đọc, detect format theo extension (override + /// bằng `format`), parse rồi upsert. Trùng `path` với doc đã có → thay thế + /// tại chỗ (re-ingest khi chạy lại `codegraph init` là idempotent). + pub async fn ingest_file(&mut self, path: &str, format: Option<&str>) -> Result { + let source = std::fs::read_to_string(path) + .map_err(|e| anyhow::anyhow!("failed to read {path}: {e}"))?; + let format = match format { + Some(f) => f.to_string(), + None => crate::parsers::detect_format(path)?, + }; + let existing = self.docs.values().find(|d| d.path == path).map(|d| d.id); + let doc_id = existing.unwrap_or(0); + let parser = crate::parsers::parser_for(&format)?; + let doc = parser.parse(path, &source, doc_id)?; + self.upsert_document(doc).await + } + /// Rebuild all materialized tries from persisted node/doc metadata. pub async fn rebuild(&mut self) -> Result<()> { // Load node list. @@ -120,12 +149,10 @@ impl DocumentGraph { self.docs.insert(doc.id, doc); } } - // Rebuild tries. - self.path_trie.clear().await?; - self.type_trie.clear().await?; - self.value_trie.clear().await?; - self.struct_trie.clear().await?; - self.pattern_trie.clear().await?; + // Rebuild tries (in-memory từ node metadata). KHÔNG dùng `Search::clear` + // — nó xoá toàn bộ `clear_node_meta`/`clear_chains` của storage, xoá cả + // node/doc JSON vừa đọc lên (tries của docs start rỗng từ `new()` nên + // không cần clear persistent state). let nodes: Vec = self.nodes.values().cloned().collect(); for node in nodes { self.insert_node_into_tries(&node).await?; @@ -166,10 +193,16 @@ impl DocumentGraph { .await?; // Update lists. self.add_doc_id(doc_id).await?; + let node_ids: Vec = doc.nodes.iter().map(|n| n.id).collect(); + self.add_node_ids(&node_ids).await?; // Insert into tries. for node in &doc.nodes { self.insert_node_into_tries(node).await?; } + // Materialize nodes vào map in-memory (hydrate/stats đọc từ đây). + for node in &doc.nodes { + self.nodes.insert(node.id, node.clone()); + } self.docs.insert(doc_id, doc.clone()); Ok(doc_id) } @@ -293,7 +326,49 @@ impl DocumentGraph { Ok(Vec::new()) } } + /// Ghi danh sách node id vào chain sentinel — `rebuild()` đọc từ đây để + /// khôi phục `nodes` map khi mở lại graph từ storage. + async fn add_node_ids(&self, node_ids: &[u64]) -> Result<()> { + let mut list = { + let chain = { + let guard = self.storage.read().await; + guard.get_chain(DOC_NODE_LIST_RECORD as usize).await? + }; + chain.map(|c| c.to_vec()).unwrap_or_default() + }; + list.extend_from_slice(node_ids); + self.storage + .write() + .await + .set_chain(DOC_NODE_LIST_RECORD as usize, &list) + .await?; + Ok(()) + } fn assign_node_ids(&mut self, mut doc: Document) -> Document { + // Parser sinh id local (1..N) — remap toàn bộ (kèm parent/children/root) + // sang dải global (≥ `doc_base`) để nhiều doc trong cùng graph không + // đè node của nhau. Doc đã có id global (rebuild/re-upsert) giữ nguyên. + let is_local = doc + .nodes + .first() + .map(|n| n.id < self.doc_base) + .unwrap_or(false); + if is_local { + let offset = self.next_node_id.saturating_sub(1); + if offset > 0 { + for node in &mut doc.nodes { + node.id += offset; + if let Some(p) = node.parent.as_mut() { + *p += offset; + } + for c in &mut node.children { + *c += offset; + } + } + doc.root += offset; + } + self.next_node_id += doc.nodes.len() as u64; + } for node in &mut doc.nodes { if node.id == 0 { node.id = self.next_node_id; @@ -332,36 +407,56 @@ impl DocumentGraph { path.reverse(); path } + /// Insert một token chain vào trie — token rỗng bỏ qua (`insert_chain` với + /// key rỗng là lỗi NotFound), key trùng coi như OK (node trùng path/token + /// với node khác, hoặc re-ingest cùng path — record cũ giữ nguyên). + async fn insert_chain_allow_dup( + trie: &mut Search, + record: usize, + tokens: &[DocToken], + ) -> Result<()> { + if tokens.is_empty() { + return Ok(()); + } + let metas: Vec> = vec![None; tokens.len()]; + if let Err(e) = trie.insert_chain(record, tokens, &metas).await { + if !matches!(e, codegraph_graph::SearchError::Duplicated) { + return Err(anyhow::anyhow!(e.to_string())); + } + } + Ok(()) + } + async fn insert_node_into_tries(&mut self, node: &Node) -> Result<()> { let path_tokens = self.path_tokens(node); let type_tokens = self.type_tokens(node); let value_tokens = self.value_tokens(node); let struct_tokens = self.struct_tokens(node); let node_id = node.id; - { - let trie = &mut self.path_trie; - let record = (PATH_RECORD_BASE + node_id) as usize; - let metas: Vec> = vec![None; path_tokens.len()]; - trie.insert_chain(record, &path_tokens, &metas).await?; - } - { - let trie = &mut self.type_trie; - let record = (TYPE_RECORD_BASE + node_id) as usize; - let metas: Vec> = vec![None; type_tokens.len()]; - trie.insert_chain(record, &type_tokens, &metas).await?; - } - { - let trie = &mut self.value_trie; - let record = (VALUE_RECORD_BASE + node_id) as usize; - let metas: Vec> = vec![None; value_tokens.len()]; - trie.insert_chain(record, &value_tokens, &metas).await?; - } - { - let trie = &mut self.struct_trie; - let record = (STRUCT_RECORD_BASE + node_id) as usize; - let metas: Vec> = vec![None; struct_tokens.len()]; - trie.insert_chain(record, &struct_tokens, &metas).await?; - } + Self::insert_chain_allow_dup( + &mut self.path_trie, + (PATH_RECORD_BASE + node_id) as usize, + &path_tokens, + ) + .await?; + Self::insert_chain_allow_dup( + &mut self.type_trie, + (TYPE_RECORD_BASE + node_id) as usize, + &type_tokens, + ) + .await?; + Self::insert_chain_allow_dup( + &mut self.value_trie, + (VALUE_RECORD_BASE + node_id) as usize, + &value_tokens, + ) + .await?; + Self::insert_chain_allow_dup( + &mut self.struct_trie, + (STRUCT_RECORD_BASE + node_id) as usize, + &struct_tokens, + ) + .await?; Ok(()) } @@ -436,4 +531,47 @@ mod tests { let graph = DocumentGraph::new(storage, config); assert_eq!(graph.stats().docs, 0); } + + /// `ingest_file` hai file khác nhau → doc id khác nhau, node không đè nhau; + /// re-ingest cùng path → cùng doc id (thay thế tại chỗ); `open()` lại từ + /// storage → docs còn nguyên và counter id tiếp tục sau max id cũ. + #[tokio::test] + async fn ingest_file_resume_and_reopen() { + let dir = tempfile::tempdir().unwrap(); + let p1 = dir.path().join("a.yaml"); + let p2 = dir.path().join("b.toml"); + let p3 = dir.path().join("c.json"); + std::fs::write(&p1, "service:\n name: api\n replicas: 3\n").unwrap(); + std::fs::write(&p2, "[service]\nname = \"db\"\n").unwrap(); + std::fs::write(&p3, r#"{"service": {"name": "web"}}"#).unwrap(); + + let storage = Arc::new(TokioRwLock::new(InMemoryStorage::default())); + let mut graph = DocumentGraph::new(storage.clone(), DocConfig::default()); + let d1 = graph.ingest_file(p1.to_str().unwrap(), None).await.unwrap(); + let d2 = graph.ingest_file(p2.to_str().unwrap(), None).await.unwrap(); + assert_ne!(d1, d2); + assert_eq!(graph.stats().docs, 2); + // a.yaml: root+service+name+replicas = 4; b.toml: root+service+name = 3. + // Nếu remap local-id sai thì 2 doc đè node nhau → tổng < 7. + assert_eq!(graph.stats().nodes, 7); + + // Re-ingest cùng path → id giữ nguyên. + assert_eq!( + graph.ingest_file(p1.to_str().unwrap(), None).await.unwrap(), + d1 + ); + + // Reopen từ storage — docs phục hồi, ingest tiếp có id mới (không đè). + let mut reopened = DocumentGraph::open(storage, DocConfig::default()) + .await + .unwrap(); + assert_eq!(reopened.stats().docs, 2); + // Node list được persist — mở lại phải khôi phục đủ node. + assert_eq!(reopened.stats().nodes, 7); + let d3 = reopened + .ingest_file(p3.to_str().unwrap(), None) + .await + .unwrap(); + assert!(d3 > d1 && d3 > d2, "d3={d3} phải sau d1={d1}, d2={d2}"); + } } diff --git a/crates/codegraph-docs/src/lib.rs b/crates/codegraph-docs/src/lib.rs index 5a802e339..1f3d2a744 100644 --- a/crates/codegraph-docs/src/lib.rs +++ b/crates/codegraph-docs/src/lib.rs @@ -6,6 +6,7 @@ pub mod parsers; pub mod tokenize; pub use crate::config::DocConfig; +pub use crate::config::StorageConfig; pub use crate::graph::DocumentGraph; pub use crate::graph::{DocStats, NodePayload}; pub use crate::ir::{ByteSpan, Document, Kind, Node, Scalar}; diff --git a/crates/codegraph-docs/src/parsers/mod.rs b/crates/codegraph-docs/src/parsers/mod.rs index 4f7292a37..37bca4707 100644 --- a/crates/codegraph-docs/src/parsers/mod.rs +++ b/crates/codegraph-docs/src/parsers/mod.rs @@ -385,3 +385,33 @@ service: assert_eq!(doc.nodes.len(), 4); // root, service, name, replicas } } + +/// Detect document format từ extension: `tf`/`hcl` → hcl, `yaml`/`yml`, +/// `json`, `toml`. Lỗi khi extension không nhận diện được. +pub fn detect_format(path: &str) -> Result { + let ext = std::path::Path::new(path) + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.to_lowercase()) + .unwrap_or_default(); + match ext.as_str() { + "tf" | "hcl" => Ok("hcl".to_string()), + "yaml" | "yml" => Ok("yaml".to_string()), + "json" => Ok("json".to_string()), + "toml" => Ok("toml".to_string()), + _ => Err(anyhow::anyhow!( + "unknown format for extension .{ext}; specify --format to override" + )), + } +} + +/// Chọn parser theo format name (`"hcl"`, `"yaml"`, `"json"`, `"toml"`). +pub fn parser_for(format: &str) -> Result> { + match format { + "hcl" => Ok(Box::new(HclParser)), + "yaml" => Ok(Box::new(YamlParser)), + "json" => Ok(Box::new(JsonParser)), + "toml" => Ok(Box::new(TomlParser)), + _ => Err(anyhow::anyhow!("unsupported document format: {format}")), + } +} diff --git a/crates/codegraph-extract/Cargo.toml b/crates/codegraph-extract/Cargo.toml index d53783aa0..97354b9c4 100644 --- a/crates/codegraph-extract/Cargo.toml +++ b/crates/codegraph-extract/Cargo.toml @@ -11,6 +11,9 @@ warnings = "deny" [dependencies] codegraph-core = { path = "../codegraph-core" } codegraph-graph = { path = "../codegraph-graph", features = ["sqlite"] } +codegraph-docs = { path = "../codegraph-docs" } +glob = "0.3" +tokio = { workspace = true } tree-sitter = { workspace = true } tree-sitter-typescript = { workspace = true, optional = true } tree-sitter-javascript = { workspace = true, optional = true } diff --git a/crates/codegraph-extract/src/config.rs b/crates/codegraph-extract/src/config.rs index d30a46c3e..daccef995 100644 --- a/crates/codegraph-extract/src/config.rs +++ b/crates/codegraph-extract/src/config.rs @@ -1,12 +1,14 @@ -use crate::languages::effects::EffectClassifier; -use crate::project::{project_db_path, project_dir}; -use camino::Utf8Path; -#[cfg(feature = "binary")] -pub use codegraph_binary::BinaryConfig; -use codegraph_core::{EffectCallPattern, EffectRule, EffectType, StorageRoute}; +use camino::{Utf8Path, Utf8PathBuf}; use serde::Deserialize; use std::fs; +#[cfg(feature = "binary")] +use codegraph_binary::BinaryConfig; +use codegraph_core::{EffectCallPattern, EffectRule, EffectType, StorageRoute}; + +use crate::languages::effects::EffectClassifier; +use crate::project::{project_db_path, project_dir}; + /// How `.h` header files should be parsed when both C and C++ extractors are available. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum HeaderLanguage { @@ -66,6 +68,10 @@ struct ConfigFile { /// Embedding backend cho semantic search (fastembed / hashing) + cache model. #[serde(default)] embedding: EmbeddingSection, + /// Document graph — ingest tài liệu cấu trúc lúc `codegraph init`. + #[serde(default)] + docgraph: DocGraphSection, + /// Phân tích binary (radare2) — feature `binary`. #[cfg(feature = "binary")] #[serde(default)] @@ -96,6 +102,52 @@ struct LanguagesSection { headers: Option, } +/// Section `[docgraph]` — cấu hình document graph (ingest tài liệu lúc `init`). +#[derive(Debug, Clone, Default, Deserialize)] +pub struct DocGraphSection { + /// Bật ingest docs khi `codegraph init` (mặc định bật khi có `paths`). + #[serde(default)] + enabled: Option, + /// Danh sách glob (tính từ project root), vd `["infra/*.tf", "config/**/*.yaml"]`. + /// Mỗi entry hỗ trợ suffix `:` để override, vd `"deploy/README:hcl"`. + #[serde(default)] + paths: Vec, + /// Override storage cho docs — mặc định dataset riêng cùng backend kind của + /// `[storage]` (sqlite → `.codegraph/docs.sqlite`, lmdb → `docs.lmdb`). + #[serde(default)] + storage: Option, + /// Base id cho node/doc của document graph (mặc định 1e9). + #[serde(default)] + doc_base: Option, + /// Base id cho mined pattern (mặc định 3e9). + #[serde(default)] + pattern_base: Option, + /// Bloom-filter cap (mặc định 64). + #[serde(default)] + bloom_cap: Option, + /// Alias chuẩn hoá key, vd `aliases = [["instances", "replicas"]]`. + #[serde(default)] + aliases: Vec<(String, String)>, +} + +/// `[docgraph.storage]` — override backend/dsn cho document graph. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct DocGraphStorageSection { + /// `"sqlite"`, `"lmdb"`, `"redis"`, `"memory"`. + #[serde(default, rename = "type")] + pub type_: Option, + /// DSN override (vd `sqlite:///tmp/docs.db`). + #[serde(default)] + pub dsn: Option, +} + +impl DocGraphSection { + /// Ingest docs có bật hay không: `enabled` override, mặc định = có `paths`. + pub fn is_enabled(&self) -> bool { + self.enabled.unwrap_or(!self.paths.is_empty()) + } +} + #[derive(Debug, Default, Deserialize)] struct EmbeddingSection { /// `"fastembed"` | `"hashing"`. @@ -139,6 +191,8 @@ pub struct ExtractConfig { pub storage: StorageConfig, /// Cấu hình embedding backend (semantic search) — đọc từ `[embedding]`. pub embedding: codegraph_graph::embeddings::EmbeddingConfig, + /// Cấu hình document graph — đọc từ `[docgraph]`. + pub docgraph: DocGraphSection, /// Cấu hình phân tích binary (radare2). #[cfg(feature = "binary")] pub binary: BinaryConfig, @@ -191,6 +245,7 @@ impl ExtractConfig { repo_id: file.storage.repo_id, dsns: file.storage.dsns, }, + docgraph: file.docgraph, #[cfg(feature = "binary")] binary: file.binary.unwrap_or_default(), } @@ -284,6 +339,115 @@ impl ExtractConfig { } Some(repo_id) } + + /// DSN dataset **riêng** cho document graph (tries của docs đụng namespace + /// record/shard với code index nên KHÔNG dùng chung 1 dataset được — dùng + /// cùng backend kind nhưng file/keyspace riêng). + /// + /// - `[docgraph.storage] dsn` override → dùng nguyên văn. + /// - Mặc định theo backend kind (override được bằng `[docgraph.storage] type`): + /// - sqlite → `sqlite:///.codegraph/docs.sqlite` + /// - lmdb → `lmdb:///.codegraph/docs.lmdb` + /// - redis → DSN của `[storage]` (helper mở keyspace prefix riêng) + /// - memory → `None` (in-memory) + /// - postgres/mysql → chưa hỗ trợ dataset riêng → `None` + pub fn doc_storage_dsn(&self, root: &Utf8Path) -> Option { + if let Some(dsn) = self + .docgraph + .storage + .as_ref() + .and_then(|s| s.dsn.as_deref()) + { + return Some(dsn.to_string()); + } + let kind = self + .docgraph + .storage + .as_ref() + .and_then(|s| s.type_.as_deref()) + .map(StorageKind::parse) + .unwrap_or(self.storage.kind); + match kind { + StorageKind::Sqlite => { + Some(format!("sqlite://{}", project_dir(root).join("docs.sqlite"))) + } + StorageKind::Lmdb => { + Some(format!("lmdb://{}", project_dir(root).join("docs.lmdb"))) + } + StorageKind::Redis => self.storage.dsn.clone(), + StorageKind::Memory | StorageKind::Postgres | StorageKind::MySql => None, + } + } + + /// Config document graph + danh sách file khớp glob `[docgraph] paths` + /// (path kèm format override). Trả `None` khi `[docgraph]` không bật / + /// không khai báo `paths`. + pub fn doc_config( + &self, + root: &Utf8Path, + ) -> Option<( + codegraph_docs::DocConfig, + Vec<(Utf8PathBuf, Option)>, + )> { + if !self.docgraph.is_enabled() { + return None; + } + let mut files: Vec<(Utf8PathBuf, Option)> = Vec::new(); + for entry in &self.docgraph.paths { + let (pattern, format) = split_format_override(entry); + let full = root.join(pattern).to_string(); + let Ok(matches) = glob::glob(&full) else { + tracing::warn!("[docgraph] glob `{pattern}` không hợp lệ — bỏ qua"); + continue; + }; + for path in matches.flatten() { + if !path.is_file() { + continue; + } + let Ok(path) = Utf8PathBuf::from_path_buf(path) else { + tracing::warn!("[docgraph] path không phải UTF-8 — bỏ qua"); + continue; + }; + if !files.iter().any(|(p, _)| *p == path) { + files.push((path, format.map(str::to_string))); + } + } + } + // Không có file nào khớp → coi như không cấu hình (init bỏ qua ingest). + if files.is_empty() { + return None; + } + let dsn = self.doc_storage_dsn(root); + if dsn.is_none() && self.storage.kind.is_rdbms() { + tracing::warn!( + "[docgraph] backend RDBMS chưa hỗ trợ dataset riêng cho docs — \ + dùng in-memory (override bằng [docgraph.storage] dsn)" + ); + } + let config = codegraph_docs::DocConfig { + storage: dsn.map(|dsn| codegraph_docs::StorageConfig { + r#type: Some(dsn.split("://").next().unwrap_or("sqlite").to_string()), + dsn: Some(dsn), + }), + doc_base: self.docgraph.doc_base, + pattern_base: self.docgraph.pattern_base, + bloom_cap: self.docgraph.bloom_cap, + aliases: (!self.docgraph.aliases.is_empty()).then(|| self.docgraph.aliases.clone()), + }; + Some((config, files)) + } +} + +/// Tách suffix `:` khỏi một entry `[docgraph] paths` (chỉ nhận format +/// đã biết để không nhầm với ký tự `:` khác trong pattern). +fn split_format_override(entry: &str) -> (&str, Option<&str>) { + const FORMATS: [&str; 6] = ["hcl", "tf", "yaml", "yml", "json", "toml"]; + if let Some((pattern, format)) = entry.rsplit_once(':') { + if FORMATS.contains(&format.to_ascii_lowercase().as_str()) { + return (pattern, Some(format)); + } + } + (entry, None) } /// Setup rule config → skip rule effect unknown (warn) + giữ phần còn lại. @@ -372,6 +536,26 @@ type = "sqlite" # depth = "aaa" # "aaa" (full) hoặc "fast" (af; aar; aac — nhanh hơn cho binary lớn) # cfg_markers = true # xây marker IF/LOOP/SWITCH từ CFG của mỗi function # cache = true # cache kết quả phân tích theo (path, mtime, size) + +# [docgraph] +# Document graph — ingest tài liệu cấu trúc (HCL/Terraform, YAML, JSON, TOML) +# lúc `codegraph init`, truy vấn qua MCP (`codegraph_doc_*`) hoặc `codegraph doc`. +# Bỏ comment section + `paths` để bật: +# [docgraph] +# enabled = true # mặc định bật khi có `paths` +# Glob tính từ project root; suffix `:` override format theo entry. +# paths = ["infra/*.tf", "deploy/*.yaml", "config/settings.toml"] +# +# Storage cho docs — mặc định dataset RIÊNG cùng backend kind của [storage] +# (sqlite → .codegraph/docs.sqlite, lmdb → docs.lmdb, redis → keyspace riêng). +# [docgraph.storage] +# type = "sqlite" +# dsn = "sqlite:///tmp/docs.db" +# +# doc_base = 1_000_000_000 # base id node/doc (mặc định 1e9) +# pattern_base = 3_000_000_000 # base id mined pattern (mặc định 3e9) +# bloom_cap = 64 # bloom-filter cap cho doc search +# aliases = [["instances", "replicas"]] # chuẩn hoá key khi tra cứu "#; /// Default `config.toml` section `[binary]` (ghi chú, thêm bởi `codegraph init`). @@ -483,6 +667,86 @@ headers = "cpp" assert_eq!(StorageKind::parse("whatsapp"), StorageKind::Sqlite); } + /// Parse `[docgraph]` — glob mở rộng, format override theo entry, storage + /// override; không khai báo `paths` → `doc_config` trả `None`. + #[test] + fn docgraph_parse_and_glob() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + std::fs::write(root.join("a.tf"), "resource {}\n").unwrap(); + std::fs::create_dir_all(root.join("sub")).unwrap(); + std::fs::write(root.join("sub").join("b.yaml"), "k: v\n").unwrap(); + let cfg_path = root.join("config.toml"); + let cfg_path = Utf8Path::from_path(&cfg_path).unwrap(); + std::fs::write( + cfg_path.as_std_path(), + r#" +[docgraph] +paths = ["*.tf", "sub/*.yaml", "nothing/:hcl"] + +[docgraph.storage] +type = "sqlite" +dsn = "sqlite:///tmp/docs-test.db" +"#, + ) + .unwrap(); + let root = Utf8Path::from_path(root).unwrap(); + let cfg = ExtractConfig::load_from(cfg_path); + let (doc_cfg, files) = cfg.doc_config(root).expect("docgraph enabled"); + // Glob khớp đúng 2 file (pattern "nothing/" không có match); format + // override ":hcl" không nhầm với phần mở rộng thường. + assert_eq!(files.len(), 2); + assert!(files.iter().all(|(p, _)| p.file_name() != Some("nothing"))); + // Storage override thắng default (không phải docs.sqlite của project). + let storage = doc_cfg.storage.expect("storage config"); + assert_eq!(storage.dsn.as_deref(), Some("sqlite:///tmp/docs-test.db")); + + // Không `paths` → không ingest. + std::fs::write(cfg_path.as_std_path(), "[docgraph]\nenabled = true\n").unwrap(); + let cfg = ExtractConfig::load_from(cfg_path); + assert!(cfg.doc_config(root).is_none()); + + // Không `[docgraph]` → dsn mặc định vẫn có (docs.sqlite cho sqlite). + std::fs::write(cfg_path.as_std_path(), "").unwrap(); + let cfg = ExtractConfig::load_from(cfg_path); + let dsn = cfg.doc_storage_dsn(root).unwrap(); + assert!(dsn.ends_with("docs.sqlite"), "got {dsn}"); + } + + /// `doc_storage_dsn` override bằng `[docgraph.storage] dsn` thắng kind. + #[test] + fn doc_storage_dsn_override() { + let dir = std::env::temp_dir().join("codegraph-extract-docdsn-test"); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("config.toml"); + let path = Utf8Path::from_path(path.as_path()).unwrap(); + std::fs::write( + path.as_std_path(), + r#" +[storage] +type = "lmdb" + +[docgraph.storage] +dsn = "sqlite:///tmp/custom-docs.db" +"#, + ) + .unwrap(); + let cfg = ExtractConfig::load_from(path); + assert_eq!( + cfg.doc_storage_dsn(Utf8Path::new("/repo")).unwrap(), + "sqlite:///tmp/custom-docs.db" + ); + + // Không override → theo kind của [storage] (lmdb → docs.lmdb). + std::fs::write(path.as_std_path(), "[storage]\ntype = \"lmdb\"\n").unwrap(); + let cfg = ExtractConfig::load_from(path); + let dsn = cfg.doc_storage_dsn(Utf8Path::new("/repo")).unwrap(); + assert!(dsn.starts_with("lmdb://") && dsn.ends_with("docs.lmdb"), "got {dsn}"); + + let _ = std::fs::remove_file(path.as_std_path()); + let _ = std::fs::remove_dir(&dir); + } + /// `storage_dsn` dựng DSN theo kind; `dsn` override thắng. #[test] fn storage_dsn_built_or_overridden() { diff --git a/crates/codegraph-extract/src/docgraph.rs b/crates/codegraph-extract/src/docgraph.rs new file mode 100644 index 000000000..7d63199f7 --- /dev/null +++ b/crates/codegraph-extract/src/docgraph.rs @@ -0,0 +1,41 @@ +//! Document graph runtime — mở `DocumentGraph` từ `[docgraph]`/`[storage]` của +//! `.codegraph/config.toml` (dùng chung cho CLI `init`/`doc` và MCP server). + +use crate::config::ExtractConfig; +use camino::Utf8Path; +use codegraph_core::{Error, Result}; +use codegraph_docs::{DocConfig, DocumentGraph, StorageConfig}; +use std::sync::Arc; +use tokio::sync::RwLock as TokioRwLock; + +/// Mở document graph theo config: dataset riêng cho docs (mặc định +/// `.codegraph/docs.sqlite` với sqlite — tries của docs đụng namespace với code +/// index nên KHÔNG dùng chung dataset), rebuild tries từ storage. Không có DSN +/// hợp lệ (memory/RDBMS không override) → in-memory. +pub async fn open_doc_graph(root: &Utf8Path) -> Result { + let cfg = ExtractConfig::load(root); + let (config, _) = match cfg.doc_config(root) { + Some(pair) => pair, + // Không khai báo `[docgraph]` — vẫn mở dataset mặc định để CLI `doc` + // và MCP persist đúng (dsn mặc định theo backend kind của `[storage]`). + None => { + let dsn = cfg.doc_storage_dsn(root); + let config = DocConfig { + storage: dsn.map(|dsn| StorageConfig { + r#type: Some(dsn.split("://").next().unwrap_or("sqlite").to_string()), + dsn: Some(dsn), + }), + ..Default::default() + }; + (config, Vec::new()) + } + }; + let storage: Arc> = + match config.storage.as_ref().and_then(|s| s.dsn.as_deref()) { + Some(dsn) => codegraph_graph::open_doc_storage(dsn).await?, + None => Arc::new(TokioRwLock::new(codegraph_graph::InMemoryStorage::default())), + }; + DocumentGraph::open(storage, config) + .await + .map_err(|e| Error::Db(format!("open document graph: {e}"))) +} diff --git a/crates/codegraph-extract/src/lib.rs b/crates/codegraph-extract/src/lib.rs index fd266fc1e..33095513a 100644 --- a/crates/codegraph-extract/src/lib.rs +++ b/crates/codegraph-extract/src/lib.rs @@ -7,12 +7,14 @@ //! làm sau khi `ingest` gom toàn bộ file. pub mod config; +pub mod docgraph; pub mod languages; mod orchestrator; mod project; mod walker; -pub use config::{ExtractConfig, HeaderLanguage, DEFAULT_CONFIG_TOML}; +pub use config::{DocGraphSection, ExtractConfig, HeaderLanguage, DEFAULT_CONFIG_TOML}; +pub use docgraph::open_doc_graph; pub use orchestrator::{ExtractStats, Orchestrator}; pub use project::{init_project, project_db_path, project_dir, CODEGRAPH_DIR}; diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index 709d02d83..44af72867 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -38,6 +38,9 @@ use crate::embeddings::{EmbeddingBackend, default_backend, embedding_enabled, ma pub use crate::radix::Element; pub use crate::search::Search; pub use crate::search::SearchResume; +/// Error type của `Search::insert_chain` (ví dụ `Duplicated`) — re-export để +/// caller xử lý lỗi key trùng mà không cần `mod search` public. +pub use crate::search::Error as SearchError; use crate::storage::cached::CachedStorage; #[cfg(feature = "lmdb")] pub use crate::storage::lmdb::LmdbStorage; @@ -119,6 +122,63 @@ fn serr_search(e: crate::search::Error) -> Error { Error::Search(e.to_string()) } +/// Mở storage handle cho document graph từ DSN — dataset **riêng**, không share +/// instance với `GraphIndex` (doc tries dùng namespace shard/record riêng nên +/// phải là dataset riêng, và `DocumentGraph` cần `Arc>`). +/// +/// - `sqlite://` → `SqliteStorage` (feature `sqlite`) +/// - `lmdb://` → `LmdbStorage` (feature `lmdb`) +/// - `redis://...` → `RedisStorage` với keyspace prefix `codegraph:docs` +/// (feature `redis`) — tách khỏi index `codegraph:idx:` +pub async fn open_doc_storage(dsn: &str) -> Result>> { + if let Some(path) = dsn.strip_prefix("sqlite://") { + #[cfg(feature = "sqlite")] + { + let storage = crate::storage::sqlite::SqliteStorage::open(path) + .await + .map_err(serr)?; + return Ok(Arc::new(RwLock::new(storage))); + } + #[cfg(not(feature = "sqlite"))] + { + let _ = path; + return Err(backend_unavailable("sqlite")); + } + } + if let Some(path) = dsn.strip_prefix("lmdb://") { + #[cfg(feature = "lmdb")] + { + let storage = crate::storage::lmdb::LmdbStorage::open(path) + .await + .map_err(serr)?; + return Ok(Arc::new(RwLock::new(storage))); + } + #[cfg(not(feature = "lmdb"))] + { + let _ = path; + return Err(backend_unavailable("lmdb")); + } + } + if dsn.starts_with("redis://") || dsn.starts_with("rediss://") { + #[cfg(feature = "redis")] + { + let client = redis::Client::open(dsn) + .map_err(|e| Error::Db(format!("redis client: {e}")))?; + let storage = crate::storage::redis::RedisStorage::new(client, "codegraph:docs") + .await + .map_err(serr)?; + return Ok(Arc::new(RwLock::new(storage))); + } + #[cfg(not(feature = "redis"))] + { + return Err(backend_unavailable("redis")); + } + } + Err(Error::Db(format!( + "open_doc_storage: DSN scheme không hỗ trợ: {dsn}" + ))) +} + /// Kết quả parse một file — input của `GraphIndex::ingest` (full re-index). /// /// Mọi id trong `symbols`/`chains`/`calls` là **local per-file** (bắt đầu từ diff --git a/crates/codegraph-mcp/src/lib.rs b/crates/codegraph-mcp/src/lib.rs index 5734593ba..cb8b74050 100644 --- a/crates/codegraph-mcp/src/lib.rs +++ b/crates/codegraph-mcp/src/lib.rs @@ -94,12 +94,19 @@ impl CodegraphServer { format: OutputStyle, mermaid: bool, ) -> anyhow::Result { - let storage: Arc> = - Arc::new(TokioRwLock::new(InMemoryStorage::default())); - let doc_graph = Arc::new(TokioRwLock::new(DocumentGraph::new( - storage, - DocConfig::default(), - ))); + // Document graph mở từ `[docgraph]`/`[storage]` config của root + // (dataset riêng, persist qua các phiên). Lỗi config/backend → fallback + // in-memory thay vì chặn cả server (doc tools vẫn dùng được per-session). + let doc_graph = match codegraph_extract::open_doc_graph(&root).await { + Ok(g) => Arc::new(TokioRwLock::new(g)), + Err(e) => { + tracing::warn!("doc graph open failed ({e}) — fallback in-memory"); + Arc::new(TokioRwLock::new(DocumentGraph::new( + Arc::new(TokioRwLock::new(InMemoryStorage::default())), + DocConfig::default(), + ))) + } + }; Ok(Self { session: Session::with_root_and_format(root, format).await?, usage: Arc::new(Mutex::new(usage::UsageStats::default())), diff --git a/crates/codegraph-mcp/src/tools.rs b/crates/codegraph-mcp/src/tools.rs index 7d775982b..2fcd41f4a 100644 --- a/crates/codegraph-mcp/src/tools.rs +++ b/crates/codegraph-mcp/src/tools.rs @@ -1035,42 +1035,10 @@ pub async fn dispatch_doc_ingest( path: &str, format: Option, ) -> Result { - let source = std::fs::read_to_string(path) - .map_err(|e| Error::Invalid(format!("failed to read {path}: {e}")))?; - let ext = std::path::Path::new(path) - .extension() - .and_then(|e| e.to_str()) - .map(|e| e.to_lowercase()) - .unwrap_or_default(); - let fmt: String = match format { - Some(f) => f, - None => match ext.as_str() { - "tf" | "hcl" => "hcl".to_string(), - "yaml" | "yml" => "yaml".to_string(), - "json" => "json".to_string(), - "toml" => "toml".to_string(), - _ => { - return Err(Error::Invalid(format!( - "unknown format for extension .{ext}" - ))) - } - }, - }; - let parser: Box = match fmt.as_str() { - "hcl" => Box::new(codegraph_docs::parsers::HclParser), - "yaml" => Box::new(codegraph_docs::parsers::YamlParser), - "json" => Box::new(codegraph_docs::parsers::JsonParser), - "toml" => Box::new(codegraph_docs::parsers::TomlParser), - _ => return Err(Error::Invalid(format!("unsupported format: {fmt}"))), - }; - let doc_id = doc_graph.read().await.stats().docs as u64 + 1; - let doc = parser - .parse(path, &source, doc_id) - .map_err(|e| Error::Other(e.to_string()))?; let inserted = doc_graph .write() .await - .upsert_document(doc) + .ingest_file(path, format.as_deref()) .await .map_err(|e| Error::Other(e.to_string()))?; Ok(format!("ingested {path} → doc_id={inserted}")) diff --git a/crates/codegraph/src/main.rs b/crates/codegraph/src/main.rs index 4a30f4622..d2ec8a46b 100644 --- a/crates/codegraph/src/main.rs +++ b/crates/codegraph/src/main.rs @@ -3,10 +3,8 @@ use camino::{Utf8Path, Utf8PathBuf}; use clap::{ArgAction, Parser, Subcommand}; use codegraph_extract::{ExtractStats, Orchestrator}; use codegraph_graph::GraphIndex; -use codegraph_graph::InMemoryStorage; use codegraph_mcp::CodegraphServer; use std::sync::Arc; -use tokio::sync::RwLock as TokioRwLock; #[cfg(feature = "fastembed")] use codegraph_graph::embeddings::warm_model_cache; @@ -292,6 +290,15 @@ async fn open_index(root: &Utf8Path) -> Result { } } +/// Mở document graph theo config (`[docgraph]` + `[storage]`): dataset riêng +/// cho docs (mặc định `.codegraph/docs.sqlite` với sqlite), rebuild tries từ +/// storage. Dùng chung helper với MCP server. +async fn open_doc_graph(root: &Utf8Path) -> Result { + codegraph_extract::open_doc_graph(root) + .await + .map_err(|e| anyhow!("{e}")) +} + /// `codegraph init`: tạo `.codegraph/` + config, index ngay nếu `do_index` /// (progress bar khi `show_progress`). không gọi installer nữa. async fn cmd_init(root: &Utf8Path, do_index: bool, show_progress: bool) -> Result<()> { @@ -307,6 +314,28 @@ async fn cmd_init(root: &Utf8Path, do_index: bool, show_progress: bool) -> Resul stats.files, stats.symbols, stats.chains, stats.calls, stats.skipped ); } + ingest_configured_docs(root).await?; + Ok(()) +} + +/// Ingest các document khai báo trong `[docgraph] paths` của config.toml +/// (idempotent — doc trùng path được thay thế tại chỗ). +async fn ingest_configured_docs(root: &Utf8Path) -> Result<()> { + let Some((_, files)) = codegraph_extract::ExtractConfig::load(root).doc_config(root) else { + return Ok(()); + }; + if files.is_empty() { + return Ok(()); + } + let mut graph = open_doc_graph(root).await?; + let mut ingested = 0usize; + for (path, format) in &files { + match graph.ingest_file(path.as_str(), format.as_deref()).await { + Ok(_) => ingested += 1, + Err(e) => eprintln!("doc ingest failed for {path}: {e}"), + } + } + eprintln!("ingested {ingested}/{} documents", files.len()); Ok(()) } @@ -678,46 +707,15 @@ async fn cmd_serve( } /// `codegraph doc`: manage structured documents (HCL/Terraform, YAML, JSON, TOML). -async fn cmd_doc(_root: &Utf8Path, cmd: DocCmd) -> Result<()> { - let storage: Arc> = - Arc::new(TokioRwLock::new(InMemoryStorage::default())); - let config = codegraph_docs::DocConfig::default(); - let mut graph = codegraph_docs::DocumentGraph::new(storage, config); +/// Persist qua dataset docs theo config (`[docgraph]`/`[storage]`) — không còn +/// in-memory per-invocation. +async fn cmd_doc(root: &Utf8Path, cmd: DocCmd) -> Result<()> { + let mut graph = open_doc_graph(root).await?; match cmd { DocCmd::Ingest { path, format } => { - let source = std::fs::read_to_string(&path) - .map_err(|e| anyhow!("failed to read {path}: {e}"))?; - let ext = std::path::Path::new(&path) - .extension() - .and_then(|e| e.to_str()) - .map(|e| e.to_lowercase()) - .unwrap_or_default(); - let format = match format { - Some(f) => f, - None => match ext.as_str() { - "tf" | "hcl" => "hcl".to_string(), - "yaml" | "yml" => "yaml".to_string(), - "json" => "json".to_string(), - "toml" => "toml".to_string(), - _ => { - return Err(anyhow!( - "unknown format for extension .{ext}; use --format to override" - )) - } - }, - }; - let parser: Box = match format.as_str() { - "hcl" => Box::new(codegraph_docs::parsers::HclParser), - "yaml" => Box::new(codegraph_docs::parsers::YamlParser), - "json" => Box::new(codegraph_docs::parsers::JsonParser), - "toml" => Box::new(codegraph_docs::parsers::TomlParser), - _ => return Err(anyhow!("unsupported document format: {format}")), - }; - let doc_id = graph.stats().docs as u64 + 1; - let doc = parser.parse(&path, &source, doc_id)?; - let inserted = graph.upsert_document(doc).await?; - println!("ingested {} → doc_id={}", path, inserted); + let inserted = graph.ingest_file(&path, format.as_deref()).await?; + println!("ingested {path} → doc_id={inserted}"); } DocCmd::Search { pattern: _, depth } => { use codegraph_docs::DocToken; From bfcf7f82b0486cf5ce8f3b3e887c5bca986034cb Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Sat, 12 Sep 2026 06:58:40 +0700 Subject: [PATCH 2/6] Inplement nginx parser --- crates/codegraph-docs/src/parsers/mod.rs | 465 +++++++++++++++++++++++ crates/codegraph-extract/src/config.rs | 2 +- crates/codegraph/src/main.rs | 2 +- 3 files changed, 467 insertions(+), 2 deletions(-) diff --git a/crates/codegraph-docs/src/parsers/mod.rs b/crates/codegraph-docs/src/parsers/mod.rs index 37bca4707..fa1e8359c 100644 --- a/crates/codegraph-docs/src/parsers/mod.rs +++ b/crates/codegraph-docs/src/parsers/mod.rs @@ -370,9 +370,329 @@ fn convert_hcl_value(value: &hcl::Value, span: ByteSpan) -> RecursiveNode { } } +// ── nginx parser ────────────────────────────────────────────────────────── + +pub struct NginxParser; + +impl DocParser for NginxParser { + fn format(&self) -> &'static str { + "nginx" + } + + fn parse(&self, path: &str, source: &str, id: u64) -> Result { + let root = parse_nginx(source)?; + Ok(build_document( + path.to_string(), + self.format().to_string(), + id, + root, + )) + } +} + +#[derive(Debug, Clone, PartialEq)] +enum NginxToken { + /// Từ khoá/giá trị (nội dung chuỗi đã bỏ quote). + Word(String, u32), + /// Nội dung thô của `_by_lua_block` — giữ nguyên, không parse như nginx. + LuaCode(String, u32), + LBrace(u32), + RBrace(u32), + Semi(u32), +} + +/// Tokenize theo behavior của lexer gonginx (tham khảo `nginx/parser/lexer.go`): +/// - `#` đến cuối dòng là comment (bỏ qua). +/// - Quote `"`, `'`, `` ` `` → 1 word, hỗ trợ escape `\"`; unquote khi tạo value. +/// - `${...}` là variable reference trong word — `{`/`}` bên trong không phải +/// block delimiter (Issue 17: `set $x $a${uri}index.html;`). +/// - Word kết thúc bằng `_by_lua_block` → scan code thô đến `}` đóng (đếm +/// depth, bỏ qua `{`/`}` trong `#` comment) thành `LuaCode`. +fn tokenize_nginx(source: &str) -> Result> { + let chars: Vec = source.chars().collect(); + let mut tokens = Vec::new(); + let mut i = 0usize; + let mut line = 1u32; + let mut last_word = String::new(); + while i < chars.len() { + let c = chars[i]; + match c { + '\n' => { + line += 1; + i += 1; + } + c if c.is_whitespace() => i += 1, + '#' => { + while i < chars.len() && chars[i] != '\n' { + i += 1; + } + } + '{' => { + tokens.push(NginxToken::LBrace(line)); + last_word.clear(); + i += 1; + } + '}' => { + tokens.push(NginxToken::RBrace(line)); + last_word.clear(); + i += 1; + } + ';' => { + tokens.push(NginxToken::Semi(line)); + last_word.clear(); + i += 1; + } + q @ ('"' | '\'' | '`') => { + i += 1; + let mut word = String::new(); + loop { + match chars.get(i) { + None | Some('\n') => { + anyhow::bail!( + "unexpected end of file while scanning quoted string at line {line}" + ); + } + Some('\\') if chars.get(i + 1) == Some(&q) => { + word.push(q); + i += 2; + } + Some(c2) if *c2 == q => { + i += 1; + break; + } + Some(c2) => { + word.push(*c2); + i += 1; + } + } + } + tokens.push(NginxToken::Word(word.clone(), line)); + last_word = word; + } + _ => { + let mut word = String::new(); + let mut in_var_ref = false; + let mut prev = '\0'; + loop { + let Some(&c2) = chars.get(i) else { break }; + if in_var_ref { + if c2 == '}' { + in_var_ref = false; + } + word.push(c2); + prev = c2; + i += 1; + continue; + } + if c2.is_whitespace() || matches!(c2, ';' | '\n') { + break; + } + if c2 == '{' { + if prev == '$' { + in_var_ref = true; + word.push('{'); + prev = c2; + i += 1; + continue; + } + break; + } + if c2 == '}' { + break; + } + word.push(c2); + prev = c2; + i += 1; + } + tokens.push(NginxToken::Word(word.clone(), line)); + last_word = word; + } + } + // `_by_lua_block {` → nội dung tiếp theo là code thô đến `}` đóng. + if last_word.ends_with("_by_lua_block") + && chars.get(i) == Some(&'{') + { + i += 1; + let mut code = String::new(); + let mut depth = 0usize; + loop { + let Some(&c2) = chars.get(i) else { + anyhow::bail!( + "unexpected end of file while scanning lua code starting at line {line}" + ); + }; + if c2 == '#' { + // Comment trong lua: giữ nguyên đến cuối dòng, `{`/`}` trong + // comment không đổi depth. + while i < chars.len() && chars[i] != '\n' { + code.push(chars[i]); + i += 1; + } + continue; + } + match c2 { + '{' => depth += 1, + '}' if depth == 0 => break, + '}' => depth -= 1, + '\n' => line += 1, + _ => {} + } + code.push(c2); + i += 1; + } + tokens.push(NginxToken::LuaCode(code, line)); + last_word.clear(); + } + } + Ok(tokens) +} + +fn parse_nginx(source: &str) -> Result { + let tokens = tokenize_nginx(source)?; + let mut pos = 0; + let entries = parse_nginx_entries(&tokens, &mut pos, true)?; + Ok(RecursiveNode::Map(entries)) +} + +/// Parse một scope: directive `name args...;` hoặc block `name args... { ... }`. +/// Với scope lồng (top=false) dừng và tiêu thụ `}` đóng; scope top yêu cầu +/// hết token và không được gặp `}` lạc. +fn parse_nginx_entries( + tokens: &[NginxToken], + pos: &mut usize, + top: bool, +) -> Result> { + let mut entries: Vec<(String, RecursiveNode, ByteSpan)> = Vec::new(); + while let Some(tok) = tokens.get(*pos) { + match tok { + NginxToken::Word(name, line) => { + *pos += 1; + let mut args: Vec = Vec::new(); + let (key, value) = loop { + match tokens.get(*pos) { + Some(NginxToken::Word(arg, _)) => { + args.push(arg.clone()); + *pos += 1; + } + Some(NginxToken::Semi(_)) => { + *pos += 1; + let value = match args.len() { + 0 => RecursiveNode::Null(ByteSpan { start: 0, end: 0 }), + 1 => RecursiveNode::String(args.remove(0), ByteSpan { start: 0, end: 0 }), + _ => RecursiveNode::Array( + args.drain(..) + .map(|a| { + ( + RecursiveNode::String(a, ByteSpan { start: 0, end: 0 }), + ByteSpan { start: 0, end: 0 }, + ) + }) + .collect(), + ), + }; + break (name.clone(), value); + } + Some(NginxToken::LuaCode(code, l)) => { + // `_by_lua_block { ... }` — tokenizer đã tiêu thụ `{` + // và gói code thô thành LuaCode; chỉ còn chờ `}`. + *pos += 1; + match tokens.get(*pos) { + Some(NginxToken::RBrace(_)) => { + *pos += 1; + } + other => { + let l2 = match other { + Some( + NginxToken::Word(_, l2) + | NginxToken::LuaCode(_, l2) + | NginxToken::LBrace(l2) + | NginxToken::RBrace(l2) + | NginxToken::Semi(l2), + ) => *l2, + None => *l, + }; + anyhow::bail!( + "expected '}}' after lua code of \"{name}\" at line {l2}" + ); + } + } + break ( + name.clone(), + RecursiveNode::String(code.clone(), ByteSpan { start: 0, end: 0 }), + ); + } + Some(NginxToken::LBrace(_)) => { + *pos += 1; + let inner = parse_nginx_entries(tokens, pos, false)?; + let key = if args.is_empty() { + name.clone() + } else { + format!("{name} {}", args.join(" ")) + }; + break (key, RecursiveNode::Map(inner)); + } + Some(NginxToken::RBrace(l)) => { + anyhow::bail!("expected ';' or '{{' after \"{name}\" at line {l}"); + } + None => { + anyhow::bail!("expected ';' or '{{' after \"{name}\" at line {line}"); + } + } + }; + push_nginx_entry(&mut entries, key, value); + } + NginxToken::RBrace(line) if !top => { + *pos += 1; + return Ok(entries); + } + NginxToken::RBrace(line) => { + anyhow::bail!("unexpected '}}' at line {line}"); + } + NginxToken::Semi(line) => { + anyhow::bail!("unexpected ';' at line {line}"); + } + NginxToken::LBrace(line) => { + anyhow::bail!("unexpected '{{' at line {line}"); + } + NginxToken::LuaCode(_, line) => { + anyhow::bail!("unexpected lua code outside block at line {line}"); + } + } + } + if !top { + anyhow::bail!("missing '}}' at end of file"); + } + Ok(entries) +} + +/// Thêm entry vào scope; key trùng (nhiều `server {}`, nhiều `add_header;`) +/// gộp thành `Array`. +fn push_nginx_entry( + entries: &mut Vec<(String, RecursiveNode, ByteSpan)>, + key: String, + value: RecursiveNode, +) { + let span = ByteSpan { start: 0, end: 0 }; + if let Some(slot) = entries.iter_mut().find(|(k, _, _)| *k == key) { + match &mut slot.1 { + RecursiveNode::Array(items) => items.push((value, span)), + old => { + let prev = old.clone(); + *old = RecursiveNode::Array(vec![(prev, span), (value, span)]); + } + } + } else { + entries.push((key, value, span)); + } +} + #[cfg(test)] mod tests { use super::*; + use crate::{DocConfig, DocumentGraph}; + use codegraph_graph::InMemoryStorage; + use std::sync::Arc; + use tokio::sync::RwLock as TokioRwLock; #[test] fn yaml_parser() { @@ -384,6 +704,149 @@ service: let doc = YamlParser.parse("/tmp/a.yaml", src, 1).unwrap(); assert_eq!(doc.nodes.len(), 4); // root, service, name, replicas } + + #[test] + fn nginx_parser_nested_blocks_and_directives() { + let src = r#" +# global comment +worker_processes auto; + +http { + include mime.types; + server { + listen 8080; + server_name example.com; + location /api { + proxy_pass http://backend; + add_header X-A 1; + } + } + upstream backend { + server 10.0.0.1:8080; + server 10.0.0.2:8080; + } +} +"#; + let doc = NginxParser.parse("/etc/nginx/nginx.conf", src, 1).unwrap(); + let find = |key: &str| doc.nodes.iter().find(|n| n.key.as_deref() == Some(key)); + // root, worker_processes, http, include, server, listen, server_name, + // "location /api", proxy_pass, add_header (Array + 2 strings), + // "upstream backend", server trùng (Array + 2 strings) = 16 node. + assert_eq!(doc.nodes.len(), 16); + // Block có args → key gồm cả args. + assert!(find("location /api").is_some()); + assert!(find("upstream backend").is_some()); + // Directive nhiều args. + assert!(find("worker_processes").is_some()); + // Trùng key trong upstream gộp thành 1 entry Array với 2 con. + let ups = find("upstream backend").unwrap(); + let servers: Vec<_> = doc + .nodes + .iter() + .filter(|n| n.parent == Some(ups.id) && n.key.as_deref() == Some("server")) + .collect(); + assert_eq!(servers.len(), 1); + // Con của entry Array: 2 server theo thứ tự khai báo. + let kids: Vec<_> = doc + .nodes + .iter() + .filter(|n| n.parent == Some(servers[0].id)) + .collect(); + assert_eq!(kids.len(), 2); + assert_eq!(kids[0].value, Some(Scalar::String("10.0.0.1:8080".to_string()))); + assert_eq!(kids[1].value, Some(Scalar::String("10.0.0.2:8080".to_string()))); + } + + #[test] + fn nginx_parser_syntax_errors() { + // Thiếu ';' trước '{' lạc. + assert!(NginxParser.parse("a.conf", "foo bar }", 1).is_err()); + // Thiếu '}' cuối file. + assert!(NginxParser.parse("a.conf", "http { server { listen 80;", 1).is_err()); + // Dấu ';' đứng một mình. + assert!(NginxParser.parse("a.conf", ";", 1).is_err()); + } + + #[test] + fn nginx_parser_variables_quoted_and_lua() { + // Issue 17: `${uri}` trong value — `{`/`}` trong var-ref không phải block. + let doc = NginxParser + .parse("a.conf", "location / {\n set $serve_URL $fullurl${uri}index.html;\n}", 1) + .unwrap(); + let set = doc + .nodes + .iter() + .find(|n| n.key.as_deref() == Some("set")) + .unwrap(); + // 2 args → Array; `${uri}` giữ nguyên trong arg thứ 2. + let last = doc + .nodes + .iter() + .filter(|n| n.parent == Some(set.id)) + .last() + .unwrap(); + assert_eq!(last.value, Some(Scalar::String("$fullurl${uri}index.html".to_string()))); + + // Issue 65: quoted string chứa `{`/`}` — không đếm là block delimiter. + let doc = NginxParser + .parse( + "a.conf", + "log_format main '{' '\"msec\": \"$msec\" ' '}';\nerror_log off;", + 1, + ) + .unwrap(); + assert!(doc.nodes.iter().any(|n| n.key.as_deref() == Some("error_log"))); + + // Quoted string unquote + escape `\"`. + let doc = NginxParser + .parse("a.conf", r#"directive "with a quoted \" good.";"#, 1) + .unwrap(); + let d = doc.nodes.iter().find(|n| n.key.as_deref() == Some("directive")).unwrap(); + assert_eq!(d.value, Some(Scalar::String("with a quoted \" good.".to_string()))); + + // `_by_lua_block` — code thô giữ nguyên, `{`/`}` trong comment không đổi depth. + let doc = NginxParser + .parse( + "a.conf", + "location = /foo {\n rewrite_by_lua_block {\n t = { key=\"foo\" } # comment { unexpect\n }\n}\n", + 1, + ) + .unwrap(); + let loc = doc + .nodes + .iter() + .find(|n| n.key.as_deref() == Some("location = /foo")) + .unwrap(); + let lua = doc + .nodes + .iter() + .find(|n| n.parent == Some(loc.id) && n.key.as_deref() == Some("rewrite_by_lua_block")) + .unwrap(); + assert!(matches!(lua.value, Some(Scalar::String(ref s)) if s.contains("t = { key=\"foo\" }"))); + + // Unclosed quote → lỗi có số dòng. + let err = NginxParser.parse("a.conf", "server {\n set $a \"unterminated\n}", 1); + assert!(err.is_err()); + } + + #[test] + fn nginx_detect_format() { + assert_eq!(detect_format("conf/nginx.conf").unwrap(), "nginx"); + assert_eq!(detect_format("a.CONF").unwrap(), "nginx"); + } + + #[tokio::test] + async fn nginx_ingest_file() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("nginx.conf"); + std::fs::write(&p, "events { worker_connections 1024; }\n").unwrap(); + let storage = Arc::new(TokioRwLock::new(InMemoryStorage::default())); + let mut graph = DocumentGraph::new(storage, DocConfig::default()); + let _doc_id = graph.ingest_file(p.to_str().unwrap(), None).await.unwrap(); + // root + events + worker_connections = 3. + assert_eq!(graph.stats().nodes, 3); + assert_eq!(graph.stats().docs, 1); + } } /// Detect document format từ extension: `tf`/`hcl` → hcl, `yaml`/`yml`, @@ -399,6 +862,7 @@ pub fn detect_format(path: &str) -> Result { "yaml" | "yml" => Ok("yaml".to_string()), "json" => Ok("json".to_string()), "toml" => Ok("toml".to_string()), + "conf" | "nginx" => Ok("nginx".to_string()), _ => Err(anyhow::anyhow!( "unknown format for extension .{ext}; specify --format to override" )), @@ -412,6 +876,7 @@ pub fn parser_for(format: &str) -> Result> { "yaml" => Ok(Box::new(YamlParser)), "json" => Ok(Box::new(JsonParser)), "toml" => Ok(Box::new(TomlParser)), + "nginx" => Ok(Box::new(NginxParser)), _ => Err(anyhow::anyhow!("unsupported document format: {format}")), } } diff --git a/crates/codegraph-extract/src/config.rs b/crates/codegraph-extract/src/config.rs index daccef995..f15918882 100644 --- a/crates/codegraph-extract/src/config.rs +++ b/crates/codegraph-extract/src/config.rs @@ -441,7 +441,7 @@ impl ExtractConfig { /// Tách suffix `:` khỏi một entry `[docgraph] paths` (chỉ nhận format /// đã biết để không nhầm với ký tự `:` khác trong pattern). fn split_format_override(entry: &str) -> (&str, Option<&str>) { - const FORMATS: [&str; 6] = ["hcl", "tf", "yaml", "yml", "json", "toml"]; + const FORMATS: [&str; 8] = ["hcl", "tf", "yaml", "yml", "json", "toml", "nginx", "conf"]; if let Some((pattern, format)) = entry.rsplit_once(':') { if FORMATS.contains(&format.to_ascii_lowercase().as_str()) { return (pattern, Some(format)); diff --git a/crates/codegraph/src/main.rs b/crates/codegraph/src/main.rs index d2ec8a46b..f4a8946da 100644 --- a/crates/codegraph/src/main.rs +++ b/crates/codegraph/src/main.rs @@ -161,7 +161,7 @@ enum DocCmd { /// Path to the document file. #[arg()] path: String, - /// Override auto-detected format (hcl, yaml, json, toml). + /// Override auto-detected format (hcl, yaml, json, toml, nginx). #[arg(long)] format: Option, }, From 60f1c46c19901738bef80f023c19371f05c758ff Mon Sep 17 00:00:00 2001 From: hungpham10 <136320753+hungpham10@users.noreply.github.com> Date: Sat, 12 Sep 2026 00:00:00 +0000 Subject: [PATCH 3/6] style: apply rustfmt --- crates/codegraph-binary/src/extract.rs | 16 ++++-- crates/codegraph-docs/src/parsers/mod.rs | 62 ++++++++++++++++++------ crates/codegraph-extract/src/config.rs | 16 +++--- crates/codegraph-graph/src/lib.rs | 8 +-- 4 files changed, 73 insertions(+), 29 deletions(-) diff --git a/crates/codegraph-binary/src/extract.rs b/crates/codegraph-binary/src/extract.rs index f5ba3b008..db394ad39 100644 --- a/crates/codegraph-binary/src/extract.rs +++ b/crates/codegraph-binary/src/extract.rs @@ -637,7 +637,9 @@ mod tests { responses: HashMap, } impl R2Client for MockR2 { - fn cmd(&mut self, _cmd: &str) -> Result { Ok(String::new()) } + fn cmd(&mut self, _cmd: &str) -> Result { + Ok(String::new()) + } fn cmdj(&mut self, cmd: &str) -> Result { Ok(self.responses.get(cmd).cloned().unwrap_or(json!([]))) } @@ -655,8 +657,12 @@ mod tests { ); let exports = parse_iej(&mut mock).unwrap(); assert_eq!(exports.len(), 3); - assert!(exports.iter().any(|e| e.name.as_deref() == Some("JNI_OnLoad"))); - assert!(exports.iter().any(|e| e.name.as_deref() == Some("Java_com_example_Foo_bar"))); + assert!(exports + .iter() + .any(|e| e.name.as_deref() == Some("JNI_OnLoad"))); + assert!(exports + .iter() + .any(|e| e.name.as_deref() == Some("Java_com_example_Foo_bar"))); } #[test] @@ -665,7 +671,9 @@ mod tests { responses: HashMap, } impl R2Client for MockR2 { - fn cmd(&mut self, _cmd: &str) -> Result { Ok(String::new()) } + fn cmd(&mut self, _cmd: &str) -> Result { + Ok(String::new()) + } fn cmdj(&mut self, cmd: &str) -> Result { Ok(self.responses.get(cmd).cloned().unwrap_or(json!([]))) } diff --git a/crates/codegraph-docs/src/parsers/mod.rs b/crates/codegraph-docs/src/parsers/mod.rs index fa1e8359c..1628fb338 100644 --- a/crates/codegraph-docs/src/parsers/mod.rs +++ b/crates/codegraph-docs/src/parsers/mod.rs @@ -509,9 +509,7 @@ fn tokenize_nginx(source: &str) -> Result> { } } // `_by_lua_block {` → nội dung tiếp theo là code thô đến `}` đóng. - if last_word.ends_with("_by_lua_block") - && chars.get(i) == Some(&'{') - { + if last_word.ends_with("_by_lua_block") && chars.get(i) == Some(&'{') { i += 1; let mut code = String::new(); let mut depth = 0usize; @@ -578,12 +576,18 @@ fn parse_nginx_entries( *pos += 1; let value = match args.len() { 0 => RecursiveNode::Null(ByteSpan { start: 0, end: 0 }), - 1 => RecursiveNode::String(args.remove(0), ByteSpan { start: 0, end: 0 }), + 1 => RecursiveNode::String( + args.remove(0), + ByteSpan { start: 0, end: 0 }, + ), _ => RecursiveNode::Array( args.drain(..) .map(|a| { ( - RecursiveNode::String(a, ByteSpan { start: 0, end: 0 }), + RecursiveNode::String( + a, + ByteSpan { start: 0, end: 0 }, + ), ByteSpan { start: 0, end: 0 }, ) }) @@ -753,8 +757,14 @@ http { .filter(|n| n.parent == Some(servers[0].id)) .collect(); assert_eq!(kids.len(), 2); - assert_eq!(kids[0].value, Some(Scalar::String("10.0.0.1:8080".to_string()))); - assert_eq!(kids[1].value, Some(Scalar::String("10.0.0.2:8080".to_string()))); + assert_eq!( + kids[0].value, + Some(Scalar::String("10.0.0.1:8080".to_string())) + ); + assert_eq!( + kids[1].value, + Some(Scalar::String("10.0.0.2:8080".to_string())) + ); } #[test] @@ -762,7 +772,11 @@ http { // Thiếu ';' trước '{' lạc. assert!(NginxParser.parse("a.conf", "foo bar }", 1).is_err()); // Thiếu '}' cuối file. - assert!(NginxParser.parse("a.conf", "http { server { listen 80;", 1).is_err()); + assert!( + NginxParser + .parse("a.conf", "http { server { listen 80;", 1) + .is_err() + ); // Dấu ';' đứng một mình. assert!(NginxParser.parse("a.conf", ";", 1).is_err()); } @@ -771,7 +785,11 @@ http { fn nginx_parser_variables_quoted_and_lua() { // Issue 17: `${uri}` trong value — `{`/`}` trong var-ref không phải block. let doc = NginxParser - .parse("a.conf", "location / {\n set $serve_URL $fullurl${uri}index.html;\n}", 1) + .parse( + "a.conf", + "location / {\n set $serve_URL $fullurl${uri}index.html;\n}", + 1, + ) .unwrap(); let set = doc .nodes @@ -785,7 +803,10 @@ http { .filter(|n| n.parent == Some(set.id)) .last() .unwrap(); - assert_eq!(last.value, Some(Scalar::String("$fullurl${uri}index.html".to_string()))); + assert_eq!( + last.value, + Some(Scalar::String("$fullurl${uri}index.html".to_string())) + ); // Issue 65: quoted string chứa `{`/`}` — không đếm là block delimiter. let doc = NginxParser @@ -795,14 +816,25 @@ http { 1, ) .unwrap(); - assert!(doc.nodes.iter().any(|n| n.key.as_deref() == Some("error_log"))); + assert!( + doc.nodes + .iter() + .any(|n| n.key.as_deref() == Some("error_log")) + ); // Quoted string unquote + escape `\"`. let doc = NginxParser .parse("a.conf", r#"directive "with a quoted \" good.";"#, 1) .unwrap(); - let d = doc.nodes.iter().find(|n| n.key.as_deref() == Some("directive")).unwrap(); - assert_eq!(d.value, Some(Scalar::String("with a quoted \" good.".to_string()))); + let d = doc + .nodes + .iter() + .find(|n| n.key.as_deref() == Some("directive")) + .unwrap(); + assert_eq!( + d.value, + Some(Scalar::String("with a quoted \" good.".to_string())) + ); // `_by_lua_block` — code thô giữ nguyên, `{`/`}` trong comment không đổi depth. let doc = NginxParser @@ -822,7 +854,9 @@ http { .iter() .find(|n| n.parent == Some(loc.id) && n.key.as_deref() == Some("rewrite_by_lua_block")) .unwrap(); - assert!(matches!(lua.value, Some(Scalar::String(ref s)) if s.contains("t = { key=\"foo\" }"))); + assert!( + matches!(lua.value, Some(Scalar::String(ref s)) if s.contains("t = { key=\"foo\" }")) + ); // Unclosed quote → lỗi có số dòng. let err = NginxParser.parse("a.conf", "server {\n set $a \"unterminated\n}", 1); diff --git a/crates/codegraph-extract/src/config.rs b/crates/codegraph-extract/src/config.rs index f15918882..255b46538 100644 --- a/crates/codegraph-extract/src/config.rs +++ b/crates/codegraph-extract/src/config.rs @@ -368,12 +368,11 @@ impl ExtractConfig { .map(StorageKind::parse) .unwrap_or(self.storage.kind); match kind { - StorageKind::Sqlite => { - Some(format!("sqlite://{}", project_dir(root).join("docs.sqlite"))) - } - StorageKind::Lmdb => { - Some(format!("lmdb://{}", project_dir(root).join("docs.lmdb"))) - } + StorageKind::Sqlite => Some(format!( + "sqlite://{}", + project_dir(root).join("docs.sqlite") + )), + StorageKind::Lmdb => Some(format!("lmdb://{}", project_dir(root).join("docs.lmdb"))), StorageKind::Redis => self.storage.dsn.clone(), StorageKind::Memory | StorageKind::Postgres | StorageKind::MySql => None, } @@ -741,7 +740,10 @@ dsn = "sqlite:///tmp/custom-docs.db" std::fs::write(path.as_std_path(), "[storage]\ntype = \"lmdb\"\n").unwrap(); let cfg = ExtractConfig::load_from(path); let dsn = cfg.doc_storage_dsn(Utf8Path::new("/repo")).unwrap(); - assert!(dsn.starts_with("lmdb://") && dsn.ends_with("docs.lmdb"), "got {dsn}"); + assert!( + dsn.starts_with("lmdb://") && dsn.ends_with("docs.lmdb"), + "got {dsn}" + ); let _ = std::fs::remove_file(path.as_std_path()); let _ = std::fs::remove_dir(&dir); diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index 44af72867..ae2653ef5 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -36,11 +36,11 @@ use crate::embeddings::{EmbeddingBackend, default_backend, embedding_enabled, make_backend}; pub use crate::radix::Element; -pub use crate::search::Search; -pub use crate::search::SearchResume; /// Error type của `Search::insert_chain` (ví dụ `Duplicated`) — re-export để /// caller xử lý lỗi key trùng mà không cần `mod search` public. pub use crate::search::Error as SearchError; +pub use crate::search::Search; +pub use crate::search::SearchResume; use crate::storage::cached::CachedStorage; #[cfg(feature = "lmdb")] pub use crate::storage::lmdb::LmdbStorage; @@ -162,8 +162,8 @@ pub async fn open_doc_storage(dsn: &str) -> Result>> { if dsn.starts_with("redis://") || dsn.starts_with("rediss://") { #[cfg(feature = "redis")] { - let client = redis::Client::open(dsn) - .map_err(|e| Error::Db(format!("redis client: {e}")))?; + let client = + redis::Client::open(dsn).map_err(|e| Error::Db(format!("redis client: {e}")))?; let storage = crate::storage::redis::RedisStorage::new(client, "codegraph:docs") .await .map_err(serr)?; From caa9f17f5a384fa9909b60d19bd7f75b0a4d7e38 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Sat, 12 Sep 2026 07:12:52 +0700 Subject: [PATCH 4/6] Fix lint --- crates/codegraph-docs/src/graph.rs | 5 +- crates/codegraph-docs/src/parsers/mod.rs | 74 +++++++++++------------- crates/codegraph-extract/src/config.rs | 17 +++--- 3 files changed, 46 insertions(+), 50 deletions(-) diff --git a/crates/codegraph-docs/src/graph.rs b/crates/codegraph-docs/src/graph.rs index be65e67db..158d18ef5 100644 --- a/crates/codegraph-docs/src/graph.rs +++ b/crates/codegraph-docs/src/graph.rs @@ -419,11 +419,10 @@ impl DocumentGraph { return Ok(()); } let metas: Vec> = vec![None; tokens.len()]; - if let Err(e) = trie.insert_chain(record, tokens, &metas).await { - if !matches!(e, codegraph_graph::SearchError::Duplicated) { + if let Err(e) = trie.insert_chain(record, tokens, &metas).await + && !matches!(e, codegraph_graph::SearchError::Duplicated) { return Err(anyhow::anyhow!(e.to_string())); } - } Ok(()) } diff --git a/crates/codegraph-docs/src/parsers/mod.rs b/crates/codegraph-docs/src/parsers/mod.rs index 1628fb338..fa2e6edad 100644 --- a/crates/codegraph-docs/src/parsers/mod.rs +++ b/crates/codegraph-docs/src/parsers/mod.rs @@ -414,8 +414,7 @@ fn tokenize_nginx(source: &str) -> Result> { let mut i = 0usize; let mut line = 1u32; let mut last_word = String::new(); - while i < chars.len() { - let c = chars[i]; + while let Some(&c) = chars.get(i) { match c { '\n' => { line += 1; @@ -473,8 +472,7 @@ fn tokenize_nginx(source: &str) -> Result> { let mut word = String::new(); let mut in_var_ref = false; let mut prev = '\0'; - loop { - let Some(&c2) = chars.get(i) else { break }; + while let Some(&c2) = chars.get(i) { if in_var_ref { if c2 == '}' { in_var_ref = false; @@ -690,6 +688,38 @@ fn push_nginx_entry( } } +/// Detect document format từ extension: `tf`/`hcl` → hcl, `yaml`/`yml`, +/// `json`, `toml`. Lỗi khi extension không nhận diện được. +pub fn detect_format(path: &str) -> Result { + let ext = std::path::Path::new(path) + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.to_lowercase()) + .unwrap_or_default(); + match ext.as_str() { + "tf" | "hcl" => Ok("hcl".to_string()), + "yaml" | "yml" => Ok("yaml".to_string()), + "json" => Ok("json".to_string()), + "toml" => Ok("toml".to_string()), + "conf" | "nginx" => Ok("nginx".to_string()), + _ => Err(anyhow::anyhow!( + "unknown format for extension .{ext}; specify --format to override" + )), + } +} + +/// Chọn parser theo format name (`"hcl"`, `"yaml"`, `"json"`, `"toml"`). +pub fn parser_for(format: &str) -> Result> { + match format { + "hcl" => Ok(Box::new(HclParser)), + "yaml" => Ok(Box::new(YamlParser)), + "json" => Ok(Box::new(JsonParser)), + "toml" => Ok(Box::new(TomlParser)), + "nginx" => Ok(Box::new(NginxParser)), + _ => Err(anyhow::anyhow!("unsupported document format: {format}")), + } +} + #[cfg(test)] mod tests { use super::*; @@ -799,9 +829,7 @@ http { // 2 args → Array; `${uri}` giữ nguyên trong arg thứ 2. let last = doc .nodes - .iter() - .filter(|n| n.parent == Some(set.id)) - .last() + .iter().rfind(|n| n.parent == Some(set.id)) .unwrap(); assert_eq!( last.value, @@ -882,35 +910,3 @@ http { assert_eq!(graph.stats().docs, 1); } } - -/// Detect document format từ extension: `tf`/`hcl` → hcl, `yaml`/`yml`, -/// `json`, `toml`. Lỗi khi extension không nhận diện được. -pub fn detect_format(path: &str) -> Result { - let ext = std::path::Path::new(path) - .extension() - .and_then(|e| e.to_str()) - .map(|e| e.to_lowercase()) - .unwrap_or_default(); - match ext.as_str() { - "tf" | "hcl" => Ok("hcl".to_string()), - "yaml" | "yml" => Ok("yaml".to_string()), - "json" => Ok("json".to_string()), - "toml" => Ok("toml".to_string()), - "conf" | "nginx" => Ok("nginx".to_string()), - _ => Err(anyhow::anyhow!( - "unknown format for extension .{ext}; specify --format to override" - )), - } -} - -/// Chọn parser theo format name (`"hcl"`, `"yaml"`, `"json"`, `"toml"`). -pub fn parser_for(format: &str) -> Result> { - match format { - "hcl" => Ok(Box::new(HclParser)), - "yaml" => Ok(Box::new(YamlParser)), - "json" => Ok(Box::new(JsonParser)), - "toml" => Ok(Box::new(TomlParser)), - "nginx" => Ok(Box::new(NginxParser)), - _ => Err(anyhow::anyhow!("unsupported document format: {format}")), - } -} diff --git a/crates/codegraph-extract/src/config.rs b/crates/codegraph-extract/src/config.rs index 255b46538..98380699f 100644 --- a/crates/codegraph-extract/src/config.rs +++ b/crates/codegraph-extract/src/config.rs @@ -211,6 +211,13 @@ pub struct StorageConfig { pub dsns: Vec, } +/// Một file docs khớp glob `[docgraph] paths`: path + format override +/// (từ suffix `:` của entry, nếu có). +pub type DocFile = (Utf8PathBuf, Option); + +/// Config document graph + danh sách file docs cần ingest lúc `codegraph init`. +pub type DocFiles = (codegraph_docs::DocConfig, Vec); + impl ExtractConfig { pub fn load(root: &Utf8Path) -> Self { let path = root.join(".codegraph").join("config.toml"); @@ -381,17 +388,11 @@ impl ExtractConfig { /// Config document graph + danh sách file khớp glob `[docgraph] paths` /// (path kèm format override). Trả `None` khi `[docgraph]` không bật / /// không khai báo `paths`. - pub fn doc_config( - &self, - root: &Utf8Path, - ) -> Option<( - codegraph_docs::DocConfig, - Vec<(Utf8PathBuf, Option)>, - )> { + pub fn doc_config(&self, root: &Utf8Path) -> Option { if !self.docgraph.is_enabled() { return None; } - let mut files: Vec<(Utf8PathBuf, Option)> = Vec::new(); + let mut files: Vec = Vec::new(); for entry in &self.docgraph.paths { let (pattern, format) = split_format_override(entry); let full = root.join(pattern).to_string(); From a7a51e85daf673263dd29e3b891f999645298b63 Mon Sep 17 00:00:00 2001 From: hungpham10 <136320753+hungpham10@users.noreply.github.com> Date: Sat, 12 Sep 2026 00:20:45 +0000 Subject: [PATCH 5/6] style: apply rustfmt --- crates/codegraph-docs/src/graph.rs | 7 ++++--- crates/codegraph-docs/src/parsers/mod.rs | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/codegraph-docs/src/graph.rs b/crates/codegraph-docs/src/graph.rs index 158d18ef5..75d646afb 100644 --- a/crates/codegraph-docs/src/graph.rs +++ b/crates/codegraph-docs/src/graph.rs @@ -420,9 +420,10 @@ impl DocumentGraph { } let metas: Vec> = vec![None; tokens.len()]; if let Err(e) = trie.insert_chain(record, tokens, &metas).await - && !matches!(e, codegraph_graph::SearchError::Duplicated) { - return Err(anyhow::anyhow!(e.to_string())); - } + && !matches!(e, codegraph_graph::SearchError::Duplicated) + { + return Err(anyhow::anyhow!(e.to_string())); + } Ok(()) } diff --git a/crates/codegraph-docs/src/parsers/mod.rs b/crates/codegraph-docs/src/parsers/mod.rs index fa2e6edad..2a1a2e8c4 100644 --- a/crates/codegraph-docs/src/parsers/mod.rs +++ b/crates/codegraph-docs/src/parsers/mod.rs @@ -829,7 +829,8 @@ http { // 2 args → Array; `${uri}` giữ nguyên trong arg thứ 2. let last = doc .nodes - .iter().rfind(|n| n.parent == Some(set.id)) + .iter() + .rfind(|n| n.parent == Some(set.id)) .unwrap(); assert_eq!( last.value, From 1751445ef76e3af6fb6b1db9258f88846c02acc4 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Sat, 12 Sep 2026 07:21:59 +0700 Subject: [PATCH 6/6] Bump version to v2.1.5 --- Cargo.lock | 26 ++++++++++++------------- Cargo.toml | 2 +- packaging/aur/codegraph-rs-bin/PKGBUILD | 2 +- packaging/choco/codegraph.nuspec | 2 +- packaging/winget/codegraph.yaml | 4 ++-- scripts/install.ps1 | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03ec8a20b..e51ed268c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -720,7 +720,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "codegraph" -version = "2.1.4" +version = "2.1.5" dependencies = [ "anyhow", "camino", @@ -743,7 +743,7 @@ dependencies = [ [[package]] name = "codegraph-api" -version = "2.1.4" +version = "2.1.5" dependencies = [ "anyhow", "camino", @@ -760,7 +760,7 @@ dependencies = [ [[package]] name = "codegraph-bench" -version = "2.1.4" +version = "2.1.5" dependencies = [ "anyhow", "camino", @@ -778,7 +778,7 @@ dependencies = [ [[package]] name = "codegraph-binary" -version = "2.1.4" +version = "2.1.5" dependencies = [ "camino", "codegraph-core", @@ -795,7 +795,7 @@ dependencies = [ [[package]] name = "codegraph-context" -version = "2.1.4" +version = "2.1.5" dependencies = [ "codegraph-core", "codegraph-graph", @@ -807,7 +807,7 @@ dependencies = [ [[package]] name = "codegraph-core" -version = "2.1.4" +version = "2.1.5" dependencies = [ "async-graphql", "camino", @@ -818,7 +818,7 @@ dependencies = [ [[package]] name = "codegraph-docs" -version = "2.1.4" +version = "2.1.5" dependencies = [ "anyhow", "codegraph-core", @@ -835,7 +835,7 @@ dependencies = [ [[package]] name = "codegraph-extract" -version = "2.1.4" +version = "2.1.5" dependencies = [ "camino", "codegraph-binary", @@ -872,7 +872,7 @@ dependencies = [ [[package]] name = "codegraph-graph" -version = "2.1.4" +version = "2.1.5" dependencies = [ "async-trait", "bincode", @@ -902,7 +902,7 @@ dependencies = [ [[package]] name = "codegraph-graphql" -version = "2.1.4" +version = "2.1.5" dependencies = [ "anyhow", "async-graphql", @@ -925,7 +925,7 @@ dependencies = [ [[package]] name = "codegraph-installer" -version = "2.1.4" +version = "2.1.5" dependencies = [ "anyhow", "camino", @@ -941,7 +941,7 @@ dependencies = [ [[package]] name = "codegraph-mcp" -version = "2.1.4" +version = "2.1.5" dependencies = [ "anyhow", "axum", @@ -964,7 +964,7 @@ dependencies = [ [[package]] name = "codegraph-sboxes" -version = "2.1.4" +version = "2.1.5" dependencies = [ "camino", "codegraph-core", diff --git a/Cargo.toml b/Cargo.toml index 80def8931..62cde133f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ members = [ ] [workspace.package] -version = "2.1.4" +version = "2.1.5" edition = "2021" rust-version = "1.80" license = "MIT" diff --git a/packaging/aur/codegraph-rs-bin/PKGBUILD b/packaging/aur/codegraph-rs-bin/PKGBUILD index da3603ca5..d392cf789 100644 --- a/packaging/aur/codegraph-rs-bin/PKGBUILD +++ b/packaging/aur/codegraph-rs-bin/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: Hung Pham pkgname=codegraph-rs-bin -pkgver=2.1.4 +pkgver=2.1.5 pkgrel=1 pkgdesc="Local-first code intelligence: tree-sitter knowledge graph + MCP server (prebuilt binary)" arch=('x86_64' 'aarch64') diff --git a/packaging/choco/codegraph.nuspec b/packaging/choco/codegraph.nuspec index 1d6b6db65..b3880373d 100644 --- a/packaging/choco/codegraph.nuspec +++ b/packaging/choco/codegraph.nuspec @@ -2,7 +2,7 @@ codegraph - 2.1.4 + 2.1.5 codegraph Hung Pham https://github.com/hungpham10/codegraph-rs diff --git a/packaging/winget/codegraph.yaml b/packaging/winget/codegraph.yaml index 745bfbaf5..da9cb4b85 100644 --- a/packaging/winget/codegraph.yaml +++ b/packaging/winget/codegraph.yaml @@ -6,7 +6,7 @@ # release time (or automate it in the release pipeline before submitting to # microsoft/winget-pkgs). PackageIdentifier: hungpham10.codegraph -PackageVersion: 2.1.4 +PackageVersion: 2.1.5 PackageName: codegraph Publisher: Hung Pham PublisherUrl: https://github.com/hungpham10/codegraph-rs @@ -18,7 +18,7 @@ PackageUrl: https://github.com/hungpham10/codegraph-rs InstallerType: zip Installers: - Architecture: x64 - InstallerUrl: https://github.com/hungpham10/codegraph-rs/releases/download/v2.1.4/codegraph-x86_64-pc-windows-msvc.zip + InstallerUrl: https://github.com/hungpham10/codegraph-rs/releases/download/v2.1.5/codegraph-x86_64-pc-windows-msvc.zip InstallerSha256: 0000000000000000000000000000000000000000000000000000000000000000 InstallerType: zip ManifestType: singleton diff --git a/scripts/install.ps1 b/scripts/install.ps1 index 579cd6392..b374c965c 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -5,11 +5,11 @@ # # Usage (pin a version / download the script first): # irm https://raw.githubusercontent.com/hungpham10/codegraph-rs/main/scripts/install.ps1 -OutFile install.ps1 -# .\install.ps1 -Version 2.1.4 +# .\install.ps1 -Version 2.1.5 [CmdletBinding()] param( - # Pin a specific version, e.g. "2.1.4". Empty = latest release. + # Pin a specific version, e.g. "2.1.5". Empty = latest release. [string]$Version )