diff --git a/crates/opensymphony-memory/src/code_graph.rs b/crates/opensymphony-memory/src/code_graph.rs index b053bddf..6946f427 100644 --- a/crates/opensymphony-memory/src/code_graph.rs +++ b/crates/opensymphony-memory/src/code_graph.rs @@ -4497,23 +4497,16 @@ fn index_code_repository_at_checked( } } - // Existing memory stores may predate the snapshot tables. Migrate once - // before the read-only previous-snapshot lookup so a first index works on - // both empty and previously initialized stores. - let connection = open_index(config)?; - migrate_index(&connection).map_err(|source| CodeGraphProjectionError::Memory( - MemoryError::DuckDb { - path: config.index_path.clone(), - source, - }, - ))?; - drop(connection); - - discard_staged_code_index_rows(config, repo_id)?; + // Existing memory stores may predate the snapshot tables, so the setup + // connection is migrated before the previous-snapshot lookup. One + // connection serves every setup read and write; it is closed before files + // are parsed so other processes can open the index meanwhile. + let mut connection = open_migrated_code_index(config)?; + discard_staged_code_index_rows(&mut connection, config, repo_id)?; let paths = git_tree_paths(&config.repo_root, &commit_sha)?; let repo_prefix = git_repo_prefix(&config.repo_root)?; let config_fingerprint = code_index_config_fingerprint(config); - let previous = latest_code_snapshot(config, repo_id, &target_branch)?; + let previous = latest_code_snapshot(&connection, config, repo_id, &target_branch)?; let changed_paths = previous .as_ref() .filter(|(revision, _, _)| revision != &commit_sha) @@ -4529,7 +4522,8 @@ fn index_code_repository_at_checked( .as_ref() .is_some_and(|(_, _, fingerprint)| fingerprint == &config_fingerprint); - persist_code_snapshot_state( + persist_code_snapshot_state_on( + &connection, config, CodeSnapshotState { repo_id, @@ -4544,6 +4538,11 @@ fn index_code_repository_at_checked( indexed_at, }, )?; + // Nothing writes current rows while files are parsed, so the current + // snapshot is read once here instead of once per file. + let current_rows = CurrentCodeSnapshotRows::load(&connection, config, repo_id)?; + let current_snapshot_paths = current_code_snapshot_paths(&connection, config, repo_id)?; + drop(connection); let previous_files = previous .map(|(_, files, _)| files) @@ -4562,7 +4561,8 @@ fn index_code_repository_at_checked( let mut deleted_paths = Vec::new(); let mut edge_refresh_paths = Vec::new(); - for (path, mode, blob_id) in paths { + let mut blobs = GitBlobBatch::new(&config.repo_root); + for (path, mode, blob_id, blob_size) in paths { let skipped_directory = path.components().any(|component| { skipped_directory_name(Path::new(component.as_os_str())).is_some() }); @@ -4602,7 +4602,11 @@ fn index_code_repository_at_checked( continue; } let relative_display = path.to_string_lossy().to_string(); - let blob_size = git_blob_size(&config.repo_root, &repo_prefix, &commit_sha, &path)?; + let blob_size = blob_size.ok_or_else(|| { + CodeGraphProjectionError::InvalidRequest(format!( + "invalid target-branch blob size for {relative_display}" + )) + })?; let Some(language) = detect_language(&path) else { let content_sha256 = format!("git-blob:{blob_id}"); skipped_files.push(format!("{relative_display}: unsupported language")); @@ -4702,9 +4706,7 @@ fn index_code_repository_at_checked( )) }) .map(|previous_file| { - current_code_snapshot_file_matches( - config, - repo_id, + Ok::<_, CodeGraphProjectionError>(current_rows.matches( &CodeSnapshotMembershipInput { path: path.clone(), language: previous_file.language.clone(), @@ -4714,7 +4716,7 @@ fn index_code_repository_at_checked( analyzed: previous_file.analyzed, skip_reason: previous_file.skip_reason.clone(), }, - ) + )) .map(|matches| (previous_file, matches)) }) .transpose()? @@ -4777,7 +4779,7 @@ fn index_code_repository_at_checked( continue; } remaining_files -= 1; - let bytes = git_blob(&config.repo_root, &repo_prefix, &commit_sha, &path)?; + let bytes = blobs.read(&blob_id, &path)?; let content_sha256 = sha256_bytes_hex(&bytes); let source = match String::from_utf8(bytes) { Ok(source) => source, @@ -4893,7 +4895,7 @@ fn index_code_repository_at_checked( diagnostics.push(format!("{path}: deleted from target branch")); } } - for path in current_code_snapshot_paths(config, repo_id)? { + for path in current_snapshot_paths { if !current_files.contains_key(&path) && !deleted_paths.contains(&path) { deleted_files += 1; deleted_paths.push(path.clone()); @@ -4901,7 +4903,9 @@ fn index_code_repository_at_checked( } } - let persisted = persist_code_index_documents_in_batches( + let mut connection = open_migrated_code_index(config)?; + let persisted = persist_code_index_documents_in_batches_on( + &mut connection, config, repo_id, &commit_sha, @@ -4910,7 +4914,8 @@ fn index_code_repository_at_checked( false, )?; for batch in skipped_inputs.chunks(32) { - persist_code_intel_skipped_files_with_freshness( + persist_code_intel_skipped_files_with_freshness_on( + &mut connection, config, repo_id, Some(&commit_sha), @@ -4921,12 +4926,20 @@ fn index_code_repository_at_checked( )?; } let run_id = indexed_at.to_rfc3339(); - persist_code_snapshot_membership(config, repo_id, &commit_sha, &run_id, &memberships)?; + persist_code_snapshot_membership_on( + &mut connection, + config, + repo_id, + &commit_sha, + &run_id, + &memberships, + )?; + let current_rows = CurrentCodeSnapshotRows::load(&connection, config, repo_id)?; let mut promotion_paths = Vec::new(); for file in &memberships { let path = file.path.to_string_lossy().to_string(); if !edge_refresh_paths.iter().any(|candidate| candidate == &path) - && !current_code_snapshot_file_matches(config, repo_id, file)? + && !current_rows.matches(file) { promotion_paths.push(path); } @@ -4949,7 +4962,8 @@ fn index_code_repository_at_checked( if revalidate_target { let current_target = code_index_target(config)?; if current_target.as_ref() != Some(&(target_branch.clone(), commit_sha.clone())) { - discard_staged_code_index_rows(config, repo_id)?; + discard_staged_code_index_rows(&mut connection, config, repo_id)?; + drop(connection); let mut report = code_graph_index_report(config, repo_id)?; report.status = CodeIndexStatus::Unavailable; report.head_revision = None; @@ -4961,6 +4975,7 @@ fn index_code_repository_at_checked( } } stale_rows += promote_staged_code_snapshot( + &mut connection, config, repo_id, &commit_sha, @@ -4969,7 +4984,8 @@ fn index_code_repository_at_checked( completed_state, )?; let analyzed_documents = memberships.iter().filter(|file| file.analyzed).count(); - let current_counts = code_index_counts(config, repo_id)?; + let current_counts = code_index_counts_on(&connection, config, repo_id)?; + drop(connection); stale_rows += persisted.stale_rows; Ok(CodeIndexReport { schema_version: SchemaVersion::v1(), @@ -5076,10 +5092,12 @@ fn git_target_commit( Ok(None) } -fn git_tree_paths( - root: &Path, - commit: &str, -) -> Result, CodeGraphProjectionError> { +/// One target-tree entry: path, mode, object id, and blob size. +type GitTreeEntry = (PathBuf, String, String, Option); + +/// Lists the target tree. Sizes come from `ls-tree -l`, so callers do not +/// start a Git process per file to ask. +fn git_tree_paths(root: &Path, commit: &str) -> Result, CodeGraphProjectionError> { let mut directories = vec![String::new()]; let mut entries = Vec::new(); while let Some(directory) = directories.pop() { @@ -5091,7 +5109,7 @@ fn git_tree_paths( let output = Command::new(git_program()) .args(["-C"]) .arg(root) - .args(["ls-tree", "-z"]) + .args(["ls-tree", "-z", "-l"]) .arg(commit) .args(["--", &pathspec]) .output() @@ -5117,18 +5135,19 @@ fn git_tree_paths( let mode = parts.next().unwrap_or_default().to_string(); let object_kind = parts.next().unwrap_or_default(); let object_id = parts.next().unwrap_or_default().to_string(); + let size = parts.next().and_then(|size| size.parse::().ok()); if object_kind == "tree" { if skipped_directory_name(Path::new(path)).is_none() { directories.push(path.to_string()); } else { - entries.push((PathBuf::from(path), mode, object_id)); + entries.push((PathBuf::from(path), mode, object_id, None)); } } else if object_kind == "blob" || object_kind == "commit" { - entries.push((PathBuf::from(path), mode, object_id)); + entries.push((PathBuf::from(path), mode, object_id, size)); } } } - for (path, _, _) in &entries { + for (path, _, _, _) in &entries { if normalize_code_path(&path.to_string_lossy()).is_err() { return Err(CodeGraphProjectionError::InvalidRequest(format!( "target branch contains an invalid path: {}", @@ -5191,75 +5210,122 @@ fn git_changed_tree_paths( .collect()) } -fn git_blob( - root: &Path, - repo_prefix: &Path, - commit: &str, - path: &Path, -) -> Result, CodeGraphProjectionError> { - let object = format!("{}:{}", commit, repo_prefix.join(path).to_string_lossy()); - let output = Command::new(git_program()) - .args(["-C"]) - .arg(root) - .args(["cat-file", "blob"]) - .arg(object) - .output() - .map_err(|source| CodeGraphProjectionError::Memory(MemoryError::ResolvePath { - path: root.to_path_buf(), - source, - }))?; - if !output.status.success() { - return Err(CodeGraphProjectionError::InvalidRequest(format!( - "failed to read target-branch file {}", - path.display() - ))); - } - Ok(output.stdout) +/// Reads target-branch blobs through one long-lived `git cat-file --batch` +/// process instead of starting a Git process per file. +struct GitBlobBatch { + root: PathBuf, + process: Option, } -fn git_blob_size( - root: &Path, - repo_prefix: &Path, - commit: &str, - path: &Path, -) -> Result { - let object = format!("{}:{}", commit, repo_prefix.join(path).to_string_lossy()); - let output = Command::new(git_program()) - .args(["-C"]) - .arg(root) - .args(["cat-file", "-s"]) - .arg(object) - .output() - .map_err(|source| CodeGraphProjectionError::Memory(MemoryError::ResolvePath { - path: root.to_path_buf(), - source, - }))?; - if !output.status.success() { - return Err(CodeGraphProjectionError::InvalidRequest(format!( - "failed to inspect target-branch file {}", - path.display() - ))); +struct GitBlobBatchProcess { + child: std::process::Child, + stdin: std::process::ChildStdin, + stdout: io::BufReader, +} + +impl GitBlobBatch { + fn new(root: &Path) -> Self { + Self { + root: root.to_path_buf(), + process: None, + } } - String::from_utf8_lossy(&output.stdout) - .trim() - .parse::() - .map_err(|_| { + + fn read(&mut self, object_id: &str, path: &Path) -> Result, CodeGraphProjectionError> { + let read_failed = || { CodeGraphProjectionError::InvalidRequest(format!( - "invalid target-branch blob size for {}", + "failed to read target-branch file {}", path.display() )) - }) + }; + if object_id.is_empty() || !object_id.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(read_failed()); + } + if self.process.is_none() { + let mut child = Command::new(git_program()) + .args(["-C"]) + .arg(&self.root) + .args(["cat-file", "--batch"]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn() + .map_err(|source| { + CodeGraphProjectionError::Memory(MemoryError::ResolvePath { + path: self.root.clone(), + source, + }) + })?; + let (Some(stdin), Some(stdout)) = (child.stdin.take(), child.stdout.take()) else { + let _ = child.kill(); + let _ = child.wait(); + return Err(read_failed()); + }; + self.process = Some(GitBlobBatchProcess { + child, + stdin, + stdout: io::BufReader::new(stdout), + }); + } + let process = self.process.as_mut().expect("cat-file process started"); + let result = (|| -> io::Result>> { + use std::io::{BufRead, Read, Write}; + writeln!(process.stdin, "{object_id}")?; + process.stdin.flush()?; + let mut header = String::new(); + if process.stdout.read_line(&mut header)? == 0 { + return Ok(None); + } + let mut fields = header.trim_end_matches('\n').split(' '); + let (Some(returned_id), Some("blob"), Some(size), None) = + (fields.next(), fields.next(), fields.next(), fields.next()) + else { + return Ok(None); + }; + let Ok(size) = size.parse::() else { + return Ok(None); + }; + if !returned_id.eq_ignore_ascii_case(object_id) { + return Ok(None); + } + let mut content = vec![0_u8; size]; + process.stdout.read_exact(&mut content)?; + let mut terminator = [0_u8; 1]; + process.stdout.read_exact(&mut terminator)?; + Ok((terminator == *b"\n").then_some(content)) + })(); + match result { + Ok(Some(content)) => Ok(content), + // The stream position is unknown after a failure, so the process + // is discarded and a later read starts a fresh one. + Ok(None) | Err(_) => { + self.shutdown(); + Err(read_failed()) + } + } + } + + fn shutdown(&mut self) { + if let Some(mut process) = self.process.take() { + drop(process.stdin); + let _ = process.child.kill(); + let _ = process.child.wait(); + } + } +} + +impl Drop for GitBlobBatch { + fn drop(&mut self) { + self.shutdown(); + } } fn latest_code_snapshot( + connection: &Connection, config: &MemoryConfig, repo_id: &str, target_branch: &str, ) -> Result, CodeGraphProjectionError> { - if !config.index_path.exists() { - return Ok(None); - } - let connection = open_index_read_only(config)?; let mut statement = connection .prepare("SELECT commit_sha, config_fingerprint FROM code_index_snapshots WHERE repo_id = ? AND target_branch = ? AND status = 'completed' ORDER BY indexed_at DESC LIMIT 1") .map_err(|source| CodeGraphProjectionError::Memory(MemoryError::DuckDb { @@ -5332,17 +5398,19 @@ fn code_index_config_fingerprint(config: &MemoryConfig) -> String { ) } +#[cfg(test)] fn persist_code_snapshot_state( config: &MemoryConfig, state: CodeSnapshotState<'_>, ) -> Result<(), CodeGraphProjectionError> { - let connection = open_index(config)?; - migrate_index(&connection).map_err(|source| CodeGraphProjectionError::Memory( - MemoryError::DuckDb { - path: config.index_path.clone(), - source, - }, - ))?; + persist_code_snapshot_state_on(&open_migrated_code_index(config)?, config, state) +} + +fn persist_code_snapshot_state_on( + connection: &Connection, + config: &MemoryConfig, + state: CodeSnapshotState<'_>, +) -> Result<(), CodeGraphProjectionError> { if state.status == "running" && connection .query_row( @@ -5383,17 +5451,24 @@ fn persist_code_snapshot_state( Ok(()) } -fn discard_staged_code_index_rows( - config: &MemoryConfig, - repo_id: &str, -) -> Result<(), CodeGraphProjectionError> { - let mut connection = open_index(config)?; +/// Opens the index for writing and applies migrations, for helpers that run +/// outside a shared index-run connection. +fn open_migrated_code_index(config: &MemoryConfig) -> Result { + let connection = open_index(config)?; migrate_index(&connection).map_err(|source| CodeGraphProjectionError::Memory( MemoryError::DuckDb { path: config.index_path.clone(), source, }, ))?; + Ok(connection) +} + +fn discard_staged_code_index_rows( + connection: &mut Connection, + config: &MemoryConfig, + repo_id: &str, +) -> Result<(), CodeGraphProjectionError> { let transaction = connection.transaction().map_err(|source| { CodeGraphProjectionError::Memory(MemoryError::DuckDb { path: config.index_path.clone(), @@ -5456,21 +5531,33 @@ fn discard_staged_code_index_rows( Ok(()) } +#[cfg(test)] fn persist_code_snapshot_membership( config: &MemoryConfig, repo_id: &str, commit_sha: &str, run_id: &str, files: &[CodeSnapshotMembershipInput], +) -> Result<(), CodeGraphProjectionError> { + persist_code_snapshot_membership_on( + &mut open_migrated_code_index(config)?, + config, + repo_id, + commit_sha, + run_id, + files, + ) +} + +fn persist_code_snapshot_membership_on( + connection: &mut Connection, + config: &MemoryConfig, + repo_id: &str, + commit_sha: &str, + run_id: &str, + files: &[CodeSnapshotMembershipInput], ) -> Result<(), CodeGraphProjectionError> { for batch in files.chunks(256) { - let mut connection = open_index(config)?; - migrate_index(&connection).map_err(|source| CodeGraphProjectionError::Memory( - MemoryError::DuckDb { - path: config.index_path.clone(), - source, - }, - ))?; let transaction = connection.transaction().map_err(|source| { CodeGraphProjectionError::Memory(MemoryError::DuckDb { path: config.index_path.clone(), @@ -5509,7 +5596,39 @@ fn persist_code_snapshot_membership( Ok(()) } +/// Connection-local temporary table that holds the paths one promotion step +/// marks stale. +const CODE_CHANGED_PATHS_TABLE: &str = "opensymphony_code_changed_paths"; + +/// Replaces the contents of the temporary path table `table` with `paths`, +/// so a single set-based statement can cover every path. +fn load_code_path_set( + connection: &Connection, + config: &MemoryConfig, + table: &str, + paths: &[String], +) -> Result<(), CodeGraphProjectionError> { + let duckdb_error = |source| { + CodeGraphProjectionError::Memory(MemoryError::DuckDb { + path: config.index_path.clone(), + source, + }) + }; + connection + .execute_batch(&format!( + "CREATE TEMP TABLE IF NOT EXISTS {table} (path TEXT NOT NULL); DELETE FROM temp.{table};" + )) + .map_err(duckdb_error)?; + let mut appender = connection.appender_to_catalog_and_db(table, "temp", "main").map_err(duckdb_error)?; + for path in paths { + appender.append_row(params![path]).map_err(duckdb_error)?; + } + appender.flush().map_err(duckdb_error)?; + Ok(()) +} + fn promote_staged_code_snapshot( + connection: &mut Connection, config: &MemoryConfig, repo_id: &str, commit_sha: &str, @@ -5517,13 +5636,6 @@ fn promote_staged_code_snapshot( edge_refresh_paths: &[String], completed_state: CodeSnapshotState<'_>, ) -> Result { - let mut connection = open_index(config)?; - migrate_index(&connection).map_err(|source| CodeGraphProjectionError::Memory( - MemoryError::DuckDb { - path: config.index_path.clone(), - source, - }, - ))?; let transaction = connection.transaction().map_err(|source| { CodeGraphProjectionError::Memory(MemoryError::DuckDb { path: config.index_path.clone(), @@ -5549,38 +5661,41 @@ fn promote_staged_code_snapshot( path: config.index_path.clone(), source, }))?; - for path in changed_paths { - for table in [ - "code_documents", - "code_symbols", - "code_edges", - "code_diagnostics", - "code_skipped_files", - ] { - stale_rows += transaction - .execute( - &format!( - "UPDATE {table} SET freshness = 'stale' WHERE repo_id = ? AND path = ? AND freshness = 'current'" - ), - params![repo_id, path], - ) - .map_err(|source| CodeGraphProjectionError::Memory(MemoryError::DuckDb { - path: config.index_path.clone(), - source, - }))?; - } - } - for path in edge_refresh_paths { + // Each update below covers every listed path in one statement. Marking a + // row stale removes it from `freshness = 'current'`, so the counts equal + // those of one update per path even when a path is listed twice. + load_code_path_set(&transaction, config, CODE_CHANGED_PATHS_TABLE, changed_paths)?; + for table in [ + "code_documents", + "code_symbols", + "code_edges", + "code_diagnostics", + "code_skipped_files", + ] { stale_rows += transaction .execute( - "UPDATE code_edges AS current SET freshness = 'stale' WHERE current.repo_id = ? AND current.path = ? AND current.freshness = 'current' AND NOT EXISTS (SELECT 1 FROM code_edge_revisions AS staged WHERE staged.edge_id = current.edge_id AND staged.repo_id = ? AND staged.commit_sha = ? AND staged.freshness = 'staged')", - params![repo_id, path, repo_id, commit_sha], + &format!( + "UPDATE {table} SET freshness = 'stale' WHERE repo_id = ? AND freshness = 'current' AND path IN (SELECT path FROM temp.{CODE_CHANGED_PATHS_TABLE})" + ), + params![repo_id], ) .map_err(|source| CodeGraphProjectionError::Memory(MemoryError::DuckDb { path: config.index_path.clone(), source, }))?; } + load_code_path_set(&transaction, config, CODE_CHANGED_PATHS_TABLE, edge_refresh_paths)?; + stale_rows += transaction + .execute( + &format!( + "UPDATE code_edges AS current SET freshness = 'stale' WHERE current.repo_id = ? AND current.path IN (SELECT path FROM temp.{CODE_CHANGED_PATHS_TABLE}) AND current.freshness = 'current' AND NOT EXISTS (SELECT 1 FROM code_edge_revisions AS staged WHERE staged.edge_id = current.edge_id AND staged.repo_id = ? AND staged.commit_sha = ? AND staged.freshness = 'staged')" + ), + params![repo_id, repo_id, commit_sha], + ) + .map_err(|source| CodeGraphProjectionError::Memory(MemoryError::DuckDb { + path: config.index_path.clone(), + source, + }))?; stale_rows += transaction .execute( "UPDATE code_diagnostics AS current SET freshness = 'stale' WHERE current.repo_id = ? AND current.freshness = 'current' AND EXISTS (SELECT 1 FROM code_diagnostics AS staged WHERE staged.repo_id = current.repo_id AND staged.path = current.path AND staged.commit_sha = ? AND staged.freshness = 'staged')", @@ -5706,12 +5821,10 @@ fn promote_staged_code_snapshot( } fn current_code_snapshot_paths( + connection: &Connection, config: &MemoryConfig, repo_id: &str, ) -> Result, CodeGraphProjectionError> { - let Some(connection) = open_existing_index_read_only(config)? else { - return Ok(BTreeSet::new()); - }; let mut statement = connection .prepare( "SELECT path FROM code_documents WHERE repo_id = ? AND freshness = 'current' UNION SELECT path FROM code_skipped_files WHERE repo_id = ? AND freshness = 'current'", @@ -5733,56 +5846,100 @@ fn current_code_snapshot_paths( })) } -fn current_code_snapshot_file_matches( - config: &MemoryConfig, - repo_id: &str, - file: &CodeSnapshotMembershipInput, -) -> Result { - let Some(connection) = open_existing_index_read_only(config)? else { - return Ok(false); - }; - let path = file.path.to_string_lossy().to_string(); - let exists = if file.analyzed { - connection - .query_row( - "SELECT 1 FROM code_documents WHERE repo_id = ? AND path = ? AND content_sha256 = ? AND parser_version = ? AND query_pack_version = ? AND freshness = 'current' AND NOT worktree_dirty LIMIT 1", - params![ - repo_id, - path, - file.content_sha256, - file.parser_version, - file.query_pack_version, - ], - |row| row.get::<_, i64>(0), - ) - .optional() - .map_err(|source| MemoryError::DuckDb { +/// The current, clean code rows of one repository, loaded once so snapshot +/// membership checks do not open the index per file. +struct CurrentCodeSnapshotRows { + /// `(path, content_sha256, parser_version, query_pack_version)`. + documents: BTreeSet<(String, String, String, String)>, + /// `(path, reason, content_sha256)`. + skipped_files: BTreeSet<(String, String, String)>, +} + +impl CurrentCodeSnapshotRows { + fn load( + connection: &Connection, + config: &MemoryConfig, + repo_id: &str, + ) -> Result { + let duckdb_error = |source| { + CodeGraphProjectionError::Memory(MemoryError::DuckDb { path: config.index_path.clone(), source, - })? - .is_some() - } else { - connection - .query_row( - "SELECT 1 FROM code_skipped_files WHERE repo_id = ? AND path = ? AND reason = ? AND content_sha256 = ? AND freshness = 'current' AND NOT worktree_dirty LIMIT 1", - params![ - repo_id, - path, - file.skip_reason.as_deref().unwrap_or(""), - file.content_sha256, - ], - |row| row.get::<_, i64>(0), + }) + }; + let documents = connection + .prepare( + "SELECT path, content_sha256, parser_version, query_pack_version FROM code_documents WHERE repo_id = ? AND freshness = 'current' AND NOT worktree_dirty", ) - .optional() - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })? - .is_some() - }; - Ok(exists) + .map_err(duckdb_error)? + .query_map(params![repo_id], |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + )) + }) + .map_err(duckdb_error)? + .collect::, _>>() + .map_err(duckdb_error)? + .into_iter() + // SQL equality never matches NULL, so rows with NULL keys never matched. + .filter_map(|row| match row { + (Some(path), Some(content), Some(parser), Some(query_pack)) => { + Some((path, content, parser, query_pack)) + } + _ => None, + }) + .collect(); + let skipped_files = connection + .prepare( + "SELECT path, reason, content_sha256 FROM code_skipped_files WHERE repo_id = ? AND freshness = 'current' AND NOT worktree_dirty", + ) + .map_err(duckdb_error)? + .query_map(params![repo_id], |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + )) + }) + .map_err(duckdb_error)? + .collect::, _>>() + .map_err(duckdb_error)? + .into_iter() + .filter_map(|row| match row { + (Some(path), Some(reason), Some(content)) => Some((path, reason, content)), + _ => None, + }) + .collect(); + Ok(Self { + documents, + skipped_files, + }) + } + + /// Whether the current read model already holds `file` as recorded. + fn matches(&self, file: &CodeSnapshotMembershipInput) -> bool { + let path = file.path.to_string_lossy().to_string(); + if file.analyzed { + self.documents.contains(&( + path, + file.content_sha256.clone(), + file.parser_version.clone(), + file.query_pack_version.clone(), + )) + } else { + self.skipped_files.contains(&( + path, + file.skip_reason.clone().unwrap_or_default(), + file.content_sha256.clone(), + )) + } + } } +#[cfg(test)] fn persist_code_index_documents_in_batches( config: &MemoryConfig, repo_id: &str, @@ -5790,6 +5947,26 @@ fn persist_code_index_documents_in_batches( documents: Vec, freshness: &str, stale_existing: bool, +) -> Result { + persist_code_index_documents_in_batches_on( + &mut open_migrated_code_index(config)?, + config, + repo_id, + commit_sha, + documents, + freshness, + stale_existing, + ) +} + +fn persist_code_index_documents_in_batches_on( + connection: &mut Connection, + config: &MemoryConfig, + repo_id: &str, + commit_sha: &str, + documents: Vec, + freshness: &str, + stale_existing: bool, ) -> Result { let mut total = CodeIntelPersistReport { parsed_files: 0, @@ -5802,7 +5979,8 @@ fn persist_code_index_documents_in_batches( diagnostics: Vec::new(), }; for batch in documents.chunks(32) { - let report = persist_code_intel_documents_with_freshness( + let report = persist_code_intel_documents_with_freshness_on( + connection, config, CodeIntelPersistBatch { repo_id: repo_id.to_string(), @@ -7310,29 +7488,37 @@ fn code_index_counts(config: &MemoryConfig, repo_id: &str) -> Result Result { let mut counts = CodeIndexCounts::default(); - if code_documents_read_model_ready(&connection, &config.index_path)? { + if code_documents_read_model_ready(connection, &config.index_path)? { counts.documents = - count_code_table_rows(&connection, config, "code_documents", repo_id, "current")?; + count_code_table_rows(connection, config, "code_documents", repo_id, "current")?; counts.stale_rows += - count_code_table_rows(&connection, config, "code_documents", repo_id, "stale")?; + count_code_table_rows(connection, config, "code_documents", repo_id, "stale")?; } - if code_symbols_read_model_ready(&connection, &config.index_path)? { + if code_symbols_read_model_ready(connection, &config.index_path)? { counts.symbols = - count_code_table_rows(&connection, config, "code_symbols", repo_id, "current")?; + count_code_table_rows(connection, config, "code_symbols", repo_id, "current")?; counts.stale_rows += - count_code_table_rows(&connection, config, "code_symbols", repo_id, "stale")?; + count_code_table_rows(connection, config, "code_symbols", repo_id, "stale")?; } - if code_edges_read_model_ready(&connection, &config.index_path)? { - counts.edges = count_code_table_rows(&connection, config, "code_edges", repo_id, "current")?; + if code_edges_read_model_ready(connection, &config.index_path)? { + counts.edges = count_code_table_rows(connection, config, "code_edges", repo_id, "current")?; counts.stale_rows += - count_code_table_rows(&connection, config, "code_edges", repo_id, "stale")?; + count_code_table_rows(connection, config, "code_edges", repo_id, "stale")?; } - if code_diagnostics_read_model_ready(&connection, &config.index_path)? { + if code_diagnostics_read_model_ready(connection, &config.index_path)? { counts.diagnostics = - count_code_table_rows(&connection, config, "code_diagnostics", repo_id, "current")?; + count_code_table_rows(connection, config, "code_diagnostics", repo_id, "current")?; counts.stale_rows += - count_code_table_rows(&connection, config, "code_diagnostics", repo_id, "stale")?; + count_code_table_rows(connection, config, "code_diagnostics", repo_id, "stale")?; } Ok(counts) } @@ -9612,11 +9798,11 @@ mod code_graph_tests { let commit = git(repo.path(), &["rev-parse", "HEAD"]); let paths = super::git_tree_paths(repo.path(), &commit).expect("tree should list"); - assert!(paths.iter().any(|(path, _, _)| path == Path::new("src/lib.rs"))); + assert!(paths.iter().any(|(path, _, _, _)| path == Path::new("src/lib.rs"))); assert!(paths .iter() - .any(|(path, mode, _)| path == Path::new("node_modules") && mode == "040000")); - assert!(!paths.iter().any(|(path, _, _)| { + .any(|(path, mode, _, _)| path == Path::new("node_modules") && mode == "040000")); + assert!(!paths.iter().any(|(path, _, _, _)| { path != Path::new("node_modules") && path.starts_with(Path::new("node_modules")) })); @@ -9646,10 +9832,10 @@ mod code_graph_tests { .expect("tree should stay scoped"); assert!(paths .iter() - .any(|(path, _, _)| path == Path::new("src/lib.rs"))); + .any(|(path, _, _, _)| path == Path::new("src/lib.rs"))); assert!(!paths .iter() - .any(|(path, _, _)| path == Path::new("outside.rs"))); + .any(|(path, _, _, _)| path == Path::new("outside.rs"))); let config = MemoryConfig::load(repo.path().join("configured"), None) .expect("scoped memory config should load"); let report = index_code_repository(&config, "scoped-repo") @@ -9840,7 +10026,7 @@ mod code_graph_tests { let head = git(repo.path(), &["rev-parse", "HEAD"]); let paths = super::git_tree_paths(repo.path(), &head).expect("tree should include gitlink"); - assert!(paths.iter().any(|(path, mode, _)| { + assert!(paths.iter().any(|(path, mode, _, _)| { path == Path::new("deps/submodule") && mode == "160000" })); diff --git a/crates/opensymphony-memory/src/index.rs b/crates/opensymphony-memory/src/index.rs index 2eb4e58b..cb7115a0 100644 --- a/crates/opensymphony-memory/src/index.rs +++ b/crates/opensymphony-memory/src/index.rs @@ -1,3 +1,4 @@ +use duckdb::types::Value as SqlValue; const UNDATED_LOG_DATE: &str = "1970-01-01"; const LIVE_CAPTURE_OWNER: &str = "__live_capture__"; const LIVE_CAPTURE_OWNER_PREFIX: &str = "__live_capture__:"; @@ -762,8 +763,149 @@ fn code_edges_read_model_ready(connection: &Connection, path: &Path) -> Result Result<(), duckdb::Error> { - connection.execute_batch(&format!( + let fingerprint = memory_migration_fingerprint(); + if applied_memory_migration_fingerprint(connection)?.as_deref() == Some(fingerprint) { + return Ok(()); + } + apply_memory_migration(connection)?; + connection.execute_batch( + "CREATE TABLE IF NOT EXISTS memory_migrations (component TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, applied_at TEXT NOT NULL);", + )?; + connection.execute( + "INSERT OR REPLACE INTO memory_migrations (component, fingerprint, applied_at) VALUES ('memory', ?, CAST(current_timestamp AS TEXT))", + params![fingerprint], + )?; + Ok(()) +} + +#[cfg(test)] +mod migration_gate_tests { + use super::{applied_memory_migration_fingerprint, memory_migration_fingerprint, migrate_index}; + use duckdb::Connection; + + fn has_schema_version_table(connection: &Connection) -> bool { + connection + .query_row( + "SELECT count(*) FROM information_schema.tables WHERE table_name = 'schema_version'", + [], + |row| row.get::<_, i64>(0), + ) + .expect("table listing") + == 1 + } + + #[test] + fn migration_runs_once_per_fingerprint() { + let connection = Connection::open_in_memory().expect("in-memory index"); + assert_eq!(applied_memory_migration_fingerprint(&connection).expect("read"), None); + + migrate_index(&connection).expect("first migration"); + assert_eq!( + applied_memory_migration_fingerprint(&connection) + .expect("read") + .as_deref(), + Some(memory_migration_fingerprint()) + ); + assert!(has_schema_version_table(&connection)); + + // A matching fingerprint skips the migration, so a table removed + // behind its back stays removed. + connection + .execute_batch("DROP TABLE schema_version;") + .expect("drop table"); + migrate_index(&connection).expect("skipped migration"); + assert!(!has_schema_version_table(&connection)); + + // A different recorded fingerprint runs the migration again. + connection + .execute_batch("UPDATE memory_migrations SET fingerprint = 'previous-release';") + .expect("record an older migration"); + migrate_index(&connection).expect("repeated migration"); + assert!(has_schema_version_table(&connection)); + assert_eq!( + applied_memory_migration_fingerprint(&connection) + .expect("read") + .as_deref(), + Some(memory_migration_fingerprint()) + ); + } +} + +fn applied_memory_migration_fingerprint( + connection: &Connection, +) -> Result, duckdb::Error> { + let recorded: i64 = connection.query_row( + "SELECT count(*) FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'memory_migrations'", + [], + |row| row.get(0), + )?; + if recorded == 0 { + return Ok(None); + } + connection + .query_row( + "SELECT fingerprint FROM memory_migrations WHERE component = 'memory'", + [], + |row| row.get(0), + ) + .optional() +} + +fn memory_migration_fingerprint() -> &'static str { + static FINGERPRINT: OnceLock = OnceLock::new(); + FINGERPRINT.get_or_init(|| { + sha256_hex(&format!( + "{}\u{1f}{}\u{1f}{}\u{1f}{}", + env!("CARGO_PKG_VERSION"), + MEMORY_MIGRATION_RUST_STEPS_REVISION, + MEMORY_CODE_TABLES.join(","), + memory_migration_sql(), + )) + }) +} + +fn apply_memory_migration(connection: &Connection) -> Result<(), duckdb::Error> { + connection.execute_batch(memory_migration_sql())?; + for table in MEMORY_CODE_TABLES { + ensure_worktree_dirty_column(connection, table)?; + ensure_legacy_source_column(connection, table)?; + } + Ok(()) +} + +fn memory_migration_sql() -> &'static str { + static SQL: OnceLock = OnceLock::new(); + SQL.get_or_init(|| { + format!( r#" CREATE TABLE IF NOT EXISTS schema_version ( component TEXT PRIMARY KEY, @@ -1206,26 +1348,8 @@ CREATE INDEX IF NOT EXISTS idx_code_snapshot_membership_staging_repo ON code_sna CREATE INDEX IF NOT EXISTS idx_code_documents_staging_repo ON code_documents_staging(repo_id, commit_sha); CREATE INDEX IF NOT EXISTS idx_code_skipped_files_staging_repo ON code_skipped_files_staging(repo_id, commit_sha); "#, - ))?; - for table in [ - "code_documents", - "code_documents_staging", - "code_document_revisions", - "code_symbols", - "code_edges", - "code_edge_revisions", - "code_skipped_files", - "code_skipped_files_staging", - "code_diagnostics", - "code_diagnostic_revisions", - "code_index_snapshots", - "code_snapshot_membership", - "code_snapshot_membership_staging", - ] { - ensure_worktree_dirty_column(connection, table)?; - ensure_legacy_source_column(connection, table)?; - } - Ok(()) + ) + }) } fn ensure_worktree_dirty_column(connection: &Connection, table: &str) -> Result<(), duckdb::Error> { @@ -1281,12 +1405,29 @@ pub(crate) fn persist_code_intel_documents_with_freshness( path: config.index_path.clone(), source, })?; - let transaction = connection - .transaction() - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })?; + persist_code_intel_documents_with_freshness_on( + &mut connection, + config, + batch, + freshness, + stale_existing, + ) +} + +/// Persists one document batch on a connection the caller already opened and +/// migrated, so an index run can reuse one connection for every batch. +pub(crate) fn persist_code_intel_documents_with_freshness_on( + connection: &mut Connection, + config: &MemoryConfig, + batch: CodeIntelPersistBatch, + freshness: &str, + stale_existing: bool, +) -> Result { + let duckdb_error = |source| MemoryError::DuckDb { + path: config.index_path.clone(), + source, + }; + let transaction = connection.transaction().map_err(duckdb_error)?; if freshness == "current" && !batch.worktree_dirty && !config.repository_sources.is_empty() @@ -1299,12 +1440,7 @@ pub(crate) fn persist_code_intel_documents_with_freshness( ) { Ok(commit) => Some(commit), Err(duckdb::Error::QueryReturnedNoRows) => None, - Err(source) => { - return Err(MemoryError::DuckDb { - path: config.index_path.clone(), - source, - }); - } + Err(source) => return Err(duckdb_error(source)), }; if let Some(registered_commit) = registered_commit && registered_commit != commit_sha @@ -1315,6 +1451,11 @@ pub(crate) fn persist_code_intel_documents_with_freshness( ))); } } + if freshness == "staged" && batch.commit_sha.is_none() && !batch.documents.is_empty() { + return Err(MemoryError::InvalidInput( + "staged code documents require a commit revision".to_string(), + )); + } let indexed_at = Utc::now().to_rfc3339(); let mut report = CodeIntelPersistReport { parsed_files: batch.documents.len(), @@ -1326,10 +1467,43 @@ pub(crate) fn persist_code_intel_documents_with_freshness( skipped_files: Vec::new(), diagnostics: Vec::new(), }; - let worktree_dirty = if batch.worktree_dirty { 1_i64 } else { 0_i64 }; - for document in batch.documents { + // Rows are buffered and written with multi-row upserts unless a + // per-document stale update must run between one document's rows and the + // next. That only happens when stale marking is requested and two + // documents share a path; distinct paths are staled for the whole batch + // up front, which matches staling each document just before its rows. + let stale_as_set = stale_existing && code_document_paths_are_distinct(&batch.documents); + let buffered = !stale_existing || stale_as_set; + if stale_as_set { + report.stale_rows += stale_code_rows_for_documents( + &transaction, + &batch.repo_id, + batch.commit_sha.as_deref(), + &batch.documents, + ) + .map_err(duckdb_error)?; + } + let mut upserts = CodeIntelUpserts::new(buffered); + // Staged rows never create current rows, so when nothing is staled the + // current symbols and edges that suppress staged copies are the ones that + // exist before the batch, and one lookup per table finds them all. + let preloaded_current = if freshness == "staged" && !stale_existing { + Some(CurrentCodeRowIds::load( + &transaction, + &batch.repo_id, + batch.commit_sha.as_deref(), + batch.worktree_dirty, + &batch.documents, + ) + .map_err(duckdb_error)?) + } else { + None + }; + let worktree_dirty = batch.worktree_dirty; + let revision_commit = batch.commit_sha.as_deref().filter(|_| !batch.worktree_dirty); + for document in &batch.documents { let path = document.path.to_string_lossy().to_string(); - if stale_existing { + if stale_existing && !stale_as_set { report.stale_rows += stale_code_rows( &transaction, &CodeFreshnessKey { @@ -1341,91 +1515,75 @@ pub(crate) fn persist_code_intel_documents_with_freshness( query_pack_version: &document.query_pack_version, }, ) - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })?; + .map_err(duckdb_error)?; } if freshness == "staged" { - let Some(commit_sha) = batch.commit_sha.as_deref() else { - return Err(MemoryError::InvalidInput( - "staged code documents require a commit revision".to_string(), - )); - }; - transaction - .execute( - "INSERT OR REPLACE INTO code_documents_staging (repo_id, commit_sha, worktree_dirty, path, language, content_sha256, parser_id, parser_version, query_pack_version, byte_len, line_count, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - params![ - batch.repo_id, - commit_sha, - worktree_dirty, - path, - document.language, - document.content_sha256, - document.parser_id, - document.parser_version, - document.query_pack_version, - document.byte_len as i64, - document.line_count as i64, - indexed_at, + let commit_sha = batch.commit_sha.as_deref().unwrap_or_default(); + upserts + .documents_staging + .write( + &transaction, + vec![ + sql_text(&batch.repo_id), + sql_text(commit_sha), + SqlValue::Boolean(worktree_dirty), + sql_text(&path), + sql_text(&document.language), + sql_text(&document.content_sha256), + sql_text(&document.parser_id), + sql_text(&document.parser_version), + sql_text(&document.query_pack_version), + SqlValue::BigInt(document.byte_len as i64), + SqlValue::BigInt(document.line_count as i64), + sql_text(&indexed_at), ], ) - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })?; - report.persisted_documents += 1; + .map_err(duckdb_error)?; } else { - transaction - .execute( - "INSERT OR REPLACE INTO code_documents (repo_id, commit_sha, worktree_dirty, path, language, content_sha256, parser_id, parser_version, query_pack_version, byte_len, line_count, indexed_at, freshness) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - params![ - batch.repo_id, - batch.commit_sha.clone(), - worktree_dirty, - path, - document.language, - document.content_sha256, - document.parser_id, - document.parser_version, - document.query_pack_version, - document.byte_len as i64, - document.line_count as i64, - indexed_at, - freshness, + upserts + .documents + .write( + &transaction, + vec![ + sql_text(&batch.repo_id), + sql_optional_text(batch.commit_sha.as_deref()), + SqlValue::Boolean(worktree_dirty), + sql_text(&path), + sql_text(&document.language), + sql_text(&document.content_sha256), + sql_text(&document.parser_id), + sql_text(&document.parser_version), + sql_text(&document.query_pack_version), + SqlValue::BigInt(document.byte_len as i64), + SqlValue::BigInt(document.line_count as i64), + sql_text(&indexed_at), + sql_text(freshness), ], ) - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })?; - report.persisted_documents += 1; + .map_err(duckdb_error)?; } - if !batch.worktree_dirty - && let Some(commit_sha) = batch.commit_sha.as_deref() - { - transaction - .execute( - "INSERT OR REPLACE INTO code_document_revisions (repo_id, commit_sha, worktree_dirty, path, language, content_sha256, parser_id, parser_version, query_pack_version, indexed_at, freshness) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - params![ - batch.repo_id, - commit_sha, - worktree_dirty, - path, - document.language, - document.content_sha256, - document.parser_id, - document.parser_version, - document.query_pack_version, - indexed_at, - freshness, + report.persisted_documents += 1; + if let Some(commit_sha) = revision_commit { + upserts + .document_revisions + .write( + &transaction, + vec![ + sql_text(&batch.repo_id), + sql_text(commit_sha), + SqlValue::Boolean(worktree_dirty), + sql_text(&path), + sql_text(&document.language), + sql_text(&document.content_sha256), + sql_text(&document.parser_id), + sql_text(&document.parser_version), + sql_text(&document.query_pack_version), + sql_text(&indexed_at), + sql_text(freshness), ], ) - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })?; + .map_err(duckdb_error)?; } let prepared_symbols = prepare_code_symbols( @@ -1433,11 +1591,15 @@ pub(crate) fn persist_code_intel_documents_with_freshness( batch.commit_sha.as_deref(), batch.worktree_dirty, &path, - &document, + document, ); for prepared in &prepared_symbols { let symbol = prepared.symbol; - let current_symbol_exists = if freshness == "staged" { + let current_symbol_exists = if freshness != "staged" { + false + } else if let Some(preloaded) = &preloaded_current { + preloaded.symbols.contains(&prepared.symbol_id) + } else { transaction .query_row( "SELECT 1 FROM code_symbols WHERE symbol_id = ? AND freshness = 'current' LIMIT 1", @@ -1445,51 +1607,44 @@ pub(crate) fn persist_code_intel_documents_with_freshness( |row| row.get::<_, i64>(0), ) .optional() - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })? + .map_err(duckdb_error)? .is_some() - } else { - false }; if !current_symbol_exists { - transaction - .execute( - "INSERT OR REPLACE INTO code_symbols (symbol_id, symbol_key, repo_id, commit_sha, worktree_dirty, path, language, kind, name, container_symbol_id, container_chain, signature, start_line, start_col, end_line, end_col, start_byte, end_byte, selection_start_line, selection_end_line, content_sha256, snippet_sha256, parser_version, query_pack_version, indexed_at, freshness) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - params![ - prepared.symbol_id, - prepared.symbol_key, - batch.repo_id, - batch.commit_sha.clone(), - worktree_dirty, - path, - document.language, - symbol.kind, - symbol.name, - prepared.container_symbol_id, - symbol.container_chain.join("\u{1f}"), - symbol.signature, - symbol.start_line as i64, - symbol.start_col as i64, - symbol.end_line as i64, - symbol.end_col as i64, - symbol.start_byte as i64, - symbol.end_byte as i64, - symbol.selection_start_line as i64, - symbol.selection_end_line as i64, - document.content_sha256, - symbol.snippet_sha256, - document.parser_version, - document.query_pack_version, - indexed_at, - freshness, + upserts + .symbols + .write( + &transaction, + vec![ + sql_text(&prepared.symbol_id), + sql_text(&prepared.symbol_key), + sql_text(&batch.repo_id), + sql_optional_text(batch.commit_sha.as_deref()), + SqlValue::Boolean(worktree_dirty), + sql_text(&path), + sql_text(&document.language), + sql_text(&symbol.kind), + sql_text(&symbol.name), + sql_optional_text(prepared.container_symbol_id.as_deref()), + sql_text(&symbol.container_chain.join("\u{1f}")), + sql_optional_text(symbol.signature.as_deref()), + SqlValue::BigInt(symbol.start_line as i64), + SqlValue::BigInt(symbol.start_col as i64), + SqlValue::BigInt(symbol.end_line as i64), + SqlValue::BigInt(symbol.end_col as i64), + SqlValue::BigInt(symbol.start_byte as i64), + SqlValue::BigInt(symbol.end_byte as i64), + SqlValue::BigInt(symbol.selection_start_line as i64), + SqlValue::BigInt(symbol.selection_end_line as i64), + sql_text(&document.content_sha256), + sql_text(&symbol.snippet_sha256), + sql_text(&document.parser_version), + sql_text(&document.query_pack_version), + sql_text(&indexed_at), + sql_text(freshness), ], ) - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })?; + .map_err(duckdb_error)?; report.persisted_symbols += 1; } } @@ -1498,22 +1653,12 @@ pub(crate) fn persist_code_intel_documents_with_freshness( let resolved = resolve_code_edge(edge, &prepared_symbols); let normalized_confidence = normalize_edge_confidence(&edge.confidence, resolved.target_resolved); - let edge_id = code_row_id(&[ - &batch.repo_id, - &path, - &document.content_sha256, - &document.parser_version, - &document.query_pack_version, - &edge.edge_kind, - edge.target_hint.as_deref().unwrap_or(""), - &edge.start_line.to_string(), - &edge.start_col.to_string(), - &edge.end_line.to_string(), - &edge.end_col.to_string(), - &edge.start_byte.to_string(), - &edge.end_byte.to_string(), - ]); - let current_edge_exists = if freshness == "staged" { + let edge_id = code_edge_row_id(&batch.repo_id, &path, document, edge); + let current_edge_exists = if freshness != "staged" { + false + } else if let Some(preloaded) = &preloaded_current { + preloaded.edges.contains(&edge_id) + } else { transaction .query_row( "SELECT 1 FROM code_edges WHERE edge_id = ? AND freshness = 'current' AND NOT worktree_dirty LIMIT 1", @@ -1521,87 +1666,51 @@ pub(crate) fn persist_code_intel_documents_with_freshness( |row| row.get::<_, i64>(0), ) .optional() - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })? + .map_err(duckdb_error)? .is_some() - } else { - false + }; + let edge_values = |commit_sha: SqlValue| { + vec![ + sql_text(&edge_id), + sql_text(&batch.repo_id), + commit_sha, + SqlValue::Boolean(worktree_dirty), + sql_text(&path), + sql_text(&document.language), + sql_text(&edge.edge_kind), + sql_optional_text(resolved.source_symbol_id.as_deref()), + sql_optional_text(resolved.source_symbol_key.as_deref()), + sql_optional_text(resolved.target_symbol_id.as_deref()), + sql_optional_text(resolved.target_symbol_key.as_deref()), + sql_optional_text(edge.target_hint.as_deref()), + sql_text(normalized_confidence), + SqlValue::BigInt(edge.start_line as i64), + SqlValue::BigInt(edge.start_col as i64), + SqlValue::BigInt(edge.end_line as i64), + SqlValue::BigInt(edge.end_col as i64), + SqlValue::BigInt(edge.start_byte as i64), + SqlValue::BigInt(edge.end_byte as i64), + sql_text(&document.content_sha256), + sql_text(&document.parser_version), + sql_text(&document.query_pack_version), + sql_text(&indexed_at), + sql_text(freshness), + ] }; if !current_edge_exists { - transaction - .execute( - "INSERT OR REPLACE INTO code_edges (edge_id, repo_id, commit_sha, worktree_dirty, path, language, edge_kind, source_symbol_id, source_symbol_key, target_symbol_id, target_symbol_key, target_hint, confidence, start_line, start_col, end_line, end_col, start_byte, end_byte, content_sha256, parser_version, query_pack_version, indexed_at, freshness) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - params![ - edge_id, - batch.repo_id, - batch.commit_sha.clone(), - worktree_dirty, - path, - document.language, - edge.edge_kind, - resolved.source_symbol_id, - resolved.source_symbol_key, - resolved.target_symbol_id, - resolved.target_symbol_key, - edge.target_hint, - normalized_confidence, - edge.start_line as i64, - edge.start_col as i64, - edge.end_line as i64, - edge.end_col as i64, - edge.start_byte as i64, - edge.end_byte as i64, - document.content_sha256, - document.parser_version, - document.query_pack_version, - indexed_at, - freshness, - ], + upserts + .edges + .write( + &transaction, + edge_values(sql_optional_text(batch.commit_sha.as_deref())), ) - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })?; + .map_err(duckdb_error)?; } - if !batch.worktree_dirty - && let Some(commit_sha) = batch.commit_sha.as_deref() - { - transaction - .execute( - "INSERT OR REPLACE INTO code_edge_revisions (edge_id, repo_id, commit_sha, worktree_dirty, path, language, edge_kind, source_symbol_id, source_symbol_key, target_symbol_id, target_symbol_key, target_hint, confidence, start_line, start_col, end_line, end_col, start_byte, end_byte, content_sha256, parser_version, query_pack_version, indexed_at, freshness) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - params![ - edge_id, - batch.repo_id, - commit_sha, - worktree_dirty, - path, - document.language, - edge.edge_kind, - resolved.source_symbol_id, - resolved.source_symbol_key, - resolved.target_symbol_id, - resolved.target_symbol_key, - edge.target_hint, - normalized_confidence, - edge.start_line as i64, - edge.start_col as i64, - edge.end_line as i64, - edge.end_col as i64, - edge.start_byte as i64, - edge.end_byte as i64, - document.content_sha256, - document.parser_version, - document.query_pack_version, - indexed_at, - freshness, - ], - ) - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })?; + if let Some(commit_sha) = revision_commit { + upserts + .edge_revisions + .write(&transaction, edge_values(sql_text(commit_sha))) + .map_err(duckdb_error)?; } report.persisted_edges += 1; } @@ -1631,83 +1740,443 @@ pub(crate) fn persist_code_intel_documents_with_freshness( } else { diagnostic_id.clone() }; - transaction - .execute( - "INSERT OR REPLACE INTO code_diagnostics (diagnostic_id, repo_id, commit_sha, worktree_dirty, path, language, kind, severity, message, start_line, start_col, end_line, end_col, start_byte, end_byte, content_sha256, parser_version, query_pack_version, indexed_at, freshness) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - params![ - &read_model_diagnostic_id, - batch.repo_id, - batch.commit_sha.clone(), - worktree_dirty, - path, - document.language, - diagnostic.kind, - diagnostic.severity, - diagnostic.message, - diagnostic.start_line as i64, - diagnostic.start_col as i64, - diagnostic.end_line as i64, - diagnostic.end_col as i64, - diagnostic.start_byte as i64, - diagnostic.end_byte as i64, - document.content_sha256, - document.parser_version, - document.query_pack_version, - indexed_at, - freshness, - ], - ) - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })?; - if !batch.worktree_dirty - && let Some(commit_sha) = batch.commit_sha.as_deref() - { - transaction - .execute( - "INSERT OR REPLACE INTO code_diagnostic_revisions (diagnostic_id, repo_id, commit_sha, worktree_dirty, path, language, kind, severity, message, start_line, start_col, end_line, end_col, start_byte, end_byte, content_sha256, parser_version, query_pack_version, indexed_at, freshness) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - params![ - &diagnostic_id, - batch.repo_id, - commit_sha, - worktree_dirty, - path, - document.language, - diagnostic.kind, - diagnostic.severity, - diagnostic.message, - diagnostic.start_line as i64, - diagnostic.start_col as i64, - diagnostic.end_line as i64, - diagnostic.end_col as i64, - diagnostic.start_byte as i64, - diagnostic.end_byte as i64, - document.content_sha256, - document.parser_version, - document.query_pack_version, - indexed_at, - freshness, - ], + let diagnostic_values = |row_id: &str, commit_sha: SqlValue| { + vec![ + sql_text(row_id), + sql_text(&batch.repo_id), + commit_sha, + SqlValue::Boolean(worktree_dirty), + sql_text(&path), + sql_text(&document.language), + sql_text(&diagnostic.kind), + sql_text(&diagnostic.severity), + sql_text(&diagnostic.message), + SqlValue::BigInt(diagnostic.start_line as i64), + SqlValue::BigInt(diagnostic.start_col as i64), + SqlValue::BigInt(diagnostic.end_line as i64), + SqlValue::BigInt(diagnostic.end_col as i64), + SqlValue::BigInt(diagnostic.start_byte as i64), + SqlValue::BigInt(diagnostic.end_byte as i64), + sql_text(&document.content_sha256), + sql_text(&document.parser_version), + sql_text(&document.query_pack_version), + sql_text(&indexed_at), + sql_text(freshness), + ] + }; + upserts + .diagnostics + .write( + &transaction, + diagnostic_values( + &read_model_diagnostic_id, + sql_optional_text(batch.commit_sha.as_deref()), + ), + ) + .map_err(duckdb_error)?; + if let Some(commit_sha) = revision_commit { + upserts + .diagnostic_revisions + .write( + &transaction, + diagnostic_values(&diagnostic_id, sql_text(commit_sha)), ) - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })?; + .map_err(duckdb_error)?; } report.persisted_diagnostics += 1; } } - transaction - .commit() - .map_err(|source| MemoryError::DuckDb { - path: config.index_path.clone(), - source, - })?; + upserts.flush(&transaction).map_err(duckdb_error)?; + transaction.commit().map_err(duckdb_error)?; Ok(report) } +fn sql_text(value: &str) -> SqlValue { + SqlValue::Text(value.to_owned()) +} + +fn sql_optional_text(value: Option<&str>) -> SqlValue { + value.map_or(SqlValue::Null, sql_text) +} + +fn code_edge_row_id( + repo_id: &str, + path: &str, + document: &CodeIntelDocumentInput, + edge: &CodeIntelEdgeInput, +) -> String { + code_row_id(&[ + repo_id, + path, + &document.content_sha256, + &document.parser_version, + &document.query_pack_version, + &edge.edge_kind, + edge.target_hint.as_deref().unwrap_or(""), + &edge.start_line.to_string(), + &edge.start_col.to_string(), + &edge.end_line.to_string(), + &edge.end_col.to_string(), + &edge.start_byte.to_string(), + &edge.end_byte.to_string(), + ]) +} + +/// Rows written per multi-row `INSERT OR REPLACE` statement. +const CODE_UPSERT_ROWS_PER_STATEMENT: usize = 256; + +/// `INSERT OR REPLACE` writes for one code table. When buffered, rows are +/// collected and written with multi-row statements on `flush`; otherwise each +/// row is written immediately. +struct CodeRowUpserts { + table: &'static str, + columns: &'static [&'static str], + key_columns: &'static [usize], + buffered: bool, + rows: Vec>, +} + +impl CodeRowUpserts { + fn new( + table: &'static str, + columns: &'static [&'static str], + key_columns: &'static [usize], + buffered: bool, + ) -> Self { + Self { + table, + columns, + key_columns, + buffered, + rows: Vec::new(), + } + } + + fn statement(&self, rows: usize) -> String { + let placeholders = format!("({})", vec!["?"; self.columns.len()].join(", ")); + format!( + "INSERT OR REPLACE INTO {} ({}) VALUES {}", + self.table, + self.columns.join(", "), + vec![placeholders; rows].join(", ") + ) + } + + fn write(&mut self, connection: &Connection, row: Vec) -> Result<(), duckdb::Error> { + debug_assert_eq!(row.len(), self.columns.len()); + if self.buffered { + self.rows.push(row); + return Ok(()); + } + connection.execute(&self.statement(1), duckdb::params_from_iter(row))?; + Ok(()) + } + + /// Writes buffered rows. A statement may not upsert one key twice, so + /// only the last row per primary key is written, which leaves the table + /// as writing each row in turn would. + fn flush(&mut self, connection: &Connection) -> Result<(), duckdb::Error> { + let rows = std::mem::take(&mut self.rows); + let mut last_row_for_key = BTreeMap::new(); + for (index, row) in rows.iter().enumerate() { + let key = self + .key_columns + .iter() + .map(|column| format!("{:?}", row[*column])) + .collect::>(); + last_row_for_key.insert(key, index); + } + let mut kept = last_row_for_key.into_values().collect::>(); + kept.sort_unstable(); + for chunk in kept.chunks(CODE_UPSERT_ROWS_PER_STATEMENT) { + connection.execute( + &self.statement(chunk.len()), + duckdb::params_from_iter(chunk.iter().flat_map(|index| rows[*index].iter())), + )?; + } + Ok(()) + } +} + +/// The upserts of one document batch, one per code table. +struct CodeIntelUpserts { + documents: CodeRowUpserts, + documents_staging: CodeRowUpserts, + document_revisions: CodeRowUpserts, + symbols: CodeRowUpserts, + edges: CodeRowUpserts, + edge_revisions: CodeRowUpserts, + diagnostics: CodeRowUpserts, + diagnostic_revisions: CodeRowUpserts, +} + +const CODE_EDGE_COLUMNS: &[&str] = &[ + "edge_id", + "repo_id", + "commit_sha", + "worktree_dirty", + "path", + "language", + "edge_kind", + "source_symbol_id", + "source_symbol_key", + "target_symbol_id", + "target_symbol_key", + "target_hint", + "confidence", + "start_line", + "start_col", + "end_line", + "end_col", + "start_byte", + "end_byte", + "content_sha256", + "parser_version", + "query_pack_version", + "indexed_at", + "freshness", +]; + +const CODE_DIAGNOSTIC_COLUMNS: &[&str] = &[ + "diagnostic_id", + "repo_id", + "commit_sha", + "worktree_dirty", + "path", + "language", + "kind", + "severity", + "message", + "start_line", + "start_col", + "end_line", + "end_col", + "start_byte", + "end_byte", + "content_sha256", + "parser_version", + "query_pack_version", + "indexed_at", + "freshness", +]; + +impl CodeIntelUpserts { + fn new(buffered: bool) -> Self { + Self { + // PRIMARY KEY (repo_id, path, content_sha256, parser_version, query_pack_version) + documents: CodeRowUpserts::new( + "code_documents", + &[ + "repo_id", + "commit_sha", + "worktree_dirty", + "path", + "language", + "content_sha256", + "parser_id", + "parser_version", + "query_pack_version", + "byte_len", + "line_count", + "indexed_at", + "freshness", + ], + &[0, 3, 5, 7, 8], + buffered, + ), + // PRIMARY KEY (repo_id, commit_sha, path, parser_version, query_pack_version) + documents_staging: CodeRowUpserts::new( + "code_documents_staging", + &[ + "repo_id", + "commit_sha", + "worktree_dirty", + "path", + "language", + "content_sha256", + "parser_id", + "parser_version", + "query_pack_version", + "byte_len", + "line_count", + "indexed_at", + ], + &[0, 1, 3, 7, 8], + buffered, + ), + // PRIMARY KEY (repo_id, commit_sha, path, parser_version, query_pack_version) + document_revisions: CodeRowUpserts::new( + "code_document_revisions", + &[ + "repo_id", + "commit_sha", + "worktree_dirty", + "path", + "language", + "content_sha256", + "parser_id", + "parser_version", + "query_pack_version", + "indexed_at", + "freshness", + ], + &[0, 1, 3, 7, 8], + buffered, + ), + // PRIMARY KEY (symbol_id) + symbols: CodeRowUpserts::new( + "code_symbols", + &[ + "symbol_id", + "symbol_key", + "repo_id", + "commit_sha", + "worktree_dirty", + "path", + "language", + "kind", + "name", + "container_symbol_id", + "container_chain", + "signature", + "start_line", + "start_col", + "end_line", + "end_col", + "start_byte", + "end_byte", + "selection_start_line", + "selection_end_line", + "content_sha256", + "snippet_sha256", + "parser_version", + "query_pack_version", + "indexed_at", + "freshness", + ], + &[0], + buffered, + ), + // PRIMARY KEY (edge_id) + edges: CodeRowUpserts::new("code_edges", CODE_EDGE_COLUMNS, &[0], buffered), + // PRIMARY KEY (repo_id, commit_sha, edge_id) + edge_revisions: CodeRowUpserts::new( + "code_edge_revisions", + CODE_EDGE_COLUMNS, + &[1, 2, 0], + buffered, + ), + // PRIMARY KEY (diagnostic_id) + diagnostics: CodeRowUpserts::new( + "code_diagnostics", + CODE_DIAGNOSTIC_COLUMNS, + &[0], + buffered, + ), + // PRIMARY KEY (repo_id, commit_sha, diagnostic_id) + diagnostic_revisions: CodeRowUpserts::new( + "code_diagnostic_revisions", + CODE_DIAGNOSTIC_COLUMNS, + &[1, 2, 0], + buffered, + ), + } + } + + fn flush(&mut self, connection: &Connection) -> Result<(), duckdb::Error> { + for upserts in [ + &mut self.documents, + &mut self.documents_staging, + &mut self.document_revisions, + &mut self.symbols, + &mut self.edges, + &mut self.edge_revisions, + &mut self.diagnostics, + &mut self.diagnostic_revisions, + ] { + upserts.flush(connection)?; + } + Ok(()) + } +} + +/// Ids of current symbol and edge rows that a staged batch would otherwise +/// copy, read with one query per table. +struct CurrentCodeRowIds { + symbols: BTreeSet, + edges: BTreeSet, +} + +impl CurrentCodeRowIds { + fn load( + connection: &Connection, + repo_id: &str, + commit_sha: Option<&str>, + worktree_dirty: bool, + documents: &[CodeIntelDocumentInput], + ) -> Result { + let mut symbol_ids = Vec::new(); + let mut edge_ids = Vec::new(); + for document in documents { + let path = document.path.to_string_lossy().to_string(); + symbol_ids.extend( + prepare_code_symbols(repo_id, commit_sha, worktree_dirty, &path, document) + .into_iter() + .map(|prepared| prepared.symbol_id), + ); + edge_ids.extend( + document + .edges + .iter() + .map(|edge| code_edge_row_id(repo_id, &path, document, edge)), + ); + } + Ok(Self { + symbols: existing_code_row_ids( + connection, + &symbol_ids, + "SELECT DISTINCT current.symbol_id FROM code_symbols AS current JOIN temp.{ids} AS candidate ON current.symbol_id = candidate.id WHERE current.freshness = 'current'", + )?, + edges: existing_code_row_ids( + connection, + &edge_ids, + "SELECT DISTINCT current.edge_id FROM code_edges AS current JOIN temp.{ids} AS candidate ON current.edge_id = candidate.id WHERE current.freshness = 'current' AND NOT current.worktree_dirty", + )?, + }) + } +} + +/// Connection-local temporary table holding candidate row ids. +const CODE_CANDIDATE_IDS_TABLE: &str = "opensymphony_code_candidate_ids"; + +/// Returns the ids among `ids` that `query` finds, where `{ids}` in `query` +/// names a temporary table holding the candidates. +fn existing_code_row_ids( + connection: &Connection, + ids: &[String], + query: &str, +) -> Result, duckdb::Error> { + if ids.is_empty() { + return Ok(BTreeSet::new()); + } + connection.execute_batch(&format!( + "CREATE TEMP TABLE IF NOT EXISTS {CODE_CANDIDATE_IDS_TABLE} (id TEXT NOT NULL); DELETE FROM temp.{CODE_CANDIDATE_IDS_TABLE};" + ))?; + { + let mut appender = + connection.appender_to_catalog_and_db(CODE_CANDIDATE_IDS_TABLE, "temp", "main")?; + for id in ids { + appender.append_row(params![id])?; + } + appender.flush()?; + } + let mut statement = connection.prepare(&query.replace("{ids}", CODE_CANDIDATE_IDS_TABLE))?; + statement + .query_map([], |row| row.get::<_, String>(0))? + .collect() +} + pub fn persist_code_intel_skipped_files( config: &MemoryConfig, repo_id: &str, @@ -1740,10 +2209,7 @@ pub(crate) fn persist_code_intel_skipped_files_with_freshness( freshness: &str, stale_existing: bool, ) -> Result { - let Some(commit_sha) = commit_sha.filter(|_| !worktree_dirty) else { - return Ok(0); - }; - if skipped_files.is_empty() { + if commit_sha.is_none() || worktree_dirty || skipped_files.is_empty() { return Ok(0); } let mut connection = open_index(config)?; @@ -1751,6 +2217,37 @@ pub(crate) fn persist_code_intel_skipped_files_with_freshness( path: config.index_path.clone(), source, })?; + persist_code_intel_skipped_files_with_freshness_on( + &mut connection, + config, + repo_id, + commit_sha, + worktree_dirty, + skipped_files, + freshness, + stale_existing, + ) +} + +/// Persists skipped-file rows on a connection the caller already opened and +/// migrated. +#[allow(clippy::too_many_arguments)] +pub(crate) fn persist_code_intel_skipped_files_with_freshness_on( + connection: &mut Connection, + config: &MemoryConfig, + repo_id: &str, + commit_sha: Option<&str>, + worktree_dirty: bool, + skipped_files: &[CodeIntelSkippedFileInput], + freshness: &str, + stale_existing: bool, +) -> Result { + let Some(commit_sha) = commit_sha.filter(|_| !worktree_dirty) else { + return Ok(0); + }; + if skipped_files.is_empty() { + return Ok(0); + } let transaction = connection .transaction() .map_err(|source| MemoryError::DuckDb { @@ -2669,6 +3166,68 @@ struct CodeFreshnessKey<'a> { query_pack_version: &'a str, } +/// Connection-local temporary table holding the freshness keys of one +/// document batch. +const CODE_FRESHNESS_KEYS_TABLE: &str = "opensymphony_code_freshness_keys"; + +/// Whether every document in a batch has its own path. +fn code_document_paths_are_distinct(documents: &[CodeIntelDocumentInput]) -> bool { + let mut paths = BTreeSet::new(); + documents + .iter() + .all(|document| paths.insert(document.path.to_string_lossy())) +} + +/// Marks superseded rows stale for a whole document batch with one statement +/// per table. Callers use it only when document paths are distinct: every +/// existing row then matches at most one document, so the rows and counts are +/// those of calling `stale_code_rows` before each document's inserts. +fn stale_code_rows_for_documents( + connection: &Connection, + repo_id: &str, + commit_sha: Option<&str>, + documents: &[CodeIntelDocumentInput], +) -> Result { + connection.execute_batch(&format!( + "CREATE TEMP TABLE IF NOT EXISTS {CODE_FRESHNESS_KEYS_TABLE} (path TEXT NOT NULL, content_sha256 TEXT NOT NULL, parser_version TEXT NOT NULL, query_pack_version TEXT NOT NULL); DELETE FROM temp.{CODE_FRESHNESS_KEYS_TABLE};" + ))?; + { + let mut appender = connection.appender_to_catalog_and_db(CODE_FRESHNESS_KEYS_TABLE, "temp", "main")?; + for document in documents { + appender.append_row(params![ + document.path.to_string_lossy().to_string(), + document.content_sha256, + document.parser_version, + document.query_pack_version, + ])?; + } + appender.flush()?; + } + let mut stale_rows = 0; + for table in [ + "code_documents", + "code_symbols", + "code_edges", + "code_diagnostics", + ] { + stale_rows += connection.execute( + &format!( + "UPDATE {table} AS existing SET freshness = 'stale' FROM temp.{CODE_FRESHNESS_KEYS_TABLE} AS incoming WHERE existing.repo_id = ? AND existing.path = incoming.path AND existing.freshness = 'current' AND NOT (existing.content_sha256 = incoming.content_sha256 AND existing.parser_version = incoming.parser_version AND existing.query_pack_version = incoming.query_pack_version)" + ), + params![repo_id], + )?; + } + if let Some(commit_sha) = commit_sha { + stale_rows += connection.execute( + &format!( + "UPDATE code_skipped_files SET freshness = 'stale' WHERE repo_id = ? AND commit_sha = ? AND freshness = 'current' AND path IN (SELECT path FROM temp.{CODE_FRESHNESS_KEYS_TABLE})" + ), + params![repo_id, commit_sha], + )?; + } + Ok(stale_rows) +} + fn stale_code_rows( connection: &Connection, key: &CodeFreshnessKey<'_>, diff --git a/docs/code-intelligence.md b/docs/code-intelligence.md index 5ecbe355..400907d4 100644 --- a/docs/code-intelligence.md +++ b/docs/code-intelligence.md @@ -84,6 +84,14 @@ membership and only changed, added, or deleted paths are parsed or staled. DuckDB writes are serialized through the repository index writer; reads remain available while a gateway request performs the background job. +An index run lists the tree once with `git ls-tree -l`, which also supplies +blob sizes, and streams file contents through one `git cat-file --batch` +process. It opens the index for writing twice: once to discard leftover staged +rows, read the previous snapshot, and record the running state, and once to +persist, compare, and promote. No index connection is held while files are +parsed. Rows are written with multi-row upserts, and stale marking for changed +paths runs as one statement per table. + The gateway returns `accepted`, `progress`, `completed`, `unavailable`, or `failed` reports and journals progress/failure events plus `code_graph_updated` after completion. This shared target-branch baseline is not the live truth for diff --git a/docs/memory.md b/docs/memory.md index e8fd8f52..37f5266d 100644 --- a/docs/memory.md +++ b/docs/memory.md @@ -442,7 +442,12 @@ for the same style of kickoff bundle and writes it to server is disabled, the runner falls back to direct local memory reads. Read commands open the DuckDB index in read-only mode and do not run migrations. -Startup and write paths own schema creation or migration. +Startup and write paths own schema creation or migration. A completed migration +records its fingerprint in the `memory_migrations` table. The fingerprint +covers the migration SQL, the per-table column steps, the schema version, and +the OpenSymphony version. Later write opens skip the migration while the +recorded fingerprint matches, so a new release, a changed migration, or a fresh +or recreated store migrates once. When central configuration is active, one supervised memory server owns one instance-wide catalog. Repository inventory entries are registered in that