From 38bc131598d5d993295f97b7001811209a1b16a5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:10:12 -0800 Subject: [PATCH 01/42] fix(code-index): report skipped nested repositories in unmounted audit Keep parent repository source admission unchanged; use one shared walker for exclusion evidence instead of a second filesystem scan. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 64 +++++++++++++++- .../src/unmounted_files.rs | 76 +++++++++++++++++-- 2 files changed, 132 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 63d4be6e407..a02ecb2e20e 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -9,8 +9,9 @@ //! walk is therefore public rather than crate-private, the audit in the root //! crate reuses this policy instead of restating it. +use std::collections::BTreeSet; use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::{Arc, Mutex, PoisonError}; use ignore::overrides::{Override, OverrideBuilder}; use ignore::{Walk, WalkBuilder}; @@ -103,11 +104,51 @@ impl PathGlobScope { } } +/// Diagnostics emitted by the same source walker that enforces the nested +/// repository boundary. Only directories the walk actually reaches are named. +#[derive(Clone, Default)] +pub(crate) struct SourceWalkDiagnostics { + skipped_nested_repositories: Arc>>, +} + +impl SourceWalkDiagnostics { + pub(crate) fn skipped_nested_repositories(&self) -> Vec { + self.skipped_nested_repositories + .lock() + .unwrap_or_else(PoisonError::into_inner) + .iter() + .cloned() + .collect() + } +} + #[tracing::instrument(name = "code_index.capture.source_walk", level = "trace", skip_all)] pub fn source_walk( project_root: &Path, path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, +) -> Result { + source_walk_inner(project_root, path_glob, path_policy, None) +} + +/// Expose excluded nested repository roots without changing source admission. +/// The unmounted-file audit uses this to avoid falsely implying completeness. +/// It uses the same traversal rather than an independent repository walker. +pub(crate) fn source_walk_with_diagnostics( + project_root: &Path, + path_glob: Option<&str>, + path_policy: &IndexPathPolicyV1, +) -> Result<(Walk, SourceWalkDiagnostics), SourceWalkError> { + let diagnostics = SourceWalkDiagnostics::default(); + let walk = source_walk_inner(project_root, path_glob, path_policy, Some(diagnostics.clone()))?; + Ok((walk, diagnostics)) +} + +fn source_walk_inner( + project_root: &Path, + path_glob: Option<&str>, + path_policy: &IndexPathPolicyV1, + diagnostics: Option, ) -> Result { let overrides = build_overrides(project_root, path_glob)?; let has_positive_override = overrides @@ -171,6 +212,13 @@ pub fn source_walk( // sources. Linked worktrees and nested clones share this boundary. // The project root is deliberately exempt at depth zero above. if std::fs::symlink_metadata(entry.path().join(".git")).is_ok() { + if let Some(diagnostics) = &diagnostics { + diagnostics + .skipped_nested_repositories + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(relative); + } return false; } !path_policy.excludes_directory(&relative) @@ -220,7 +268,7 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::source_walk; + use super::{source_walk, source_walk_with_diagnostics}; fn walked_files(root: &std::path::Path, path_policy: &IndexPathPolicyV1) -> Vec { let mut files = source_walk(root, None, path_policy) @@ -435,6 +483,18 @@ mod tests { assert!(index.stdout.ends_with(b" 0\tapp\0")); assert!(root.join("app/src/feature.rs").is_file()); + let (walk, diagnostics) = source_walk_with_diagnostics( + &root, + None, + &IndexPathPolicyV1::new(vec![], vec![]).unwrap(), + ) + .expect("source walk with nested repository diagnostics"); + let _visited = walk.map(|entry| entry.expect("source entry")).count(); + assert_eq!( + diagnostics.skipped_nested_repositories(), + vec!["app".to_owned()] + ); + for rule in ["app/**", "app/src/feature.rs", "app/src/fe*.rs"] { let policy = IndexPathPolicyV1::new(vec![], vec![rule.into()]).unwrap(); let files = walked_files(&root, &policy); diff --git a/crates/tracedecay-code-index/src/unmounted_files.rs b/crates/tracedecay-code-index/src/unmounted_files.rs index 1d1cb19e497..3f140bdebf8 100644 --- a/crates/tracedecay-code-index/src/unmounted_files.rs +++ b/crates/tracedecay-code-index/src/unmounted_files.rs @@ -152,16 +152,19 @@ pub struct ProjectAudit { pub(super) struct ProjectFiles { root: PathBuf, files: Vec, + skipped_nested_repositories: Vec, } impl ProjectFiles { fn collect(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Result { - let walk = - crate::source_walk::source_walk(project_root, None, path_policy).map_err(|error| { - TraceDecayError::Config { - message: format!("source walk rejected its own scope: {}", error.message), - } - })?; + let (walk, diagnostics) = crate::source_walk::source_walk_with_diagnostics( + project_root, + None, + path_policy, + ) + .map_err(|error| TraceDecayError::Config { + message: format!("source walk rejected its own scope: {}", error.message), + })?; let mut files = Vec::new(); for entry in walk { let Ok(entry) = entry else { continue }; @@ -174,6 +177,7 @@ impl ProjectFiles { Ok(Self { root: project_root.to_path_buf(), files, + skipped_nested_repositories: diagnostics.skipped_nested_repositories(), }) } @@ -258,6 +262,32 @@ pub fn audit_project(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Re }; let mut ecosystems = vec![rust, typescript]; ecosystems.extend(unmodelled_ecosystems(&files)); + if !files.skipped_nested_repositories.is_empty() { + // Preserve existing ecosystem caveats and bound the diagnostic text. + // Skipped child repositories are not indexed as parent source. + let named = files + .skipped_nested_repositories + .iter() + .take(8) + .cloned() + .collect::>(); + let remaining = files.skipped_nested_repositories.len() - named.len(); + let summary = format!( + "Skipped nested repositories outside parent index: {}{}", + named.join(", "), + if remaining == 0 { + String::new() + } else { + format!(" (and {remaining} more)") + } + ); + for ecosystem in &mut ecosystems { + ecosystem.note = Some(match ecosystem.note.take() { + Some(existing) => format!("{existing} {summary}"), + None => summary.clone(), + }); + } + } Ok(ProjectAudit { ecosystems }) } @@ -395,6 +425,40 @@ mod tests { ); } + #[test] + fn audit_reports_nested_repository_exclusion_without_claiming_child_code() { + let root = tempfile::tempdir().expect("project fixture"); + write( + root.path(), + "Cargo.toml", + "[package]\nname = \"parent\"\nversion = \"0.0.0\"\n", + ); + write(root.path(), "src/lib.rs", "pub fn parent() {}\n"); + write(root.path(), "app/.git", "gitdir: ../.git/modules/app\n"); + write(root.path(), "app/src/child.rs", "pub fn child() {}\n"); + write(root.path(), "unrelated/.git/HEAD", "ref: refs/heads/main\n"); + write(root.path(), "unrelated/src/foreign.rs", "pub fn foreign() {}\n"); + + let audit = audit_project(root.path(), &no_exclusions()).expect("audit"); + let rust = ecosystem(&audit, "rust"); + assert_eq!( + rust.scanned_file_count, 1, + "child repositories are not parent source" + ); + assert!(rust.unmounted.iter().all(|file| { + !file.file.starts_with("app/") && !file.file.starts_with("unrelated/") + })); + let note = rust.note.as_deref().expect("explicit scope diagnostic"); + assert!(note.contains("Skipped nested repositories"), "{note}"); + assert!(note.contains("app") && note.contains("unrelated"), "{note}"); + assert!(audit.ecosystems.iter().all(|ecosystem| { + ecosystem + .note + .as_deref() + .is_some_and(|note| note.contains("Skipped nested repositories")) + })); + } + /// Both modelled ecosystems report in one pass over a polyglot repository. #[test] fn a_polyglot_project_reports_one_section_per_ecosystem() { From 1f8c1997090e4631ee38f50a32af4186e8153c12 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:14:21 -0800 Subject: [PATCH 02/42] style(code-index): format nested-repository diagnostics --- .../tracedecay-code-index/src/source_walk.rs | 7 ++++++- .../src/unmounted_files.rs | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index a02ecb2e20e..c0eebaeda39 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -140,7 +140,12 @@ pub(crate) fn source_walk_with_diagnostics( path_policy: &IndexPathPolicyV1, ) -> Result<(Walk, SourceWalkDiagnostics), SourceWalkError> { let diagnostics = SourceWalkDiagnostics::default(); - let walk = source_walk_inner(project_root, path_glob, path_policy, Some(diagnostics.clone()))?; + let walk = source_walk_inner( + project_root, + path_glob, + path_policy, + Some(diagnostics.clone()), + )?; Ok((walk, diagnostics)) } diff --git a/crates/tracedecay-code-index/src/unmounted_files.rs b/crates/tracedecay-code-index/src/unmounted_files.rs index 3f140bdebf8..5e5ee2e690d 100644 --- a/crates/tracedecay-code-index/src/unmounted_files.rs +++ b/crates/tracedecay-code-index/src/unmounted_files.rs @@ -157,14 +157,11 @@ pub(super) struct ProjectFiles { impl ProjectFiles { fn collect(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Result { - let (walk, diagnostics) = crate::source_walk::source_walk_with_diagnostics( - project_root, - None, - path_policy, - ) - .map_err(|error| TraceDecayError::Config { - message: format!("source walk rejected its own scope: {}", error.message), - })?; + let (walk, diagnostics) = + crate::source_walk::source_walk_with_diagnostics(project_root, None, path_policy) + .map_err(|error| TraceDecayError::Config { + message: format!("source walk rejected its own scope: {}", error.message), + })?; let mut files = Vec::new(); for entry in walk { let Ok(entry) = entry else { continue }; @@ -437,7 +434,11 @@ mod tests { write(root.path(), "app/.git", "gitdir: ../.git/modules/app\n"); write(root.path(), "app/src/child.rs", "pub fn child() {}\n"); write(root.path(), "unrelated/.git/HEAD", "ref: refs/heads/main\n"); - write(root.path(), "unrelated/src/foreign.rs", "pub fn foreign() {}\n"); + write( + root.path(), + "unrelated/src/foreign.rs", + "pub fn foreign() {}\n", + ); let audit = audit_project(root.path(), &no_exclusions()).expect("audit"); let rust = ecosystem(&audit, "rust"); From 8d6f07e1bb326630a313e1911d8bd271f337dd6c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:16:18 -0800 Subject: [PATCH 03/42] fix(code-index): bound repository audits and report walk failures Limit skipped-repository samples during traversal, honor directory policy before collecting diagnostics, and quote bounded path labels. Propagate source-walk failures instead of returning an empty successful audit. Add behavioral coverage for bounded sampling, policy scope, preserved ecosystem caveats, Unicode/control-character paths, and missing roots. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 85 +++++++-- .../source_walk/nested_repository_tests.rs | 169 ++++++++++++++++++ .../src/unmounted_files.rs | 29 +-- 3 files changed, 246 insertions(+), 37 deletions(-) create mode 100644 crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index c0eebaeda39..e360e4e6807 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -104,21 +104,72 @@ impl PathGlobScope { } } +const MAX_REPOSITORY_NAMES: usize = 8; +const MAX_REPOSITORY_PATH_CHARS: usize = 160; + /// Diagnostics emitted by the same source walker that enforces the nested -/// repository boundary. Only directories the walk actually reaches are named. +/// repository boundary. Only directories within the walk's policy are named. #[derive(Clone, Default)] pub(crate) struct SourceWalkDiagnostics { - skipped_nested_repositories: Arc>>, + skipped_nested_repositories: Arc>, +} + +#[derive(Default)] +struct NestedRepositorySummary { + total: usize, + names: BTreeSet, } impl SourceWalkDiagnostics { - pub(crate) fn skipped_nested_repositories(&self) -> Vec { - self.skipped_nested_repositories + fn record_skipped_nested_repository(&self, relative: String) { + let mut summary = self + .skipped_nested_repositories .lock() - .unwrap_or_else(PoisonError::into_inner) + .unwrap_or_else(PoisonError::into_inner); + // A pruned directory is visited once. Keep only the smallest names, + // independent of filesystem iteration order, rather than collecting + // every path and truncating only after the walk has finished. + summary.total += 1; + summary.names.insert(relative); + if summary.names.len() > MAX_REPOSITORY_NAMES { + let _ = summary.names.pop_last(); + } + } + + pub(crate) fn skipped_nested_repositories_note(&self) -> Option { + let summary = self + .skipped_nested_repositories + .lock() + .unwrap_or_else(PoisonError::into_inner); + if summary.total == 0 { + return None; + } + let names = summary + .names .iter() - .cloned() - .collect() + .map(|path| { + // Bound by characters before escaping, so truncation cannot + // split UTF-8 or an escape. Quoting prevents a filename from + // injecting new lines or delimiters into an agent's report. + let mut chars = path.chars(); + let prefix = chars + .by_ref() + .take(MAX_REPOSITORY_PATH_CHARS) + .collect::(); + let suffix = if chars.next().is_some() { "…" } else { "" }; + format!("{prefix:?}{suffix}") + }) + .collect::>(); + let remaining = summary.total - summary.names.len(); + let suffix = if remaining == 0 { + String::new() + } else { + format!(" (and {remaining} more)") + }; + Some(format!( + "Skipped nested repositories outside parent index: {}{suffix}", + names.join(", ") + )) } } @@ -211,6 +262,11 @@ fn source_walk_inner( if !is_dir { return !path_policy.excludes(&relative); } + // Excluded directories are outside this audit, not missing source + // caused by the repository boundary. Do not probe or name them. + if path_policy.excludes_directory(&relative) { + return false; + } // Includes select paths, not repository authority. A parent // gitlink does not capture the child's files or prove their // freshness, so even registered submodules remain separate @@ -218,15 +274,11 @@ fn source_walk_inner( // The project root is deliberately exempt at depth zero above. if std::fs::symlink_metadata(entry.path().join(".git")).is_ok() { if let Some(diagnostics) = &diagnostics { - diagnostics - .skipped_nested_repositories - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(relative); + diagnostics.record_skipped_nested_repository(relative); } return false; } - !path_policy.excludes_directory(&relative) + true }); if let Some(overrides) = overrides { builder.overrides(overrides); @@ -265,6 +317,9 @@ fn build_overrides( } } +#[cfg(test)] +mod nested_repository_tests; + #[cfg(test)] mod tests { use std::fs; @@ -496,8 +551,8 @@ mod tests { .expect("source walk with nested repository diagnostics"); let _visited = walk.map(|entry| entry.expect("source entry")).count(); assert_eq!( - diagnostics.skipped_nested_repositories(), - vec!["app".to_owned()] + diagnostics.skipped_nested_repositories_note().as_deref(), + Some("Skipped nested repositories outside parent index: \"app\"") ); for rule in ["app/**", "app/src/feature.rs", "app/src/fe*.rs"] { diff --git a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs new file mode 100644 index 00000000000..5242c8305fa --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs @@ -0,0 +1,169 @@ +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use tracedecay_domain::IndexPathPolicyV1; +use tracedecay_domain::errors::TraceDecayError; + +use super::{SourceWalkDiagnostics, source_walk_with_diagnostics}; +use crate::unmounted_files::audit_project; + +fn write(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + std::fs::create_dir_all(path.parent().expect("fixture parent")).unwrap(); + std::fs::write(path, contents).unwrap(); +} + +#[test] +fn repository_diagnostic_sample_is_bounded_and_order_independent() { + let expected = (0..8) + .map(|index| format!("repo-{index:02}")) + .collect::>(); + let orders = [ + (0..24).collect::>(), + (0..24).rev().collect::>(), + ]; + for order in orders { + let diagnostics = SourceWalkDiagnostics::default(); + assert!(diagnostics.skipped_nested_repositories_note().is_none()); + for index in order { + diagnostics.record_skipped_nested_repository(format!("repo-{index:02}")); + let summary = diagnostics.skipped_nested_repositories.lock().unwrap(); + assert!(summary.names.len() <= 8, "collection must be bounded too"); + } + let summary = diagnostics.skipped_nested_repositories.lock().unwrap(); + assert_eq!(summary.total, 24); + assert_eq!(summary.names, expected); + drop(summary); + let note = diagnostics.skipped_nested_repositories_note().unwrap(); + assert_eq!( + note, + "Skipped nested repositories outside parent index: \ + \"repo-00\", \"repo-01\", \"repo-02\", \"repo-03\", \ + \"repo-04\", \"repo-05\", \"repo-06\", \"repo-07\" (and 16 more)" + ); + } +} + +#[test] +fn repository_diagnostics_follow_policy_without_crossing_boundaries() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "keep.rs", "pub fn parent() {}\n"); + for directory in ["app", "excluded", "node_modules"] { + write(root.path(), &format!("{directory}/.git"), "gitdir: foreign\n"); + write(root.path(), &format!("{directory}/child.rs"), "child source\n"); + } + for (include, expected_note) in [ + ( + vec![], + "Skipped nested repositories outside parent index: \"app\"", + ), + ( + vec!["excluded/child.rs".to_owned()], + "Skipped nested repositories outside parent index: \"app\", \"excluded\"", + ), + ] { + let policy = IndexPathPolicyV1::new( + vec!["excluded/**".to_owned(), "node_modules/**".to_owned()], + include, + ) + .unwrap(); + let (walk, diagnostics) = source_walk_with_diagnostics(root.path(), None, &policy) + .expect("source walk"); + let files = walk + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .map(|entry| entry.path().strip_prefix(root.path()).unwrap().to_path_buf()) + .collect::>(); + assert_eq!(files, vec![PathBuf::from("keep.rs")]); + assert_eq!( + diagnostics.skipped_nested_repositories_note().as_deref(), + Some(expected_note) + ); + } +} + +#[test] +fn audit_bounds_repository_summary_and_preserves_unsupported_caveat() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "main.go", "package main\n"); + for index in (0..12).rev() { + write( + root.path(), + &format!("repo-{index:02}/.git/HEAD"), + "ref: refs/heads/main\n", + ); + write( + root.path(), + &format!("repo-{index:02}/child.go"), + "package child\n", + ); + } + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + let audit = audit_project(root.path(), &policy).expect("audit"); + let go = audit + .ecosystems + .iter() + .find(|ecosystem| ecosystem.ecosystem == "go") + .expect("parent Go source was counted"); + assert_eq!(go.scanned_file_count, 1); + assert_eq!(go.status.as_str(), "unsupported"); + assert_eq!( + go.note.as_deref(), + Some( + "1 go source file(s) are present and were not audited, \ + this report cannot say whether any of them is unreachable \ + Skipped nested repositories outside parent index: \ + \"repo-00\", \"repo-01\", \"repo-02\", \"repo-03\", \ + \"repo-04\", \"repo-05\", \"repo-06\", \"repo-07\" (and 4 more)" + ) + ); +} + +#[test] +fn repository_diagnostic_paths_are_clipped_on_character_boundaries() { + let diagnostics = SourceWalkDiagnostics::default(); + diagnostics.record_skipped_nested_repository("é".repeat(161)); + assert_eq!( + diagnostics.skipped_nested_repositories_note().unwrap(), + format!( + "Skipped nested repositories outside parent index: \"{}\"…", + "é".repeat(160) + ) + ); +} + +#[cfg(unix)] +#[test] +fn audit_quotes_control_characters_in_repository_names() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "keep.py", "print('parent')\n"); + write(root.path(), "line\nbreak/.git", "gitdir: foreign\n"); + write(root.path(), "line\nbreak/child.py", "print('child')\n"); + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + let audit = audit_project(root.path(), &policy).expect("audit"); + let python = audit + .ecosystems + .iter() + .find(|ecosystem| ecosystem.ecosystem == "python") + .expect("parent Python source was counted"); + assert_eq!(python.scanned_file_count, 1); + let note = python.note.as_deref().expect("scope diagnostic"); + assert!(note.contains("\"line\\nbreak\""), "{note}"); + assert!(!note.contains('\n'), "a path must not inject report lines"); + assert!(root.path().join("line\nbreak/child.py").is_file()); +} + +#[test] +fn missing_source_root_is_an_error_not_a_clean_audit() { + let root = tempfile::tempdir().unwrap(); + let missing = root.path().join("missing"); + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + match audit_project(&missing, &policy) { + Err(TraceDecayError::File { path, message }) => { + assert_eq!(path, missing.display().to_string()); + assert!(message.contains("source walk did not complete")); + } + Err(error) => panic!("unexpected error: {error}"), + Ok(_) => panic!("a failed walk must not report an empty successful audit"), + } +} diff --git a/crates/tracedecay-code-index/src/unmounted_files.rs b/crates/tracedecay-code-index/src/unmounted_files.rs index 5e5ee2e690d..31cf2d6f594 100644 --- a/crates/tracedecay-code-index/src/unmounted_files.rs +++ b/crates/tracedecay-code-index/src/unmounted_files.rs @@ -152,7 +152,7 @@ pub struct ProjectAudit { pub(super) struct ProjectFiles { root: PathBuf, files: Vec, - skipped_nested_repositories: Vec, + nested_repository_note: Option, } impl ProjectFiles { @@ -164,7 +164,10 @@ impl ProjectFiles { })?; let mut files = Vec::new(); for entry in walk { - let Ok(entry) = entry else { continue }; + let entry = entry.map_err(|error| TraceDecayError::File { + message: format!("source walk did not complete: {error}"), + path: project_root.display().to_string(), + })?; if !entry.file_type().is_some_and(|kind| kind.is_file()) { continue; } @@ -174,7 +177,7 @@ impl ProjectFiles { Ok(Self { root: project_root.to_path_buf(), files, - skipped_nested_repositories: diagnostics.skipped_nested_repositories(), + nested_repository_note: diagnostics.skipped_nested_repositories_note(), }) } @@ -259,25 +262,7 @@ pub fn audit_project(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Re }; let mut ecosystems = vec![rust, typescript]; ecosystems.extend(unmodelled_ecosystems(&files)); - if !files.skipped_nested_repositories.is_empty() { - // Preserve existing ecosystem caveats and bound the diagnostic text. - // Skipped child repositories are not indexed as parent source. - let named = files - .skipped_nested_repositories - .iter() - .take(8) - .cloned() - .collect::>(); - let remaining = files.skipped_nested_repositories.len() - named.len(); - let summary = format!( - "Skipped nested repositories outside parent index: {}{}", - named.join(", "), - if remaining == 0 { - String::new() - } else { - format!(" (and {remaining} more)") - } - ); + if let Some(summary) = &files.nested_repository_note { for ecosystem in &mut ecosystems { ecosystem.note = Some(match ecosystem.note.take() { Some(existing) => format!("{existing} {summary}"), From 1b49877815c81451007e66a4f2d54e4a92a06f6d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:39:59 -0800 Subject: [PATCH 04/42] style(code-index): format nested repository regressions Apply the exact formatter changes reported by the hosted Bazel rustfmt_check_2024 failure on PR #4801. Refs #4593 --- .../source_walk/nested_repository_tests.rs | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs index 5242c8305fa..0b77991f58e 100644 --- a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs @@ -49,8 +49,16 @@ fn repository_diagnostics_follow_policy_without_crossing_boundaries() { let root = tempfile::tempdir().unwrap(); write(root.path(), "keep.rs", "pub fn parent() {}\n"); for directory in ["app", "excluded", "node_modules"] { - write(root.path(), &format!("{directory}/.git"), "gitdir: foreign\n"); - write(root.path(), &format!("{directory}/child.rs"), "child source\n"); + write( + root.path(), + &format!("{directory}/.git"), + "gitdir: foreign\n", + ); + write( + root.path(), + &format!("{directory}/child.rs"), + "child source\n", + ); } for (include, expected_note) in [ ( @@ -67,12 +75,18 @@ fn repository_diagnostics_follow_policy_without_crossing_boundaries() { include, ) .unwrap(); - let (walk, diagnostics) = source_walk_with_diagnostics(root.path(), None, &policy) - .expect("source walk"); + let (walk, diagnostics) = + source_walk_with_diagnostics(root.path(), None, &policy).expect("source walk"); let files = walk .map(|entry| entry.expect("source entry")) .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) - .map(|entry| entry.path().strip_prefix(root.path()).unwrap().to_path_buf()) + .map(|entry| { + entry + .path() + .strip_prefix(root.path()) + .unwrap() + .to_path_buf() + }) .collect::>(); assert_eq!(files, vec![PathBuf::from("keep.rs")]); assert_eq!( From 624cc8338a1cfe8486b0638ba88fbbae0df9fa33 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:05:56 -0800 Subject: [PATCH 05/42] fix(code-index): align nested checkout capture with source walks Use one source boundary for interactive walks, Git candidate classification, worktree capture (including ignored-file admissions), and freshness. Parent tracked files hidden by a new nested .git marker become tombstones, invalidate clean HEAD-tree reuse, and return when the marker disappears. Reject source paths passing through symlinked directories or lexical escapes. Add real Git classification and filesystem regressions for nested directory, gitdir-file, symlink, path traversal and removal behavior. Refs #4593 --- .../code_index_scheduler/classification.rs | 134 +++++++++++++++++- .../code_index_scheduler/freshness_witness.rs | 14 +- .../src/code_index_scheduler/reconcile.rs | 8 ++ .../tracedecay-code-index/src/source_walk.rs | 107 +++++++++++++- 4 files changed, 251 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index 8f057e94a28..ab168be9965 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -14,10 +14,11 @@ //! consumes the derived candidate and changed sets to build an incremental //! batch. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use gix::bstr::ByteSlice; +use tracedecay_code_index::source_walk::has_nested_repository_ancestor; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; /// Failure to classify worktree changes through gix. @@ -25,6 +26,8 @@ use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREA pub enum ClassificationErrorV1 { #[error("code-index classification: {0}")] Git(String), + #[error("code-index source boundary: {0}")] + SourceBoundary(#[from] std::io::Error), } /// The truthful disposition of one repository-relative path. @@ -91,6 +94,9 @@ pub struct WorktreeChangeClassificationV1 { /// Present paths whose Git bytes are not UTF-8, so no `String` above /// names them. non_utf8_paths: BTreeSet>, + /// Files still tracked by the parent but hidden behind a nested checkout + /// or symlinked ancestor. These are tombstones, not indexable candidates. + nested_repository_exclusions: BTreeSet, } impl WorktreeChangeClassificationV1 { @@ -202,11 +208,40 @@ impl WorktreeChangeClassificationV1 { } } + // A nested .git marker can appear under paths that are still in + // the parent's index without changing Git status. The live walker + // refuses those files; classification must also exclude them and + // invalidate clean-tree and active-generation reuse. Resolve each + // parent directory once rather than probing per source file. + let mut nested_repository_exclusions = BTreeSet::new(); + if let Some(project_root) = repository.workdir() { + let mut checked_parents = BTreeMap::new(); + for path in committed_baseline + .iter() + .chain(changes.iter().map(|change| &change.path)) + { + let relative = Path::new(path); + let parent = relative.parent().unwrap_or_else(|| Path::new("")); + let excluded = match checked_parents.get(parent) { + Some(excluded) => *excluded, + None => { + let excluded = has_nested_repository_ancestor(project_root, relative)?; + checked_parents.insert(parent.to_path_buf(), excluded); + excluded + } + }; + if excluded { + nested_repository_exclusions.insert(path.clone()); + } + } + } + Ok(Self { committed_baseline, changes, non_utf8_changes, non_utf8_paths, + nested_repository_exclusions, }) } @@ -241,13 +276,16 @@ impl WorktreeChangeClassificationV1 { candidates.remove(&change.path); } } + candidates.retain(|path| !self.nested_repository_exclusions.contains(path)); candidates } /// Whether the worktree carries any change relative to the committed /// baseline, UTF-8-named or not. pub fn has_changes(&self) -> bool { - !(self.changes().is_empty() && self.non_utf8_changes().is_empty()) + !(self.changes().is_empty() + && self.non_utf8_changes().is_empty() + && self.nested_repository_exclusions.is_empty()) } /// Whether any classified change is a merge conflict. @@ -264,10 +302,13 @@ impl WorktreeChangeClassificationV1 { /// tombstones flow through). This is a hint to narrow work, not the identity /// authority; the generation planner still compares content digests. pub fn changed_paths(&self) -> BTreeSet { - self.changes + let mut changed = self + .changes .iter() .map(|change| change.path.clone()) - .collect() + .collect::>(); + changed.extend(self.nested_repository_exclusions.iter().cloned()); + changed } /// Paths removed from the present snapshot (staged or unstaged deletions). @@ -345,3 +386,88 @@ fn classify_item(item: &gix::status::Item) -> Option { }, }) } + +#[cfg(test)] +mod nested_repository_tests { + use std::fs; + use std::path::Path; + use std::process::Command; + + use super::WorktreeChangeClassificationV1; + + fn fixture_git(root: &Path, args: &[&str]) { + let program = + tracedecay_runtime_core::git::try_git_program().expect("fixture Git program"); + let mut command = Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("fixture Git command"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn nested_checkout_marker_tombstones_parent_tracked_sources() { + let root = tempfile::tempdir().unwrap(); + fixture_git(root.path(), &["init", "-q"]); + for path in ["src/keep.rs", "nested/foreign.rs", "linked/foreign.rs"] { + let full = root.path().join(path); + fs::create_dir_all(full.parent().unwrap()).unwrap(); + fs::write(full, "pub fn source() {}\n").unwrap(); + } + fixture_git(root.path(), &["add", "."]); + fixture_git(root.path(), &["commit", "-qm", "parent tracked files"]); + + let repository = tracedecay_runtime_core::git_open::open(root.path()).unwrap(); + let before = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!(!before.has_changes()); + assert!(before.candidate_paths().contains("nested/foreign.rs")); + + fs::create_dir(root.path().join("nested/.git")).unwrap(); + fs::write( + root.path().join("nested/.git/HEAD"), + "ref: refs/heads/main\n", + ) + .unwrap(); + fs::write( + root.path().join("linked/.git"), + "gitdir: /untrusted/worktrees/linked\n", + ) + .unwrap(); + let hidden = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + let candidates = hidden.candidate_paths(); + assert!(candidates.contains("src/keep.rs")); + assert!(!candidates.contains("nested/foreign.rs")); + assert!(!candidates.contains("linked/foreign.rs")); + assert!(hidden.has_changes(), "a clean HEAD-tree capture is now unsafe"); + assert!(hidden.changed_paths().contains("nested/foreign.rs")); + assert!(hidden.changed_paths().contains("linked/foreign.rs")); + + fs::remove_dir_all(root.path().join("nested/.git")).unwrap(); + fs::remove_file(root.path().join("linked/.git")).unwrap(); + let restored = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!(!restored.has_changes()); + assert!(restored.candidate_paths().contains("nested/foreign.rs")); + assert!(restored.candidate_paths().contains("linked/foreign.rs")); + } +} diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 8392b549551..48639d5bebf 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -129,7 +129,19 @@ fn source_candidates( .map(|admission| admission.logical_path.as_str()) .collect::>(); let mut candidate_paths = classification.candidate_paths(); - candidate_paths.extend(admitted_paths.iter().map(|path| (*path).to_owned())); + let project_root = repository.workdir().ok_or_else(|| { + CodeIndexSchedulerErrorV1::Git("source freshness requires a worktree".to_owned()) + })?; + for path in &admitted_paths { + // Explicit ignored-file enrollment lifts Git ignore rules, not the + // nested repository boundary shared with capture and live search. + if !tracedecay_code_index::source_walk::has_nested_repository_ancestor( + project_root, + Path::new(path), + )? { + candidate_paths.insert((*path).to_owned()); + } + } let mut unrepresentable = classification .non_utf8_paths() .iter() diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 9ca91320c7e..e2d51bfeb61 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -3974,6 +3974,14 @@ impl CodeIndexWorktreeSchedulerV1 { control: Option<&dyn CodeIndexExecutionControlV1>, explicitly_admitted: bool, ) -> Result { + // Explicit ignored-source admission cannot cross a nested checkout + // any more than an ordinary Git candidate or interactive source walk. + if tracedecay_code_index::source_walk::has_nested_repository_ancestor( + &self.project_root, + std::path::Path::new(logical_path), + )? { + return Ok(CapturedFileOutcomeV1::Absent); + } let absolute = self.project_root.join(logical_path); match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index e360e4e6807..6a363aed0d2 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -10,7 +10,7 @@ //! crate reuses this policy instead of restating it. use std::collections::BTreeSet; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; use ignore::overrides::{Override, OverrideBuilder}; @@ -107,6 +107,52 @@ impl PathGlobScope { const MAX_REPOSITORY_NAMES: usize = 8; const MAX_REPOSITORY_PATH_CHARS: usize = 160; +fn nested_repository_marker(directory: &Path) -> std::io::Result { + match std::fs::symlink_metadata(directory.join(".git")) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +/// Reject a source beneath another checkout or a symlinked directory. +/// The root's own Git metadata is exempt. This boundary is shared by +/// interactive source walks and the indexed worktree candidate roster, +/// so neither may admit bytes the other refuses. +pub fn has_nested_repository_ancestor( + project_root: &Path, + relative_path: &Path, +) -> std::io::Result { + if relative_path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "source path must be repository-relative", + )); + } + + let mut parent = relative_path.parent(); + while let Some(relative_dir) = parent { + if relative_dir.as_os_str().is_empty() { + break; + } + let directory = project_root.join(relative_dir); + match std::fs::symlink_metadata(&directory) { + Ok(metadata) if !metadata.is_dir() => return Ok(true), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + if nested_repository_marker(&directory)? { + return Ok(true); + } + parent = relative_dir.parent(); + } + Ok(false) +} + /// Diagnostics emitted by the same source walker that enforces the nested /// repository boundary. Only directories within the walk's policy are named. #[derive(Clone, Default)] @@ -272,13 +318,18 @@ fn source_walk_inner( // freshness, so even registered submodules remain separate // sources. Linked worktrees and nested clones share this boundary. // The project root is deliberately exempt at depth zero above. - if std::fs::symlink_metadata(entry.path().join(".git")).is_ok() { - if let Some(diagnostics) = &diagnostics { - diagnostics.record_skipped_nested_repository(relative); + match nested_repository_marker(entry.path()) { + Ok(true) => { + if let Some(diagnostics) = &diagnostics { + diagnostics.record_skipped_nested_repository(relative); + } + false } - return false; + Ok(false) => true, + // The walk cannot prove that an unreadable Git marker is + // absent. Never admit files across that boundary. + Err(_) => false, } - true }); if let Some(overrides) = overrides { builder.overrides(overrides); @@ -328,7 +379,49 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::{source_walk, source_walk_with_diagnostics}; + use super::{ + has_nested_repository_ancestor, source_walk, source_walk_with_diagnostics, + }; + + #[test] + fn nested_repository_boundary_excludes_parent_markers_and_path_escape() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("parent/nested/.git")).unwrap(); + fs::write(root.path().join("parent/nested/.git/HEAD"), "ref: refs/heads/main\n").unwrap(); + assert!( + !has_nested_repository_ancestor(root.path(), std::path::Path::new("parent/source.rs")) + .unwrap() + ); + assert!( + has_nested_repository_ancestor( + root.path(), + std::path::Path::new("parent/nested/source.rs") + ) + .unwrap() + ); + assert!( + has_nested_repository_ancestor(root.path(), std::path::Path::new("../escape.rs")) + .is_err() + ); + assert!( + has_nested_repository_ancestor(root.path(), std::path::Path::new("/absolute.rs")) + .is_err() + ); + } + + #[cfg(unix)] + #[test] + fn nested_repository_boundary_excludes_symlinked_ancestors() { + use std::os::unix::fs::symlink; + + let root = TempDir::new().unwrap(); + let external = TempDir::new().unwrap(); + symlink(external.path(), root.path().join("foreign")).unwrap(); + assert!( + has_nested_repository_ancestor(root.path(), std::path::Path::new("foreign/source.rs")) + .unwrap() + ); + } fn walked_files(root: &std::path::Path, path_policy: &IndexPathPolicyV1) -> Vec { let mut files = source_walk(root, None, path_policy) From 9beb7b89f6ae6ca0c03b0713e561d5d792745334 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 22:39:23 -0800 Subject: [PATCH 06/42] style(code-index): format nested repository boundary tests Apply four exact replacements from hosted Bazel rustfmt_check_2024 at PR #4801 head 624cc833; no behavioral source change. Refs #4593 --- .../src/code_index_scheduler/classification.rs | 8 +++++--- crates/tracedecay-code-index/src/source_walk.rs | 10 ++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index ab168be9965..f8ef32be303 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -396,8 +396,7 @@ mod nested_repository_tests { use super::WorktreeChangeClassificationV1; fn fixture_git(root: &Path, args: &[&str]) { - let program = - tracedecay_runtime_core::git::try_git_program().expect("fixture Git program"); + let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git program"); let mut command = Command::new(program); command.current_dir(root); for (key, _) in std::env::vars_os() { @@ -459,7 +458,10 @@ mod nested_repository_tests { assert!(candidates.contains("src/keep.rs")); assert!(!candidates.contains("nested/foreign.rs")); assert!(!candidates.contains("linked/foreign.rs")); - assert!(hidden.has_changes(), "a clean HEAD-tree capture is now unsafe"); + assert!( + hidden.has_changes(), + "a clean HEAD-tree capture is now unsafe" + ); assert!(hidden.changed_paths().contains("nested/foreign.rs")); assert!(hidden.changed_paths().contains("linked/foreign.rs")); diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 6a363aed0d2..515a679fce2 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -379,15 +379,17 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::{ - has_nested_repository_ancestor, source_walk, source_walk_with_diagnostics, - }; + use super::{has_nested_repository_ancestor, source_walk, source_walk_with_diagnostics}; #[test] fn nested_repository_boundary_excludes_parent_markers_and_path_escape() { let root = TempDir::new().unwrap(); fs::create_dir_all(root.path().join("parent/nested/.git")).unwrap(); - fs::write(root.path().join("parent/nested/.git/HEAD"), "ref: refs/heads/main\n").unwrap(); + fs::write( + root.path().join("parent/nested/.git/HEAD"), + "ref: refs/heads/main\n", + ) + .unwrap(); assert!( !has_nested_repository_ancestor(root.path(), std::path::Path::new("parent/source.rs")) .unwrap() From 2386440dfa50f47393d1f380f0d4f43c707e95c9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:43:28 -0800 Subject: [PATCH 07/42] fix(code-index): invalidate cached rosters on nested Git markers Sample .git marker evidence for tracked and explicitly admitted source ancestors even when Git dirwalk never recurses into tracked-only folders. Test marker creation and removal against a real Git parent index. Refs #4593 --- .../code_index_scheduler/freshness_witness.rs | 132 +++++++++++++++++- 1 file changed, 131 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 48639d5bebf..1e5e03e8601 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -490,6 +490,7 @@ fn roster_evidence( repository: &gix::Repository, project_root: &Path, sampled_at: ChangeClockReading, + ignored_source_admissions: &[CodeIndexIgnoredSourceAdmissionV1], ) -> Option)>> { let mut recorder = DirectoryEvidenceV1 { root: project_root.to_path_buf(), @@ -513,6 +514,37 @@ fn roster_evidence( } recorder.record_directory(project_root.to_path_buf()); let index = repository.index_or_empty().ok()?; + // Git's directory walk may not descend into tracked-only directories. + // Their nested .git markers still change source admission without a + // parent index or HEAD update, so key each tracked and explicitly + // admitted source ancestor independently of dirwalk traversal. + let mut source_ancestors = BTreeSet::new(); + let mut collect_ancestors = |relative: &Path| { + let mut current = relative.parent(); + while let Some(parent) = current { + if parent.as_os_str().is_empty() { + break; + } + source_ancestors.insert(parent.to_path_buf()); + current = parent.parent(); + } + }; + for entry in index.entries() { + collect_ancestors(gix::path::from_bstr(entry.path(&index)).ok()?); + } + for admission in ignored_source_admissions { + if validate_code_logical_path(&admission.logical_path).is_err() { + return None; + } + collect_ancestors(Path::new(&admission.logical_path)); + } + for relative in source_ancestors { + let directory = project_root.join(relative); + recorder.record_directory(directory.clone()); + // A missing marker is evidence too: a newly created gitdir file or + // directory must invalidate the roster even if Git ignores it. + recorder.record(directory.join(".git"), false); + } let options = repository .dirwalk_options() .ok()? @@ -597,7 +629,12 @@ impl SourceSweepCacheV1 { let Ok(repository) = tracedecay_runtime_core::git_open::open(project_root) else { return (false, stats); }; - let evidence = roster_evidence(&repository, project_root, sampled_at); + let evidence = roster_evidence( + &repository, + project_root, + sampled_at, + ignored_source_admissions, + ); let Ok(SourceCandidatesV1 { candidates, unrepresentable, @@ -1029,3 +1066,96 @@ mod tests { assert_eq!(first, sample(&path, sampled_at).unwrap().unwrap()); } } + +#[cfg(all(test, unix))] +mod tracked_parent_boundary_tests { + use std::path::Path; + use std::sync::Arc; + use std::time::{Duration, Instant}; + + use super::{CachedCandidateRosterV1, ChangeClockReading, roster_evidence}; + + fn git(root: &Path, args: &[&str]) { + let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git"); + let mut command = std::process::Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run fixture Git"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn cached_tracked_directory_roster_invalidates_on_nested_git_marker() { + let root = tempfile::tempdir().expect("parent checkout"); + let tracked = root.path().join("src/nested/tracked.rs"); + std::fs::create_dir_all(tracked.parent().unwrap()).unwrap(); + std::fs::write(&tracked, "pub fn tracked() {}\n").unwrap(); + git(root.path(), &["init", "-q"]); + git(root.path(), &["add", "src/nested/tracked.rs"]); + git(root.path(), &["commit", "-qm", "parent source"]); + + let repository = + tracedecay_runtime_core::git_open::open(root.path()).expect("open parent Git"); + let deadline = Instant::now() + Duration::from_secs(5); + let evidence = loop { + if let Some(evidence) = + roster_evidence(&repository, root.path(), ChangeClockReading::now(), &[]) + { + break evidence; + } + assert!( + Instant::now() < deadline, + "tracked parent directories never became settleable" + ); + std::thread::sleep(Duration::from_millis(20)); + }; + let marker = root.path().join("src/nested/.git"); + assert!( + evidence + .iter() + .any(|(path, key)| path == &marker && key.is_none()), + "missing nested marker must be sampled independently of gix dirwalk" + ); + + let cached = CachedCandidateRosterV1 { + git_metadata_signature: "unchanged-parent-git".to_owned(), + admitted_paths: Vec::new(), + evidence, + candidates: Arc::new(Vec::new()), + unrepresentable: Arc::new(Vec::new()), + }; + assert!(cached.holds("unchanged-parent-git", &[])); + std::fs::write(&marker, "gitdir: ../../.git/modules/nested\n") + .expect("create nested gitdir marker"); + assert!( + !cached.holds("unchanged-parent-git", &[]), + "a new nested repository marker must invalidate the cached roster even when parent Git metadata is unchanged" + ); + std::fs::remove_file(&marker).unwrap(); + assert!( + cached.holds("unchanged-parent-git", &[]), + "the pre-marker evidence should be valid again after removing it" + ); + } +} From 0119cf9ff865c41a92c4a9ecb1cd09f15ea0322d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:44:27 -0800 Subject: [PATCH 08/42] test(code-index): assert roster invalidation across marker removal Directory metadata changes when the marker is removed, so the original cached roster must not be treated as restored merely by marker absence. Refs #4593 --- .../src/code_index_scheduler/freshness_witness.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 1e5e03e8601..8eb1ba4fe0d 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -1154,8 +1154,8 @@ mod tracked_parent_boundary_tests { ); std::fs::remove_file(&marker).unwrap(); assert!( - cached.holds("unchanged-parent-git", &[]), - "the pre-marker evidence should be valid again after removing it" + !cached.holds("unchanged-parent-git", &[]), + "removing the marker also changes the tracked ancestor directory evidence" ); } } From 704a9aac135046fe1d720242a482561a61eb5274 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:16:52 -0800 Subject: [PATCH 09/42] fix(code-index): refuse markerless parent gitlink directories A registered gitlink remains an untrusted nested source boundary even if its child .git marker is absent. Keep live grep and audit in default-deny parity with the parent index; include/glob cannot lift repository trust. Exercise a real registered Git submodule with its marker removed. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 515a679fce2..9e3f4e973b1 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -115,6 +115,52 @@ fn nested_repository_marker(directory: &Path) -> std::io::Result { } } +/// Default-deny all gitlinks in the parent's Git index even when a child +/// checkout's .git marker has been removed. A gitlink is a repository boundary, +/// not an ordinary directory the live source walker can authenticate. +/// This is deny-only; it never enables submodule admission. +fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { + let marker = project_root.join(".git"); + let repository = match tracedecay_runtime_core::git_open::open(project_root) { + Ok(repository) => repository, + Err(error) => { + // Non-Git projects still support source walks. An initialized + // checkout whose Git authority is unreadable must fail closed. + if marker.is_file() || marker.join("HEAD").is_file() { + return Err(SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git boundaries: {error}"), + }); + } + return Ok(BTreeSet::new()); + } + }; + if repository.workdir().and_then(|workdir| workdir.canonicalize().ok()) + != project_root.canonicalize().ok() + { + return Err(SourceWalkError { + glob: String::new(), + message: "source walk root is not the opened Git worktree".to_owned(), + }); + } + let index = repository.index_or_empty().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git index: {error}"), + })?; + let mut paths = BTreeSet::new(); + for entry in index.entries() { + if entry.mode.is_commit() { + let path: &[u8] = entry.path(&index).as_ref(); + let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { + glob: String::new(), + message: "non-UTF-8 parent gitlink cannot be safely scoped".to_owned(), + })?; + paths.insert(relative.to_owned()); + } + } + Ok(paths) +} + /// Reject a source beneath another checkout or a symlinked directory. /// The root's own Git metadata is exempt. This boundary is shared by /// interactive source walks and the indexed worktree candidate roster, @@ -260,6 +306,7 @@ fn source_walk_inner( let path_glob_scope = path_glob.and_then(PathGlobScope::from_path_glob); let filter_root = project_root.to_path_buf(); let path_policy = path_policy.clone(); + let parent_gitlinks = parent_gitlink_paths(project_root)?; let mut builder = WalkBuilder::new(project_root); builder @@ -318,6 +365,12 @@ fn source_walk_inner( // freshness, so even registered submodules remain separate // sources. Linked worktrees and nested clones share this boundary. // The project root is deliberately exempt at depth zero above. + if parent_gitlinks.contains(&relative) { + if let Some(diagnostics) = &diagnostics { + diagnostics.record_skipped_nested_repository(relative); + } + return false; + } match nested_repository_marker(entry.path()) { Ok(true) => { if let Some(diagnostics) = &diagnostics { @@ -668,6 +721,33 @@ mod tests { "{rule}: an explicit scope cannot cross repositories" ); } + + // A missing child .git marker must not turn the parent's stage-0 + // gitlink into ordinary live source under an explicit include. + fs::remove_file(root.join("app/.git")).expect("remove child gitdir marker"); + assert!(root.join("app/src/feature.rs").is_file()); + let policy = IndexPathPolicyV1::new(vec![], vec!["app/**".into()]).unwrap(); + let (walk, diagnostics) = + source_walk_with_diagnostics(&root, None, &policy).expect("source walk"); + let files = walk + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .map(|entry| entry.into_path()) + .collect::>(); + assert!( + files.iter().all(|path| !path.starts_with(root.join("app"))), + "a registered gitlink stays excluded without a child marker: {files:?}" + ); + assert_eq!( + diagnostics.skipped_nested_repositories_note().as_deref(), + Some("Skipped nested repositories outside parent index: \"app\"") + ); + let explicitly_scoped = source_walk(&root, Some("app/**"), &policy) + .expect("explicit gitlink scope") + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .count(); + assert_eq!(explicitly_scoped, 0); } #[test] From 1899d261c9c6486f572a0cbdf619d07315b65378 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:50:36 -0800 Subject: [PATCH 10/42] style(code-index): satisfy Bazel formatting for gitlink guard Apply the two exact rustfmt_check_2024 hunks from hosted run 42466514013. No source behavior changes; authenticated submodule indexing remains pending. Refs #4593 --- crates/tracedecay-code-index/src/source_walk.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 9e3f4e973b1..86056966076 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -135,7 +135,9 @@ fn parent_gitlink_paths(project_root: &Path) -> Result, SourceW return Ok(BTreeSet::new()); } }; - if repository.workdir().and_then(|workdir| workdir.canonicalize().ok()) + if repository + .workdir() + .and_then(|workdir| workdir.canonicalize().ok()) != project_root.canonicalize().ok() { return Err(SourceWalkError { @@ -143,10 +145,12 @@ fn parent_gitlink_paths(project_root: &Path) -> Result, SourceW message: "source walk root is not the opened Git worktree".to_owned(), }); } - let index = repository.index_or_empty().map_err(|error| SourceWalkError { - glob: String::new(), - message: format!("cannot inspect parent Git index: {error}"), - })?; + let index = repository + .index_or_empty() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git index: {error}"), + })?; let mut paths = BTreeSet::new(); for entry in index.entries() { if entry.mode.is_commit() { From 77332ede0f9b9cb557bd41185203eba9500c3f0a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:54:15 -0800 Subject: [PATCH 11/42] fix(code-index): deny ignored admission beneath Git gitlinks A parent stage-0 gitlink remains tracked repository source even when its child checkout loses the .git marker. Check every indexed ancestor before an ignored dependency is admitted or revalidated, preserving the existing proof for ordinary ignored sources. Add a real local Git submodule regression that removes the child marker and checks both the admission proof and production source-read refusal. Refs #4593 --- .../ignored_dependencies.rs | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index a5475fc5906..089b3f80d12 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -861,6 +861,20 @@ fn prove_gix_ignored(root: &Path, logical_path: &str) -> Result<(), CodeIndexSch { return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); } + // A registered parent gitlink is still tracked source when the child + // checkout loses its .git marker. Ignore rules may match its descendants, + // but that cannot authorize indexing those descendants as dependencies. + let mut ancestor = Path::new(logical_path).parent(); + while let Some(relative) = ancestor { + if relative + .to_str() + .and_then(|path| index.entry_by_path(path.as_bytes().as_bstr())) + .is_some_and(|entry| entry.mode.is_commit()) + { + return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); + } + ancestor = relative.parent(); + } let mut excludes = repository .excludes( &index, @@ -933,3 +947,111 @@ fn map_reconcile_interruption_result( ) -> Result { result.map_err(map_reconcile_interruption) } + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::Path; + use std::process::{Command, Output}; + + use super::{ + CodeIndexIgnoredDependencyRefusalV1, CodeIndexSchedulerErrorV1, prove_gix_ignored, + read_bounded_admitted_source, + }; + + fn fixture_git(root: &Path, args: &[&str]) -> Output { + let program = tracedecay_runtime_core::git::try_git_program().expect("Git program"); + let mut command = Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run fixture Git"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + output + } + + #[test] + fn parent_gitlink_cannot_be_admitted_as_ignored_dependency_without_gitdir_marker() { + let fixture = tempfile::tempdir().expect("repository fixture"); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(&child).unwrap(); + + fixture_git(&child, &["init", "-q"]); + fs::write(child.join("index.ts"), "export type Child = number;\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child"]); + + fixture_git(&parent, &["init", "-q"]); + fs::write(parent.join(".gitignore"), "node_modules/\n").unwrap(); + fixture_git(&parent, &["add", ".gitignore"]); + fixture_git(&parent, &["commit", "-qm", "parent"]); + fixture_git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-f", + "-q", + "../child", + "node_modules/pkg", + ], + ); + + let staged = fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]); + assert!(staged.stdout.starts_with(b"160000 ")); + assert!(staged.stdout.ends_with(b" 0\tnode_modules/pkg\n")); + assert!(parent.join(".gitmodules").is_file()); + + fs::remove_file(parent.join("node_modules/pkg/.git")).unwrap(); + assert!(parent.join("node_modules/pkg/index.ts").is_file()); + let denied = prove_gix_ignored(&parent, "node_modules/pkg/index.ts"); + assert!(matches!( + denied, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + let admitted = read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); + assert!(matches!( + admitted, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + + // Ordinary ignored sources are still eligible; only gitlink ancestry + // withdraws the source trust that the ignored-dependency proof needs. + fs::create_dir_all(parent.join("node_modules/loose")).unwrap(); + fs::write( + parent.join("node_modules/loose/index.ts"), + "export type Loose = string;\n", + ) + .unwrap(); + prove_gix_ignored(&parent, "node_modules/loose/index.ts") + .expect("non-repository ignored source remains admissible"); + } +} From 2b230e8232f6ffd23675223c261fcfbd0a26e80a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:25:50 -0800 Subject: [PATCH 12/42] fix(code-index): preserve HEAD gitlink boundary on staged removal Refs #4593. Full authenticated child indexing still incomplete. --- .../tracedecay-code-index/src/source_walk.rs | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 86056966076..d0d3c932ab4 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -162,6 +162,47 @@ fn parent_gitlink_paths(project_root: &Path) -> Result, SourceW paths.insert(relative.to_owned()); } } + // A staged gitlink removal does not erase the checkout boundary from + // HEAD. Without this second authority, a child whose .git marker was + // removed becomes ordinary live source until the parent commits. + let head = repository.head().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git HEAD: {error}"), + })?; + if head.id().is_some() { + let commit = repository.head_commit().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot resolve parent Git HEAD: {error}"), + })?; + let tree = commit.tree().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot read parent Git HEAD tree: {error}"), + })?; + let entries = tree + .traverse() + .breadthfirst + .files() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot walk parent Git HEAD tree: {error}"), + })?; + for entry in entries { + if !entry.mode.is_commit() { + continue; + } + let path: &[u8] = entry.filepath.as_ref(); + let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { + glob: String::new(), + message: "non-UTF-8 HEAD gitlink cannot be safely scoped".to_owned(), + })?; + paths.insert(relative.to_owned()); + } + } else if head.referent_name().is_none() { + return Err(SourceWalkError { + glob: String::new(), + message: "parent Git HEAD has neither a commit nor an unborn branch".to_owned(), + }); + } Ok(paths) } @@ -752,6 +793,36 @@ mod tests { .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) .count(); assert_eq!(explicitly_scoped, 0); + + // Staging removal of the gitlink also removes it from the parent + // index, but HEAD still owns that repository boundary. Neither an + // explicit include nor a path glob may adopt its live child bytes. + fixture_git(&root, &["add", ".gitmodules", "src/parent.rs"]); + fixture_git(&root, &["commit", "-qm", "parent with submodule"]); + fixture_git(&root, &["rm", "--cached", "-f", "app"]); + assert!( + fixture_git(&root, &["ls-files", "--stage", "--", "app"]) + .stdout + .is_empty() + ); + assert!( + fixture_git(&root, &["ls-tree", "HEAD", "--", "app"]) + .stdout + .starts_with(b"160000 commit ") + ); + assert!(root.join("app/src/feature.rs").is_file()); + + let files = walked_files(&root, &policy); + assert!( + files.iter().all(|path| !path.starts_with("app")), + "HEAD gitlink is still an excluded source boundary: {files:?}" + ); + let scoped = source_walk(&root, Some("app/**"), &policy) + .expect("scope cannot cross HEAD gitlink") + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .count(); + assert_eq!(scoped, 0); } #[test] From f925fe659de81b368a6bd6c2cb40835cb472f179 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:07:45 -0800 Subject: [PATCH 13/42] fix(code-index): deny staged-removal gitlink in source admission Refs #4593. Reuse the parent's index-plus-HEAD boundary for ignored sources and cover the real staged-removal checkout scenario. The authenticated opt-in indexing feature remains incomplete. --- .../ignored_dependencies.rs | 42 ++++++++++++++++--- .../tracedecay-code-index/src/source_walk.rs | 2 +- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index 089b3f80d12..709e72145e6 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -14,6 +14,7 @@ use tracedecay_code_index::production::{ CodeIndexInterruptionV1, CodeIndexProductionErrorV1, MAX_IGNORED_DEPENDENCY_ENTRYPOINT_BYTES_V1, }; +use tracedecay_code_index::source_walk::parent_gitlink_paths; use tracedecay_contracts::ResolvedScope; use tracedecay_contracts::code_index_freshness::CodeIndexConvergenceParkedV1; use tracedecay_domain::{CodeGenerationId, SanitizerDispositionV1}; @@ -861,15 +862,15 @@ fn prove_gix_ignored(root: &Path, logical_path: &str) -> Result<(), CodeIndexSch { return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); } - // A registered parent gitlink is still tracked source when the child - // checkout loses its .git marker. Ignore rules may match its descendants, - // but that cannot authorize indexing those descendants as dependencies. + // Admission and live source walks share the union of staged and HEAD + // gitlinks: staging a removal cannot grant trust over the child checkout. + let parent_gitlinks = parent_gitlink_paths(root) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; let mut ancestor = Path::new(logical_path).parent(); while let Some(relative) = ancestor { if relative .to_str() - .and_then(|path| index.entry_by_path(path.as_bytes().as_bstr())) - .is_some_and(|entry| entry.mode.is_commit()) + .is_some_and(|path| parent_gitlinks.contains(path)) { return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); } @@ -1043,6 +1044,37 @@ mod tests { )) )); + // Removing the staged gitlink does not authorize the child while + // the parent HEAD still owns the repository boundary. + fixture_git(&parent, &["add", ".gitmodules", ".gitignore"]); + fixture_git(&parent, &["commit", "-qm", "parent with submodule"]); + fixture_git(&parent, &["rm", "--cached", "-f", "node_modules/pkg"]); + assert!( + fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]) + .stdout + .is_empty() + ); + assert!( + fixture_git(&parent, &["ls-tree", "HEAD", "--", "node_modules/pkg"]) + .stdout + .starts_with(b"160000 commit ") + ); + let denied = prove_gix_ignored(&parent, "node_modules/pkg/index.ts"); + assert!(matches!( + denied, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + let denied_read = + read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); + assert!(matches!( + denied_read, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + // Ordinary ignored sources are still eligible; only gitlink ancestry // withdraws the source trust that the ignored-dependency proof needs. fs::create_dir_all(parent.join("node_modules/loose")).unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index d0d3c932ab4..8d49b8dc925 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -119,7 +119,7 @@ fn nested_repository_marker(directory: &Path) -> std::io::Result { /// checkout's .git marker has been removed. A gitlink is a repository boundary, /// not an ordinary directory the live source walker can authenticate. /// This is deny-only; it never enables submodule admission. -fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { +pub fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { let marker = project_root.join(".git"); let repository = match tracedecay_runtime_core::git_open::open(project_root) { Ok(repository) => repository, From bcd2b46a77824ebeb7dd0fbe6806336c1f21fa13 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:52:59 -0800 Subject: [PATCH 14/42] style(code-index): fix ignored gitlink regression formatting Refs #4593 --- .../src/code_index_scheduler/ignored_dependencies.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index 709e72145e6..cfae3260dc3 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -1066,8 +1066,7 @@ mod tests { CodeIndexIgnoredDependencyRefusalV1::NotIgnored )) )); - let denied_read = - read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); + let denied_read = read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); assert!(matches!( denied_read, Err(CodeIndexSchedulerErrorV1::IgnoredDependency( From 5edd6c6cb4915da74f5f7a392e340b364064710f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:56:52 -0800 Subject: [PATCH 15/42] fix(code-index): preserve HEAD gitlink boundaries in classification Refs #4593 --- .../code_index_scheduler/classification.rs | 84 +++++++++++++++++-- 1 file changed, 77 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index f8ef32be303..c6c110eba04 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -18,7 +18,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use gix::bstr::ByteSlice; -use tracedecay_code_index::source_walk::has_nested_repository_ancestor; +use tracedecay_code_index::source_walk::{has_nested_repository_ancestor, parent_gitlink_paths}; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; /// Failure to classify worktree changes through gix. @@ -215,6 +215,11 @@ impl WorktreeChangeClassificationV1 { // parent directory once rather than probing per source file. let mut nested_repository_exclusions = BTreeSet::new(); if let Some(project_root) = repository.workdir() { + // The live walker also denies HEAD gitlinks staged for removal. + // Without this check, Git reports their markerless child sources + // as untracked, and worktree capture can re-admit those bytes. + let parent_gitlinks = parent_gitlink_paths(project_root) + .map_err(|error| ClassificationErrorV1::Git(error.message))?; let mut checked_parents = BTreeMap::new(); for path in committed_baseline .iter() @@ -222,12 +227,20 @@ impl WorktreeChangeClassificationV1 { { let relative = Path::new(path); let parent = relative.parent().unwrap_or_else(|| Path::new("")); - let excluded = match checked_parents.get(parent) { - Some(excluded) => *excluded, - None => { - let excluded = has_nested_repository_ancestor(project_root, relative)?; - checked_parents.insert(parent.to_path_buf(), excluded); - excluded + let excluded = if relative + .ancestors() + .filter_map(Path::to_str) + .any(|ancestor| parent_gitlinks.contains(ancestor)) + { + true + } else { + match checked_parents.get(parent) { + Some(excluded) => *excluded, + None => { + let excluded = has_nested_repository_ancestor(project_root, relative)?; + checked_parents.insert(parent.to_path_buf(), excluded); + excluded + } } }; if excluded { @@ -472,4 +485,61 @@ mod nested_repository_tests { assert!(restored.candidate_paths().contains("nested/foreign.rs")); assert!(restored.candidate_paths().contains("linked/foreign.rs")); } + + #[test] + fn staged_gitlink_removal_without_child_marker_does_not_admit_child_sources() { + let fixture = tempfile::tempdir().unwrap(); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(&child).unwrap(); + + fixture_git(&child, &["init", "-q"]); + fs::create_dir_all(child.join("src")).unwrap(); + fs::write(child.join("src/lib.rs"), "pub fn child() {}\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child"]); + + fixture_git(&parent, &["init", "-q"]); + fs::create_dir_all(parent.join("src")).unwrap(); + fs::write(parent.join("src/lib.rs"), "pub fn parent() {}\n").unwrap(); + fixture_git(&parent, &["add", "."]); + fixture_git(&parent, &["commit", "-qm", "parent"]); + fixture_git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + fixture_git(&parent, &["add", ".gitmodules", "app"]); + fixture_git(&parent, &["commit", "-qm", "register submodule"]); + + // Git still records app as a mode-160000 gitlink in HEAD, but + // removing the staged gitlink and child marker makes child files + // appear untracked to the parent. They must never enter its index. + fs::remove_file(parent.join("app/.git")).unwrap(); + fixture_git(&parent, &["rm", "-q", "--cached", "app"]); + assert!(parent.join("app/src/lib.rs").is_file()); + + let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!( + classified + .changes() + .iter() + .any(|change| change.path == "app/src/lib.rs"), + "fixture must exercise a child file reported by parent Git status" + ); + assert!(classified.has_changes(), "clean HEAD reuse must be refused"); + assert!(!classified.candidate_paths().contains("app/src/lib.rs")); + assert!(classified.changed_paths().contains("app/src/lib.rs")); + assert!(classified.candidate_paths().contains("src/lib.rs")); + } + } From d31966d29f5630b479168dafad49ae162ba96c27 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:58:03 -0800 Subject: [PATCH 16/42] fix(code-index): preserve clean registered gitlink state Refs #4593 --- .../src/code_index_scheduler/classification.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index c6c110eba04..f5dee93d95c 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -229,6 +229,7 @@ impl WorktreeChangeClassificationV1 { let parent = relative.parent().unwrap_or_else(|| Path::new("")); let excluded = if relative .ancestors() + .skip(1) .filter_map(Path::to_str) .any(|ancestor| parent_gitlinks.contains(ancestor)) { @@ -520,6 +521,13 @@ mod nested_repository_tests { fixture_git(&parent, &["add", ".gitmodules", "app"]); fixture_git(&parent, &["commit", "-qm", "register submodule"]); + // A registered, clean gitlink is not itself a source file or a + // dirty nested descendant. It must not defeat clean HEAD reuse. + let clean_repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let clean = WorktreeChangeClassificationV1::classify(&clean_repository).unwrap(); + assert!(!clean.has_changes()); + assert!(clean.candidate_paths().contains("src/lib.rs")); + // Git still records app as a mode-160000 gitlink in HEAD, but // removing the staged gitlink and child marker makes child files // appear untracked to the parent. They must never enter its index. @@ -541,5 +549,4 @@ mod nested_repository_tests { assert!(classified.changed_paths().contains("app/src/lib.rs")); assert!(classified.candidate_paths().contains("src/lib.rs")); } - } From a04503a06d1d98b458e86d8499ba9f7d557ca60d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:53:08 -0800 Subject: [PATCH 17/42] fix(code-index): deny gitlink children in freshness roster Refs #4593. Do not merge until integrated submodule admission and Bazel regression acceptance. --- .../code_index_scheduler/freshness_witness.rs | 98 +++++++++++++++++-- 1 file changed, 90 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 8eb1ba4fe0d..28f60ca5431 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -132,15 +132,27 @@ fn source_candidates( let project_root = repository.workdir().ok_or_else(|| { CodeIndexSchedulerErrorV1::Git("source freshness requires a worktree".to_owned()) })?; + // Explicit ignored-file enrollment lifts Git ignore rules, never the + // parent's index/HEAD gitlink boundary. A staged gitlink removal and + // missing child .git marker must not re-admit its files through freshness. + let parent_gitlinks = tracedecay_code_index::source_walk::parent_gitlink_paths(project_root) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; for path in &admitted_paths { - // Explicit ignored-file enrollment lifts Git ignore rules, not the - // nested repository boundary shared with capture and live search. - if !tracedecay_code_index::source_walk::has_nested_repository_ancestor( - project_root, - Path::new(path), - )? { - candidate_paths.insert((*path).to_owned()); + let relative = Path::new(path); + let beneath_gitlink = relative + .ancestors() + .skip(1) + .filter_map(Path::to_str) + .any(|ancestor| parent_gitlinks.contains(ancestor)); + if beneath_gitlink + || tracedecay_code_index::source_walk::has_nested_repository_ancestor( + project_root, + relative, + )? + { + continue; } + candidate_paths.insert((*path).to_owned()); } let mut unrepresentable = classification .non_utf8_paths() @@ -1073,7 +1085,10 @@ mod tracked_parent_boundary_tests { use std::sync::Arc; use std::time::{Duration, Instant}; - use super::{CachedCandidateRosterV1, ChangeClockReading, roster_evidence}; + use super::{ + CachedCandidateRosterV1, ChangeClockReading, CodeIndexIgnoredSourceAdmissionV1, + roster_evidence, source_candidates, + }; fn git(root: &Path, args: &[&str]) { let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git"); @@ -1105,6 +1120,73 @@ mod tracked_parent_boundary_tests { ); } + #[test] + fn ignored_source_cannot_reenter_markerless_parent_gitlink_freshness() { + let fixture = tempfile::tempdir().expect("Git fixture"); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + std::fs::create_dir_all(&parent).unwrap(); + std::fs::create_dir_all(&child).unwrap(); + + git(&child, &["init", "-q"]); + std::fs::write(child.join("child.rs"), "pub fn child() {}\n").unwrap(); + git(&child, &["add", "child.rs"]); + git(&child, &["commit", "-qm", "child"]); + + git(&parent, &["init", "-q"]); + std::fs::create_dir_all(parent.join("ignored")).unwrap(); + std::fs::write(parent.join(".gitignore"), "ignored/\n").unwrap(); + std::fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); + std::fs::write( + parent.join("ignored/ordinary.rs"), + "pub fn ordinary() {}\n", + ) + .unwrap(); + git(&parent, &["add", ".gitignore", "parent.rs"]); + git(&parent, &["commit", "-qm", "parent"]); + git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + git(&parent, &["add", ".gitmodules", "app"]); + git(&parent, &["commit", "-qm", "registered submodule"]); + + // Git still records the child gitlink in HEAD after staging its + // removal. Its missing marker makes child.rs appear untracked, but + // an old explicit ignored admission must not revive it in freshness. + std::fs::remove_file(parent.join("app/.git")).unwrap(); + git(&parent, &["rm", "-q", "--cached", "app"]); + let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let admitted = [ + CodeIndexIgnoredSourceAdmissionV1 { + logical_path: "app/child.rs".to_owned(), + }, + CodeIndexIgnoredSourceAdmissionV1 { + logical_path: "ignored/ordinary.rs".to_owned(), + }, + ]; + let candidates = source_candidates(&repository, &admitted).unwrap(); + let paths = candidates + .candidates + .iter() + .map(|candidate| candidate.logical_path.as_str()) + .collect::>(); + assert!(paths.contains("parent.rs")); + assert!(paths.contains("ignored/ordinary.rs")); + assert!( + !paths.contains("app/child.rs"), + "ignored admission must not cross a HEAD gitlink boundary" + ); + } + #[test] fn cached_tracked_directory_roster_invalidates_on_nested_git_marker() { let root = tempfile::tempdir().expect("parent checkout"); From d6e0edfe128b74abab74a255185729ca827129d6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:20:49 -0800 Subject: [PATCH 18/42] style(code-index): format nested repository fixture --- .../src/code_index_scheduler/freshness_witness.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 28f60ca5431..61606433316 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -1137,11 +1137,7 @@ mod tracked_parent_boundary_tests { std::fs::create_dir_all(parent.join("ignored")).unwrap(); std::fs::write(parent.join(".gitignore"), "ignored/\n").unwrap(); std::fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); - std::fs::write( - parent.join("ignored/ordinary.rs"), - "pub fn ordinary() {}\n", - ) - .unwrap(); + std::fs::write(parent.join("ignored/ordinary.rs"), "pub fn ordinary() {}\n").unwrap(); git(&parent, &["add", ".gitignore", "parent.rs"]); git(&parent, &["commit", "-qm", "parent"]); git( From 1bbf847d0340238b91df20188278caec6d8fa6ea Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:26:06 -0800 Subject: [PATCH 19/42] fix(code-index): reuse verified Git boundaries during freshness --- .../code_index_scheduler/classification.rs | 44 +++++++++++-------- .../code_index_scheduler/freshness_witness.rs | 9 +--- .../tracedecay-code-index/src/source_walk.rs | 19 ++++---- .../source_walk/nested_repository_tests.rs | 14 ++++++ 4 files changed, 52 insertions(+), 34 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index f5dee93d95c..b69b8c16d46 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -97,6 +97,7 @@ pub struct WorktreeChangeClassificationV1 { /// Files still tracked by the parent but hidden behind a nested checkout /// or symlinked ancestor. These are tombstones, not indexable candidates. nested_repository_exclusions: BTreeSet, + parent_gitlinks: BTreeSet, } impl WorktreeChangeClassificationV1 { @@ -213,26 +214,28 @@ impl WorktreeChangeClassificationV1 { // refuses those files; classification must also exclude them and // invalidate clean-tree and active-generation reuse. Resolve each // parent directory once rather than probing per source file. - let mut nested_repository_exclusions = BTreeSet::new(); + let mut classification = Self { + committed_baseline, + changes, + non_utf8_changes, + non_utf8_paths, + ..Self::default() + }; if let Some(project_root) = repository.workdir() { // The live walker also denies HEAD gitlinks staged for removal. // Without this check, Git reports their markerless child sources // as untracked, and worktree capture can re-admit those bytes. - let parent_gitlinks = parent_gitlink_paths(project_root) + classification.parent_gitlinks = parent_gitlink_paths(project_root) .map_err(|error| ClassificationErrorV1::Git(error.message))?; let mut checked_parents = BTreeMap::new(); - for path in committed_baseline + for path in classification + .committed_baseline .iter() - .chain(changes.iter().map(|change| &change.path)) + .chain(classification.changes.iter().map(|change| &change.path)) { let relative = Path::new(path); let parent = relative.parent().unwrap_or_else(|| Path::new("")); - let excluded = if relative - .ancestors() - .skip(1) - .filter_map(Path::to_str) - .any(|ancestor| parent_gitlinks.contains(ancestor)) - { + let excluded = if classification.beneath_gitlink(relative) { true } else { match checked_parents.get(parent) { @@ -245,18 +248,23 @@ impl WorktreeChangeClassificationV1 { } }; if excluded { - nested_repository_exclusions.insert(path.clone()); + classification + .nested_repository_exclusions + .insert(path.clone()); } } } - Ok(Self { - committed_baseline, - changes, - non_utf8_changes, - non_utf8_paths, - nested_repository_exclusions, - }) + Ok(classification) + } + + /// Explicitly enrolled files must respect the same Git boundary snapshot + /// as ordinary candidates, including a gitlink staged for removal. + pub(super) fn beneath_gitlink(&self, path: &Path) -> bool { + path.ancestors() + .skip(1) + .filter_map(Path::to_str) + .any(|ancestor| self.parent_gitlinks.contains(ancestor)) } /// Paths whose Git bytes are not UTF-8, present or not. diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 61606433316..c0edd6eb308 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -135,16 +135,9 @@ fn source_candidates( // Explicit ignored-file enrollment lifts Git ignore rules, never the // parent's index/HEAD gitlink boundary. A staged gitlink removal and // missing child .git marker must not re-admit its files through freshness. - let parent_gitlinks = tracedecay_code_index::source_walk::parent_gitlink_paths(project_root) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; for path in &admitted_paths { let relative = Path::new(path); - let beneath_gitlink = relative - .ancestors() - .skip(1) - .filter_map(Path::to_str) - .any(|ancestor| parent_gitlinks.contains(ancestor)); - if beneath_gitlink + if classification.beneath_gitlink(relative) || tracedecay_code_index::source_walk::has_nested_repository_ancestor( project_root, relative, diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 8d49b8dc925..bc41a8beb7d 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -120,19 +120,22 @@ fn nested_repository_marker(directory: &Path) -> std::io::Result { /// not an ordinary directory the live source walker can authenticate. /// This is deny-only; it never enables submodule admission. pub fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { - let marker = project_root.join(".git"); let repository = match tracedecay_runtime_core::git_open::open(project_root) { Ok(repository) => repository, Err(error) => { // Non-Git projects still support source walks. An initialized // checkout whose Git authority is unreadable must fail closed. - if marker.is_file() || marker.join("HEAD").is_file() { - return Err(SourceWalkError { + return match nested_repository_marker(project_root) { + Ok(false) => Ok(BTreeSet::new()), + Ok(true) => Err(SourceWalkError { glob: String::new(), message: format!("cannot inspect parent Git boundaries: {error}"), - }); - } - return Ok(BTreeSet::new()); + }), + Err(marker_error) => Err(SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git marker: {marker_error}"), + }), + }; } }; if repository @@ -642,7 +645,7 @@ mod tests { #[test] fn nested_repository_is_not_part_of_the_parent_source_tree() { let root = TempDir::new().expect("project root"); - fs::create_dir(root.path().join(".git")).expect("parent git directory"); + fixture_git(root.path(), &["init", "-q"]); fs::create_dir_all(root.path().join("src")).expect("parent source directory"); fs::write(root.path().join("src/lib.rs"), "pub fn parent() {}\n").expect("parent source"); fs::create_dir_all(root.path().join("plain")).expect("plain directory"); @@ -828,7 +831,7 @@ mod tests { #[test] fn submodule_metadata_cannot_break_an_unrelated_source_include() { let root = TempDir::new().expect("project root"); - fs::create_dir_all(root.path().join(".git")).unwrap(); + fixture_git(root.path(), &["init", "-q"]); fs::create_dir_all(root.path().join("src")).unwrap(); fs::create_dir_all(root.path().join("app/.git")).unwrap(); fs::write(root.path().join("src/keep.rs"), "pub fn keep() {}\n").unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs index 0b77991f58e..406671e045d 100644 --- a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs @@ -181,3 +181,17 @@ fn missing_source_root_is_an_error_not_a_clean_audit() { Ok(_) => panic!("a failed walk must not report an empty successful audit"), } } + +#[test] +fn incomplete_git_root_is_refused_before_walking_sources() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "src/lib.rs", "pub fn source() {}"); + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + assert!(audit_project(root.path(), &policy).is_ok()); + + std::fs::create_dir(root.path().join(".git")).unwrap(); + assert!(matches!( + audit_project(root.path(), &policy), + Err(TraceDecayError::Config { message }) if message.contains("parent Git boundaries") + )); +} From dd1ddca5b0fdcb5220e81ac7fa56187410a95b4e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:49:52 -0800 Subject: [PATCH 20/42] fix(code-index): recognize submodule modes in the Git index --- crates/tracedecay-code-index/src/source_walk.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index bc41a8beb7d..d8e5ca885fc 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -156,7 +156,7 @@ pub fn parent_gitlink_paths(project_root: &Path) -> Result, Sou })?; let mut paths = BTreeSet::new(); for entry in index.entries() { - if entry.mode.is_commit() { + if entry.mode.is_submodule() { let path: &[u8] = entry.path(&index).as_ref(); let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { glob: String::new(), From 1f7b37aa41da122818ed3ea67b75657bc9266f00 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:55:55 -0800 Subject: [PATCH 21/42] fix(code-index): borrow Git paths for roster freshness checks --- .../src/code_index_scheduler/freshness_witness.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index c0edd6eb308..721491ddc45 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -535,7 +535,7 @@ fn roster_evidence( } }; for entry in index.entries() { - collect_ancestors(gix::path::from_bstr(entry.path(&index)).ok()?); + collect_ancestors(gix::path::from_bstr(entry.path(&index)).ok()?.as_ref()); } for admission in ignored_source_admissions { if validate_code_logical_path(&admission.logical_path).is_err() { From 5eaa4993902f1f9dd545eb92b4aed304737f2a36 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:59:40 -0800 Subject: [PATCH 22/42] test(code-index): bind ignored reads to the canonical fixture root --- .../ignored_dependencies.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index cfae3260dc3..2819eb78153 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -955,6 +955,8 @@ mod tests { use std::path::Path; use std::process::{Command, Output}; + use tracedecay_runtime_core::path_safety::canonical_existing_identity; + use super::{ CodeIndexIgnoredDependencyRefusalV1, CodeIndexSchedulerErrorV1, prove_gix_ignored, read_bounded_admitted_source, @@ -998,6 +1000,7 @@ mod tests { let child = fixture.path().join("child"); fs::create_dir_all(&parent).unwrap(); fs::create_dir_all(&child).unwrap(); + let parent = canonical_existing_identity(&parent).expect("canonical project root"); fixture_git(&child, &["init", "-q"]); fs::write(child.join("index.ts"), "export type Child = number;\n").unwrap(); @@ -1037,12 +1040,15 @@ mod tests { )) )); let admitted = read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); - assert!(matches!( - admitted, - Err(CodeIndexSchedulerErrorV1::IgnoredDependency( - CodeIndexIgnoredDependencyRefusalV1::NotIgnored - )) - )); + assert!( + matches!( + admitted, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + ), + "unexpected source admission: {admitted:?}" + ); // Removing the staged gitlink does not authorize the child while // the parent HEAD still owns the repository boundary. From bc7c1fc9edbee03318afd4d408374b0c3cb5adca Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:55:19 -0800 Subject: [PATCH 23/42] fix(code-index): refuse symlinked source leaf capture Refs #4593. Preserve live-walk exclusion through classification, worktree capture, and freshness even when a tracked symlink resolves into a registered child repository. Adds real Git submodule and symlink regressions. Partial boundary repair; keep PR draft. --- .../code_index_scheduler/classification.rs | 63 ++++++++++++++++++- .../code_index_scheduler/freshness_witness.rs | 13 ++++ .../src/code_index_scheduler/reconcile.rs | 6 +- .../tracedecay-code-index/src/source_walk.rs | 40 +++++++++++- 4 files changed, 118 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index b69b8c16d46..0550c4a843c 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -18,7 +18,9 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use gix::bstr::ByteSlice; -use tracedecay_code_index::source_walk::{has_nested_repository_ancestor, parent_gitlink_paths}; +use tracedecay_code_index::source_walk::{ + has_nested_repository_ancestor, is_source_symlink, parent_gitlink_paths, +}; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; /// Failure to classify worktree changes through gix. @@ -235,7 +237,13 @@ impl WorktreeChangeClassificationV1 { { let relative = Path::new(path); let parent = relative.parent().unwrap_or_else(|| Path::new("")); - let excluded = if classification.beneath_gitlink(relative) { + // The live walker never follows a symlink file, even when + // the lexical parent is ordinary and the target is inside a + // registered submodule. Do not capture its target through a + // clean-tree or worktree path. + let excluded = if classification.beneath_gitlink(relative) + || is_source_symlink(project_root, relative)? + { true } else { match checked_parents.get(parent) { @@ -495,6 +503,57 @@ mod nested_repository_tests { assert!(restored.candidate_paths().contains("linked/foreign.rs")); } + #[cfg(unix)] + #[test] + fn tracked_symlink_into_submodule_is_not_parent_source() { + use std::os::unix::fs::symlink; + + let fixture = tempfile::tempdir().unwrap(); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(&child).unwrap(); + fixture_git(&child, &["init", "-q"]); + fs::create_dir_all(child.join("src")).unwrap(); + fs::write(child.join("src/lib.rs"), "pub fn child() {}\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child"]); + fixture_git(&parent, &["init", "-q"]); + fixture_git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + symlink("app/src/lib.rs", parent.join("bridge.rs")).unwrap(); + fixture_git(&parent, &["add", ".gitmodules", "app", "bridge.rs"]); + fixture_git(&parent, &["commit", "-qm", "registered child and source alias"]); + assert_eq!( + fs::read_to_string(parent.join("bridge.rs")).unwrap(), + "pub fn child() {}\n", + "fixture must prove the parent symlink actually reaches child bytes" + ); + assert!( + fs::symlink_metadata(parent.join("bridge.rs")) + .unwrap() + .file_type() + .is_symlink() + ); + + let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!(!classified.candidate_paths().contains("bridge.rs")); + assert!(classified.changed_paths().contains("bridge.rs")); + assert!(classified.has_changes(), "do not reuse HEAD tree source bytes"); + assert!(!classified.candidate_paths().contains("app/src/lib.rs")); + } + #[test] fn staged_gitlink_removal_without_child_marker_does_not_admit_child_sources() { let fixture = tempfile::tempdir().unwrap(); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 721491ddc45..d81c425e9e3 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -142,6 +142,7 @@ fn source_candidates( project_root, relative, )? + || tracedecay_code_index::source_walk::is_source_symlink(project_root, relative)? { continue; } @@ -243,6 +244,18 @@ fn read_candidate( project_root: &Path, candidate: &StatCandidateV1, ) -> Result, CodeIndexSchedulerErrorV1> { + // A link installed after candidate classification must not turn a + // freshness proof into a read of a different repository's source bytes. + if tracedecay_code_index::source_walk::is_source_symlink( + project_root, + Path::new(&candidate.logical_path), + )? { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "source symlink is outside the parent source inventory", + ) + .into()); + } if candidate.explicitly_admitted { ignored_dependencies::read_explicitly_admitted_source( project_root, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index e2d51bfeb61..5724543efff 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -3976,9 +3976,13 @@ impl CodeIndexWorktreeSchedulerV1 { ) -> Result { // Explicit ignored-source admission cannot cross a nested checkout // any more than an ordinary Git candidate or interactive source walk. + let relative = std::path::Path::new(logical_path); if tracedecay_code_index::source_walk::has_nested_repository_ancestor( &self.project_root, - std::path::Path::new(logical_path), + relative, + )? || tracedecay_code_index::source_walk::is_source_symlink( + &self.project_root, + relative, )? { return Ok(CapturedFileOutcomeV1::Absent); } diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index d8e5ca885fc..a72a87b77ea 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -247,6 +247,28 @@ pub fn has_nested_repository_ancestor( Ok(false) } +/// A worktree source entry must be a real file, not a link to bytes outside +/// the parent's Git authority. The shared live walker does not follow leaf +/// symlinks; classification, capture, and freshness must make the same choice. +/// Missing paths remain possible tombstones and are not symlink exclusions. +pub fn is_source_symlink(project_root: &Path, relative_path: &Path) -> std::io::Result { + if relative_path.as_os_str().is_empty() + || relative_path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "source path must be repository-relative", + )); + } + match std::fs::symlink_metadata(project_root.join(relative_path)) { + Ok(metadata) => Ok(metadata.file_type().is_symlink()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + /// Diagnostics emitted by the same source walker that enforces the nested /// repository boundary. Only directories within the walk's policy are named. #[derive(Clone, Default)] @@ -480,7 +502,9 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::{has_nested_repository_ancestor, source_walk, source_walk_with_diagnostics}; + use super::{ + has_nested_repository_ancestor, is_source_symlink, source_walk, source_walk_with_diagnostics, + }; #[test] fn nested_repository_boundary_excludes_parent_markers_and_path_escape() { @@ -526,6 +550,20 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn readable_symlink_leaf_is_not_an_admissible_source() { + use std::os::unix::fs::symlink; + + let root = TempDir::new().unwrap(); + fs::write(root.path().join("real.rs"), "pub fn real() {}\n").unwrap(); + symlink("real.rs", root.path().join("alias.rs")).unwrap(); + assert!(is_source_symlink(root.path(), std::path::Path::new("alias.rs")).unwrap()); + assert!(!is_source_symlink(root.path(), std::path::Path::new("real.rs")).unwrap()); + assert!(!is_source_symlink(root.path(), std::path::Path::new("missing.rs")).unwrap()); + assert!(is_source_symlink(root.path(), std::path::Path::new("../escape.rs")).is_err()); + } + fn walked_files(root: &std::path::Path, path_policy: &IndexPathPolicyV1) -> Vec { let mut files = source_walk(root, None, path_policy) .expect("source walk") From 632729906cbf0fe46b858bd556e716550db062b2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:52:34 -0800 Subject: [PATCH 24/42] test(automation): bind scheduler fixtures to backend identities Repair three hosted automation_runner_test failures by supplying the configured executable identity required by the production prefilter. Use the same identity for the prefilter and retained runner, preserving manual-schedule, exact-ledger, live-lock and lock-race assertions. Add a production-prefilter regression for unconfigured refusal, deduplication and same-occurrence recovery after binding, with zero backend calls and no effectful terminal. Do not relax backend admission, change timeouts, or alter the graph publication changes. Refs #4532 --- .../tests/automation_runner_test/jobs.rs | 124 ++++++++++++++++-- 1 file changed, 110 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay/tests/automation_runner_test/jobs.rs b/crates/tracedecay/tests/automation_runner_test/jobs.rs index 740283d56e8..7342786381b 100644 --- a/crates/tracedecay/tests/automation_runner_test/jobs.rs +++ b/crates/tracedecay/tests/automation_runner_test/jobs.rs @@ -86,6 +86,7 @@ fn enabled_job_config() -> AutomationSettingsV1 { struct ContentBackend { calls: AtomicUsize, content: &'static str, + executable: Option, } impl ContentBackend { @@ -93,9 +94,15 @@ impl ContentBackend { Self { calls: AtomicUsize::new(0), content, + executable: None, } } + fn with_executable(mut self, executable: &Path) -> Self { + self.executable = Some(executable.to_path_buf()); + self + } + fn calls(&self) -> usize { self.calls.load(Ordering::SeqCst) } @@ -126,7 +133,7 @@ impl AgentTaskBackend for ContentBackend { } fn executable(&self) -> Option<&std::path::Path> { - None + self.executable.as_deref() } } @@ -988,6 +995,10 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let mut job = sample_job("sched-job"); job.schedule = Some("manual".to_string()); let config = enabled_job_config(); + // Reach the schedule gate with a bound identity, not the earlier + // backend-unconfigured refusal. This gate never invokes the backend. + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); // Manual schedule: every scheduler tick of the same occurrence reaches // the prefilter, which records the skip once. @@ -996,7 +1007,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let run = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1019,7 +1030,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let repeated = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1031,6 +1042,76 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { repeated.ledger_record, records[0], "an immediate repeat skip must return the exact durable prior row" ); + assert_eq!(backend.calls(), 0); +} + +#[tokio::test] +async fn scheduler_prefilter_rejects_unconfigured_backend_and_recovers_without_effect() { + let temp = tempdir().unwrap(); + let dashboard_root = temp.path().join("dashboard"); + let job = sample_job("backend-recovery"); + let config = enabled_job_config(); + let occurrence = "backend-recovery-occurrence"; + let backend = ContentBackend::new("unused"); + + let first = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("a due job still requires a configured backend"); + assert_eq!(first.report["reason"], json!("backend_unconfigured")); + assert_eq!(first.ledger_record.status, AutomationRunStatus::Skipped); + assert_ne!(first.run_id, occurrence); + let repeated = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(repeated.ledger_record, first.ledger_record); + + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = backend.with_executable(executable.path()); + assert!( + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), + "binding the backend must admit the same due occurrence without cooldown" + ); + assert_eq!(backend.calls(), 0); + assert_eq!( + load_run_records(&dashboard_root, 10).await.unwrap(), + vec![first.ledger_record], + "advisory admission must not append an effectful terminal" + ); + assert!( + tracedecay_automation_runtime::automation::run_ledger::find_run_record_exact_bounded( + &dashboard_root, + occurrence, + ) + .await + .unwrap() + .is_none() + ); } #[tokio::test] @@ -1206,15 +1287,22 @@ async fn scheduler_prefilter_live_lock_wins_over_not_due_summary() { let config = enabled_job_config(); let occurrence = "locked-prefilter-occurrence"; - let prefilter = - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .expect("the live job lock must produce a diagnostic"); + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); + let prefilter = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("the live job lock must produce a diagnostic"); assert_eq!(prefilter.report["reason"], json!("scheduler_lock_active")); assert_ne!(prefilter.run_id, occurrence); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, @@ -1262,12 +1350,21 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { job.schedule = Some("every 1h".to_owned()); let config = enabled_job_config(); let occurrence = "prefilter-gap-occurrence"; + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); assert!( - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .is_none(), + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), "a new interval job is advisory-due before effect admission" ); let _raced_in = @@ -1276,7 +1373,6 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { .unwrap() .expect("the advisory prefilter must release its reservation lock"); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, From 3fdb50ddbe2b9fe19f80f3bb94b96d2c738c66b0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:03:19 -0800 Subject: [PATCH 25/42] feat(code-index): index authenticated direct submodules --- Cargo.lock | 1 + .../src/code_index_scheduler.rs | 2 +- .../code_index_scheduler/freshness_witness.rs | 128 +++++++-- .../code_index_scheduler/git_tree_capture.rs | 119 ++++++-- .../src/code_index_scheduler/identity.rs | 60 +++- .../ignored_dependencies.rs | 2 +- .../src/code_index_scheduler/reconcile.rs | 124 +++++++-- .../tests/index_path_policy_tests.rs | 213 ++++++++++++++ .../tests/code_index_scheduler/reconcile.rs | 26 +- crates/tracedecay-code-index/Cargo.toml | 1 + .../benches/partitioned_codec.rs | 2 + .../benches/sealed_storage.rs | 2 + .../src/production/ignored_sources.rs | 3 + .../src/production/lexical_page_source.rs | 1 + .../production/lexical_page_source_tests.rs | 2 + .../src/production/mod.rs | 20 ++ .../src/production/sparse_increment_tests.rs | 1 + .../src/production/worker_tests.rs | 2 + .../tracedecay-code-index/src/source_walk.rs | 262 +++++++++++++----- .../src/source_walk}/classification.rs | 115 +++++++- .../src/source_walk/submodules.rs | 188 +++++++++++++ .../production_orchestration.rs | 5 + .../parallel_equivalence.rs | 1 + .../tests/resident_accounting.rs | 2 + .../src/config/mod.rs | 34 ++- crates/tracedecay-domain/src/configuration.rs | 2 + crates/tracedecay-domain/src/lib.rs | 7 +- .../src/source_path_policy.rs | 18 +- .../src/configuration/registry.rs | 20 +- .../src/configuration/store.rs | 11 +- .../src/configuration/store/tests/mod.rs | 45 +-- .../src/handlers/analysis/field_sites.rs | 7 +- .../src/handlers/analysis/mod.rs | 8 +- .../src/handlers/graph/rename_preview.rs | 16 ++ .../benches/code_lexical_catchup.rs | 2 + .../src/bin/tracedecay_index_bench.rs | 2 + .../src/bin/tracedecay_search_bench.rs | 2 + .../candidate_producers.rs | 2 + .../src/git_repository.rs | 2 +- .../src/git_repository/native_integration.rs | 47 +++- .../native_integration_authority.rs | 3 + .../src/candidate_output.rs | 2 + .../src/candidate_output/cancellation.rs | 2 + docs/USER-GUIDE.md | 16 ++ 44 files changed, 1321 insertions(+), 209 deletions(-) rename crates/{tracedecay-code-index-runtime/src/code_index_scheduler => tracedecay-code-index/src/source_walk}/classification.rs (85%) create mode 100644 crates/tracedecay-code-index/src/source_walk/submodules.rs diff --git a/Cargo.lock b/Cargo.lock index 860f7e0e1d2..4238a98de78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6430,6 +6430,7 @@ dependencies = [ "base64 0.23.1", "criterion", "flate2", + "gix", "hex", "ignore", "miniz_oxide", diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs index a325e598138..d3ea24dc19c 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs @@ -201,7 +201,7 @@ mod activation; pub mod branch_generations; pub mod branch_publication; mod cadence; -mod classification; +use tracedecay_code_index::source_walk::classification; mod demand_admission; mod freshness_witness; pub use freshness_witness::sealed_source_bytes; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index d81c425e9e3..b98634731bb 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -76,10 +76,12 @@ pub struct WorktreeStatSweepV1 { pub fn worktree_stat_sweep( project_root: &Path, ignored_source_admissions: &[CodeIndexIgnoredSourceAdmissionV1], + include_submodules: bool, ) -> Result { let repository = tracedecay_runtime_core::git_open::open(project_root) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - let candidate_roster = source_candidates(&repository, ignored_source_admissions)?; + let candidate_roster = + source_candidates(&repository, ignored_source_admissions, include_submodules)?; // One sweep span covers the O(candidates) stat walk. Never emit one // trace event per file. let mut buf = Vec::new(); @@ -120,9 +122,11 @@ struct SourceCandidatesV1 { fn source_candidates( repository: &gix::Repository, ignored_source_admissions: &[CodeIndexIgnoredSourceAdmissionV1], + include_submodules: bool, ) -> Result { - let classification = classification::WorktreeChangeClassificationV1::classify(repository) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let classification = + classification::WorktreeChangeClassificationV1::for_policy(repository, include_submodules) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; let registry = StaticLanguageRegistry::new(); let admitted_paths = ignored_source_admissions .iter() @@ -647,17 +651,27 @@ impl SourceSweepCacheV1 { let Ok(repository) = tracedecay_runtime_core::git_open::open(project_root) else { return (false, stats); }; - let evidence = roster_evidence( - &repository, - project_root, - sampled_at, - ignored_source_admissions, - ); + // Parent-only directory evidence cannot vouch for child Git indexes, + // refs or untracked entries. Enumerate both Git rosters on every child proof. + let evidence = (!self.path_policy.includes_submodules()) + .then(|| { + roster_evidence( + &repository, + project_root, + sampled_at, + ignored_source_admissions, + ) + }) + .flatten(); let Ok(SourceCandidatesV1 { candidates, unrepresentable, .. - }) = source_candidates(&repository, ignored_source_admissions) + }) = source_candidates( + &repository, + ignored_source_admissions, + self.path_policy.includes_submodules(), + ) else { return (false, stats); }; @@ -735,12 +749,14 @@ impl SourceSweepCacheV1 { &mut self, project_root: &Path, manifest: &SourceContentManifestV1, + candidate_paths: &BTreeSet, shutting_down: &AtomicBool, ) -> Result, CodeIndexSchedulerErrorV1> { let registry = StaticLanguageRegistry::new(); let candidates = manifest .files .keys() + .filter(|path| candidate_paths.contains(*path)) .filter_map(|logical_path| source_candidate(®istry, logical_path.clone(), false)) .collect::>(); let present = present_candidates( @@ -838,7 +854,12 @@ impl SourceSweepCacheV1 { if disputed.is_empty() { return Ok(disputed); } - Ok(clean_filter_mismatches(project_root, disputed, manifest)) + Ok(clean_filter_mismatches( + project_root, + disputed, + manifest, + self.path_policy.includes_submodules(), + )) } } @@ -882,10 +903,56 @@ fn clean_filter_mismatches<'a>( project_root: &Path, disputed: Vec<&'a StatCandidateV1>, manifest: &SourceContentManifestV1, + include_submodules: bool, ) -> Vec<&'a StatCandidateV1> { let Ok(repository) = tracedecay_runtime_core::git_open::open(project_root) else { return disputed; }; + let mut remaining = disputed; + let mut mismatches = Vec::new(); + if include_submodules { + let Ok(children) = + tracedecay_code_index::source_walk::submodules::live_submodules(&repository) + else { + return remaining; + }; + for (prefix, binding) in children { + let (local, other): (Vec<_>, Vec<_>) = remaining + .into_iter() + .partition(|candidate| Path::new(&candidate.logical_path).starts_with(&prefix)); + remaining = other; + if local.is_empty() { + continue; + } + match tracedecay_runtime_core::git_open::open(&binding.git_dir) { + Ok(child) => mismatches.extend(filter_mismatches_in_repository( + project_root, + &child, + Path::new(&prefix), + local, + manifest, + )), + Err(_) => mismatches.extend(local), + } + } + } + mismatches.extend(filter_mismatches_in_repository( + project_root, + &repository, + Path::new(""), + remaining, + manifest, + )); + mismatches +} + +fn filter_mismatches_in_repository<'a>( + project_root: &Path, + repository: &gix::Repository, + prefix: &Path, + disputed: Vec<&'a StatCandidateV1>, + manifest: &SourceContentManifestV1, +) -> Vec<&'a StatCandidateV1> { let Ok((mut pipeline, index)) = repository.filter_pipeline(None) else { return disputed; }; @@ -895,15 +962,17 @@ fn clean_filter_mismatches<'a>( let Some(expected) = manifest.files.get(&candidate.logical_path) else { return true; }; + let Ok(rela_path) = Path::new(&candidate.logical_path).strip_prefix(prefix) else { + return true; + }; if candidate.explicitly_admitted || index - .entry_by_path(BStr::new(candidate.logical_path.as_str())) + .entry_by_path(BStr::new(rela_path.as_os_str().as_encoded_bytes())) .is_none() { return true; } - let rela_path = Path::new(&candidate.logical_path); - let Ok(file) = std::fs::File::open(project_root.join(rela_path)) else { + let Ok(file) = std::fs::File::open(project_root.join(&candidate.logical_path)) else { return true; }; let Ok(mut converted) = pipeline.convert_to_git(file, rela_path, &index) else { @@ -930,8 +999,13 @@ pub fn sealed_source_bytes( project_root: &Path, logical_path: &str, language: &LanguageId, + include_submodules: bool, sealed: &ContentDigest, ) -> Result>, CodeIndexSchedulerErrorV1> { + if tracedecay_code_index::source_walk::is_source_symlink(project_root, Path::new(logical_path))? + { + return Ok(None); + } let raw = ignored_dependencies::read_bounded_snapshot_source(&project_root.join(logical_path), None)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &raw)?; @@ -942,15 +1016,35 @@ pub fn sealed_source_bytes( // hoist them into a caller-held reader if a caller ever needs hundreds. let repository = tracedecay_runtime_core::git_open::open(project_root) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let mut repository = repository; + let mut relative = Path::new(logical_path); + if include_submodules { + let children = tracedecay_code_index::source_walk::submodules::live_submodules(&repository) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; + if let Some((prefix, binding)) = children + .iter() + .find(|(prefix, _)| relative.starts_with(prefix)) + { + let child_path = relative + .strip_prefix(prefix) + .map_err(|_| CodeIndexSchedulerErrorV1::Git("invalid child filter path".into()))?; + repository = tracedecay_runtime_core::git_open::open(&binding.git_dir) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + relative = child_path; + } + } let (mut pipeline, index) = repository .filter_pipeline(None) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - if index.entry_by_path(BStr::new(logical_path)).is_none() { + if index + .entry_by_path(BStr::new(relative.as_os_str().as_encoded_bytes())) + .is_none() + { return Ok(None); } let mut filtered = Vec::new(); pipeline - .convert_to_git(raw.as_slice(), Path::new(logical_path), &index) + .convert_to_git(raw.as_slice(), relative, &index) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? .read_to_end(&mut filtered)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &filtered)?; @@ -1175,7 +1269,7 @@ mod tracked_parent_boundary_tests { logical_path: "ignored/ordinary.rs".to_owned(), }, ]; - let candidates = source_candidates(&repository, &admitted).unwrap(); + let candidates = source_candidates(&repository, &admitted, false).unwrap(); let paths = candidates .candidates .iter() diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs index 7db65b9c882..922d0c889e9 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs @@ -16,7 +16,9 @@ use tracedecay_domain::{ }; use tracedecay_privacy::CODE_SOURCE_SANITIZER_VERSION_V1; use tracedecay_query::code_search::CodeIndexSearchUnavailableReasonV1; -use tracedecay_runtime_core::git_repository::resolve_tree_link; +use tracedecay_runtime_core::git_repository::{ + GitNativeCandidateTreeVisitError, GitTreeSubmodulePins, resolve_tree_link, +}; use tracedecay_runtime_core::resident_memory::ResidentMemoryReservationV1; use super::{ @@ -161,7 +163,7 @@ pub(super) fn representable_logical_path(git_path: &[u8]) -> Option<&str> { /// Whether capture would parse `git_path` as source. A committed symlink at /// such a path is read through to its target, as a checkout reads it, so its /// link text never becomes source. -fn is_source_path(registry: &StaticLanguageRegistry, git_path: &[u8]) -> bool { +pub(super) fn is_source_path(registry: &StaticLanguageRegistry, git_path: &[u8]) -> bool { representable_logical_path(git_path) .and_then(|logical_path| Path::new(logical_path).extension()) .and_then(|extension| extension.to_str()) @@ -396,9 +398,8 @@ impl CodeIndexWorktreeSchedulerV1 { } /// HEAD's tree standing in for a clean checkout. Git stores a symlink as - /// its target path, while worktree capture and every source sweep read - /// through the link, so a tree holding a source symlink would seal bytes - /// the next sweep disproves. That tree is `GenerationUnavailable` here and + /// its target path, while the live roster excludes the link. A tree holding + /// a source symlink would seal bytes the next sweep disproves. That tree is `GenerationUnavailable` here and /// the checkout is captured from disk instead. pub(super) fn capture_clean_head_tree_snapshot( &self, @@ -458,8 +459,15 @@ impl CodeIndexWorktreeSchedulerV1 { source.tree.clone(), control, |visitor| { + let mut gitlinks = Vec::new(); for entry in entries { control.termination().map_or(Ok(()), Err)?; + if self.path_policy.includes_submodules() && entry.mode.is_commit() { + let path = entry.filepath.to_str().map_err(|_| { + CodeIndexSearchUnavailableReasonV1::GenerationUnavailable + })?; + gitlinks.push((path.to_owned(), entry.oid)); + } if entry.mode.is_tree() || entry.mode.is_commit() { continue; } @@ -480,11 +488,61 @@ impl CodeIndexWorktreeSchedulerV1 { } visitor(entry.filepath.as_ref(), &blob.data)?; } - Ok(()) + let pins = GitTreeSubmodulePins::from_tree(&tree, gitlinks) + .map_err(|_| CodeIndexSearchUnavailableReasonV1::GenerationUnavailable)?; + self.visit_pinned_submodule_blobs(&repository, &pins, control, visitor) }, ) } + fn visit_pinned_submodule_blobs( + &self, + parent: &gix::Repository, + pins: &GitTreeSubmodulePins, + control: &branch_generations::BranchGenerationReadControlV1, + visitor: &mut dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1>, + ) -> Result<(), CodeIndexSearchUnavailableReasonV1> { + let unavailable = CodeIndexSearchUnavailableReasonV1::GenerationUnavailable; + let children = + tracedecay_code_index::source_walk::submodules::pinned_submodules(parent, pins) + .map_err(|_| unavailable)?; + let registry = StaticLanguageRegistry::new(); + for (prefix, (repo, pin)) in children { + control.termination().map_or(Ok(()), Err)?; + let tree = repo + .find_commit(pin) + .and_then(|commit| commit.tree()) + .map_err(|_| unavailable)?; + for entry in tree + .traverse() + .breadthfirst + .files() + .map_err(|_| unavailable)? + { + control.termination().map_or(Ok(()), Err)?; + if !entry.mode.is_blob() { + continue; + } + let path = [prefix.as_bytes(), b"/", entry.filepath.as_slice()].concat(); + let Some(logical_path) = representable_logical_path(&path) else { + // Capture records the unrepresentable name without consulting its blob. + visitor(&path, &[])?; + continue; + }; + if !is_source_path(®istry, &path) + || !tracedecay_code_index::source_walk::submodules::is_source_location( + Path::new(logical_path), + ) + { + continue; + } + let blob = repo.find_blob(entry.oid).map_err(|_| unavailable)?; + visitor(&path, &blob.data)?; + } + } + Ok(()) + } + #[tracing::instrument( name = "daemon.code_index.capture.native_candidate_tree", level = "trace", @@ -510,14 +568,24 @@ impl CodeIndexWorktreeSchedulerV1 { expected_tree.clone(), control, |visitor| { - candidate.visit_blobs(|path| is_source_path(®istry, path), visitor).map_err(|error| match error { - tracedecay_runtime_core::git_repository::GitNativeCandidateTreeVisitError::Repository(_) => { - control + let pins = candidate + .visit_blobs( + self.path_policy.includes_submodules(), + |path| is_source_path(®istry, path), + &mut *visitor, + ) + .map_err(|error| match error { + GitNativeCandidateTreeVisitError::Repository(_) => control .termination() - .unwrap_or(CodeIndexSearchUnavailableReasonV1::GenerationUnavailable) - } - tracedecay_runtime_core::git_repository::GitNativeCandidateTreeVisitError::Visitor(reason) => reason, - }) + .unwrap_or(CodeIndexSearchUnavailableReasonV1::GenerationUnavailable), + GitNativeCandidateTreeVisitError::Visitor(reason) => reason, + })?; + if pins.gitlinks.is_empty() { + return Ok(()); + } + let parent = tracedecay_runtime_core::git_open::open(&self.project_root) + .map_err(|_| CodeIndexSearchUnavailableReasonV1::GenerationUnavailable)?; + self.visit_pinned_submodule_blobs(&parent, &pins, control, visitor) }, ) } @@ -601,6 +669,7 @@ impl CodeIndexWorktreeSchedulerV1 { snapshot_content_identity(&files, &sanitization_receipts, &omitted_sources); Ok(CapturedSnapshotV1 { repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: self.path_policy.includes_submodules(), tree: Some(tree), dirty: tracedecay_domain::RepositoryDirtyStateV1::Clean, }, @@ -1326,16 +1395,24 @@ mod tests { ], "only source links that name a file in the tree are captured" ); + let live = present( + &scheduler + .capture_authoritative_snapshot_without_active_generation_reuse(None) + .expect("capture the checkout from disk") + .snapshot, + ); assert_eq!( - branch, - present( - &scheduler - .capture_authoritative_snapshot_without_active_generation_reuse(None) - .expect("capture the checkout from disk") - .snapshot - ), - "a branch capture must index the bytes a checkout reads through each link" + live.iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + ["AGENTS.md", "src/real.rs"] ); + for regular in live { + assert!( + branch.contains(®ular), + "historical and live regular files must carry the same source digest" + ); + } let notes = captured .snapshot .files diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 88ca93c0a42..728e42d7b45 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -154,6 +154,11 @@ pub struct GitMetadataFingerprintV1 { /// branch switch changes this even when mtime resolution is too coarse to /// observe, so content is compared alongside mtimes. head_contents: Option, + // Include the registered child pin, bound checkout and resolved HEAD in the same + // read-admission proof as the parent. A failed probe cannot renew that proof. + submodule_refs: Option, + submodule_indexes: Option, + submodule_probe_failed: bool, } impl GitMetadataFingerprintV1 { @@ -176,19 +181,60 @@ impl GitMetadataFingerprintV1 { packed_refs: mtime(&common_dir.join("packed-refs")), refs_heads: refs_heads_signature(&common_dir.join("refs").join("heads")), head_contents: std::fs::read_to_string(&head_path).ok(), + ..Self::default() } } + pub(super) fn for_policy(project_root: &Path, include_submodules: bool) -> Self { + let mut fingerprint = Self::capture(project_root); + if include_submodules { + let probes = (|| { + let parent = tracedecay_runtime_core::git_open::open(project_root).ok()?; + let children = + tracedecay_code_index::source_walk::submodules::live_submodules(&parent) + .ok()?; + let mut refs = Vec::new(); + let mut indexes = Vec::new(); + for (path, binding) in children { + let child = tracedecay_runtime_core::git_open::open(&binding.git_dir).ok()?; + let head = child.head_id().ok()?.detach(); + refs.push(( + path.clone(), + binding.git_dir, + binding.pin.to_string(), + head.to_string(), + )); + indexes.push((path, mtime(&child.index_path()))); + } + Some(( + tracedecay_domain::canonical_sha256(&refs).ok()?, + tracedecay_domain::canonical_sha256(&indexes).ok()?, + )) + })(); + match probes { + Some((refs, indexes)) => { + fingerprint.submodule_refs = Some(refs.as_str().to_owned()); + fingerprint.submodule_indexes = Some(indexes.as_str().to_owned()); + } + None => fingerprint.submodule_probe_failed = true, + } + } + fingerprint + } + /// Whether the sampled metadata differs from a previously captured value. pub fn differs_from(&self, other: &GitMetadataFingerprintV1) -> bool { - self != other + self.submodule_probe_failed || other.submodule_probe_failed || self != other } /// Whether HEAD and every branch ref are unchanged. The index is excluded: /// opening the repository may refresh `.git/index` on its own, while a /// commit, checkout, or ref update always moves HEAD or a ref. pub fn same_refs_as(&self, other: &GitMetadataFingerprintV1) -> bool { - self.head == other.head + !self.submodule_probe_failed + && !other.submodule_probe_failed + && self.submodule_refs == other.submodule_refs + && self.head == other.head && self.packed_refs == other.packed_refs && self.refs_heads == other.refs_heads && self.head_contents == other.head_contents @@ -206,7 +252,7 @@ impl GitMetadataFingerprintV1 { time.and_then(|time| time.duration_since(UNIX_EPOCH).ok()) .map_or_else(|| "-".to_owned(), |elapsed| elapsed.as_nanos().to_string()) } - format!( + let mut signature = format!( "head={}|index={}|packed_refs={}|refs_heads={}|head_contents={}", render_time(self.head), render_time(self.index), @@ -216,7 +262,13 @@ impl GitMetadataFingerprintV1 { || "-".to_owned(), |value| super::sha256_hex(value.as_bytes()) ), - ) + ); + if let (Some(refs), Some(indexes)) = (&self.submodule_refs, &self.submodule_indexes) { + signature.push_str(&format!( + "|submodule_refs={refs}|submodule_indexes={indexes}" + )); + } + signature } } diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index 2819eb78153..23dacd76f56 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -260,7 +260,7 @@ impl CodeIndexWorktreeSchedulerV1 { ) -> Result { let source_control = self.reconcile_control(); let began_under = source_control.expected_epoch(); - let captured_under = super::identity::GitMetadataFingerprintV1::capture(&self.project_root); + let captured_under = self.freshness_fence().git_metadata(&self.project_root); let _workers = self.ensure_worker_plan()?; let _worker_memory = match self.reserve_worker_memory() { Ok(reservation) => reservation, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 5724543efff..4113a4ff273 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -567,6 +567,7 @@ pub(crate) struct SourceFreshnessFenceV1 { source_epoch: Arc, /// Held only by source sweeps, never across a scheduler pass. sweep_cache: Arc>, + include_submodules: bool, pending_source: Option>, } @@ -590,6 +591,10 @@ pub(super) struct SourceFreshnessFenceStateV1 { } impl SourceFreshnessFenceV1 { + pub(super) fn git_metadata(&self, root: &Path) -> identity::GitMetadataFingerprintV1 { + identity::GitMetadataFingerprintV1::for_policy(root, self.include_submodules) + } + fn unverified(source_epoch: Arc, path_policy: IndexPathPolicyV1) -> Self { Self { state: Arc::new(Mutex::new(SourceFreshnessFenceStateV1 { @@ -604,6 +609,7 @@ impl SourceFreshnessFenceV1 { })), last_reconciled_at_micros: Arc::new(AtomicI64::new(0)), source_epoch, + include_submodules: path_policy.includes_submodules(), sweep_cache: Arc::new(Mutex::new(SourceSweepCacheV1::new(path_policy))), pending_source: None, } @@ -723,7 +729,8 @@ impl SourceFreshnessFenceV1 { pub(super) fn verified_git_metadata_moved(&self, project_root: &Path) -> bool { let state = self.snapshot(); state.verified_against_source - && identity::GitMetadataFingerprintV1::capture(project_root) + && self + .git_metadata(project_root) .differs_from(&state.git_metadata) } @@ -760,7 +767,8 @@ impl SourceFreshnessFenceV1 { } state.verified_against_source && self.source_epoch.load(Ordering::Acquire) == state.reconciled_source_epoch - && !identity::GitMetadataFingerprintV1::capture(project_root) + && !self + .git_metadata(project_root) .differs_from(&state.git_metadata) } @@ -893,12 +901,13 @@ impl SourceFreshnessFenceV1 { &self, project_root: &Path, manifest: &SourceContentManifestV1, + candidate_paths: &BTreeSet, shutting_down: &AtomicBool, ) -> Result, CodeIndexSchedulerErrorV1> { self.sweep_cache .lock() .unwrap_or_else(PoisonError::into_inner) - .disputed_sources(project_root, manifest, shutting_down) + .disputed_sources(project_root, manifest, candidate_paths, shutting_down) } /// Sweep the current witness and report what the sweep had to read. @@ -916,7 +925,7 @@ impl SourceFreshnessFenceV1 { self.sweep_sources( project_root, &freshness.source_roster, - &identity::GitMetadataFingerprintV1::capture(project_root), + &self.git_metadata(project_root), &witness.content_manifest, shutting_down, ) @@ -937,7 +946,7 @@ impl SourceFreshnessFenceV1 { if !freshness.verified_against_source { return FreshnessProbeVerdictV1::Unverified; } - let git_metadata = identity::GitMetadataFingerprintV1::capture(project_root); + let git_metadata = self.git_metadata(project_root); if git_metadata.differs_from(&freshness.git_metadata) { return FreshnessProbeVerdictV1::Moved; } @@ -1336,6 +1345,7 @@ impl CodeIndexWorktreeSchedulerV1 { .with_shutdown_signal(Arc::clone(&shutting_down)) .with_decoded_content(byte_pool.decoded_content.clone()); let production_config = CodeIndexProductionConfigV1 { + include_submodules: path_policy.includes_submodules(), project_id: project_id.clone(), repository: repository_id.clone(), sanitizer_revision, @@ -1996,7 +2006,7 @@ impl CodeIndexWorktreeSchedulerV1 { { return Ok(None); } - let metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let metadata = self.freshness_fence.git_metadata(&self.project_root); if witness.git_metadata_signature != metadata.stable_signature() { return Ok(None); } @@ -2052,7 +2062,7 @@ impl CodeIndexWorktreeSchedulerV1 { if witness.generation_id != pointer.generation_id { return false; } - let metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let metadata = self.freshness_fence.git_metadata(&self.project_root); if witness.git_metadata_signature != metadata.stable_signature() { return false; } @@ -2234,7 +2244,7 @@ impl CodeIndexWorktreeSchedulerV1 { )); } self.identity = resolved; - let sampled_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let sampled_metadata = self.freshness_fence.git_metadata(&self.project_root); let Some(sampled_sweep) = self.worktree_stat_sweep().ok() else { return Ok(None); }; @@ -2343,7 +2353,7 @@ impl CodeIndexWorktreeSchedulerV1 { let captured = self .capture_authoritative_snapshot_without_active_generation_reuse(Some(&control)) .map_err(|error| prefer_resident_memory_refusal(&control, error))?; - let git_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let git_metadata = self.freshness_fence.git_metadata(&self.project_root); let stat_signature = self.worktree_stat_signature().ok(); let drained_hints = { let mut hints = self @@ -2927,7 +2937,7 @@ impl CodeIndexWorktreeSchedulerV1 { // bound to; a HEAD move under the same worktree is allowed and simply // records a new source revision, so the served generation is never // mis-attributed across identities. - let captured_under = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let captured_under = self.freshness_fence.git_metadata(&self.project_root); let resolved = identity::IndexingIdentityV1::resolve(&self.project_root) .map_err(|error| CodeIndexSchedulerErrorV1::Identity(error.to_string()))?; if !resolved.authorizes_reuse_of(&self.identity) { @@ -3199,7 +3209,7 @@ impl CodeIndexWorktreeSchedulerV1 { source_epoch: u64, captured_under: &identity::GitMetadataFingerprintV1, ) { - let sampled = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let sampled = self.freshness_fence.git_metadata(&self.project_root); let metadata = if sampled.same_refs_as(captured_under) { sampled } else { @@ -3318,7 +3328,9 @@ impl CodeIndexWorktreeSchedulerV1 { } let freshness = self.freshness_fence.snapshot(); if !freshness.verified_against_source - || identity::GitMetadataFingerprintV1::capture(&self.project_root) + || self + .freshness_fence + .git_metadata(&self.project_root) .differs_from(&freshness.git_metadata) || freshness.last_reconciled_at.elapsed() >= self.policy.staleness_threshold { @@ -3338,7 +3350,9 @@ impl CodeIndexWorktreeSchedulerV1 { } let freshness = self.freshness_fence.snapshot(); if !freshness.verified_against_source - || identity::GitMetadataFingerprintV1::capture(&self.project_root) + || self + .freshness_fence + .git_metadata(&self.project_root) .differs_from(&freshness.git_metadata) { self.request_background_reconcile(); @@ -3364,7 +3378,7 @@ impl CodeIndexWorktreeSchedulerV1 { .sweep_sources( &self.project_root, &self.ignored_source_admissions, - &identity::GitMetadataFingerprintV1::capture(&self.project_root), + &self.freshness_fence.git_metadata(&self.project_root), manifest, &self.shutting_down, ) @@ -3376,7 +3390,7 @@ impl CodeIndexWorktreeSchedulerV1 { fn source_witness_matches_worktree(&self, freshness: &SourceFreshnessFenceStateV1) -> bool { self.freshness_fence.source_witness_matches( freshness, - &identity::GitMetadataFingerprintV1::capture(&self.project_root), + &self.freshness_fence.git_metadata(&self.project_root), &self.project_root, &self.shutting_down, ) @@ -3432,7 +3446,7 @@ impl CodeIndexWorktreeSchedulerV1 { // Sample after the walk. `git_open::open` inside the digest comparison can // move index metadata; storing the post-walk sample is what keeps the // next probe from calling that side effect a new generation. - let git_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let git_metadata = self.freshness_fence.git_metadata(&self.project_root); self.freshness_fence.rebind_admission_clock(git_metadata); self.freshness_fence .source_currency_witness_for(generation_id, snapshot_content_identity) @@ -3450,7 +3464,11 @@ impl CodeIndexWorktreeSchedulerV1 { fn worktree_stat_sweep( &self, ) -> Result { - freshness_witness::worktree_stat_sweep(&self.project_root, &self.ignored_source_admissions) + freshness_witness::worktree_stat_sweep( + &self.project_root, + &self.ignored_source_admissions, + self.path_policy.includes_submodules(), + ) } /// Deliver debounced hook hints (exact touched paths) into the incremental @@ -3495,7 +3513,9 @@ impl CodeIndexWorktreeSchedulerV1 { // suppressing cadence on open-time clocks is not. return Ok(Some(self.reconcile_now()?)); } - let git_changed = identity::GitMetadataFingerprintV1::capture(&self.project_root) + let git_changed = self + .freshness_fence + .git_metadata(&self.project_root) .differs_from(&freshness.git_metadata); if git_changed { // Tier 1: a git-mediated mutation is authoritative evidence; reconcile. @@ -3957,7 +3977,28 @@ impl CodeIndexWorktreeSchedulerV1 { .ignored_source_admissions .iter() .any(|admission| admission.logical_path == logical_path); - self.capture_admitted_candidate(registry, logical_path, control, explicitly_admitted) + let child_root = if self.path_policy.includes_submodules() && !explicitly_admitted { + let repository = tracedecay_runtime_core::git_open::open(&self.project_root) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let classification = + classification::WorktreeChangeClassificationV1::for_policy(&repository, true) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + if classification.beneath_gitlink(Path::new(logical_path)) + && classification.child_root(logical_path).is_none() + { + return Ok(CapturedFileOutcomeV1::Absent); + } + classification.child_root(logical_path).map(str::to_owned) + } else { + None + }; + self.capture_admitted_candidate( + registry, + logical_path, + control, + explicitly_admitted, + child_root.as_deref(), + ) } fn ignored_admission_paths(&self) -> BTreeSet<&str> { @@ -3973,20 +4014,41 @@ impl CodeIndexWorktreeSchedulerV1 { logical_path: &str, control: Option<&dyn CodeIndexExecutionControlV1>, explicitly_admitted: bool, + child_root: Option<&str>, ) -> Result { // Explicit ignored-source admission cannot cross a nested checkout // any more than an ordinary Git candidate or interactive source walk. - let relative = std::path::Path::new(logical_path); + let relative = Path::new(logical_path); + let absolute = self.project_root.join(relative); + let (source_root, source_relative) = match child_root { + Some(prefix) if !explicitly_admitted => { + if !git_tree_capture::is_source_path(registry, logical_path.as_bytes()) { + return Ok(CapturedFileOutcomeV1::Absent); + } + let root = self.project_root.join(prefix); + if root.canonicalize().ok().as_ref() != Some(&root) + || !absolute + .symlink_metadata() + .is_ok_and(|metadata| metadata.is_file()) + { + return Ok(CapturedFileOutcomeV1::Absent); + } + let relative = relative.strip_prefix(prefix).map_err(|_| { + CodeIndexSchedulerErrorV1::Git("invalid bound submodule path".into()) + })?; + (root, relative) + } + _ => (self.project_root.clone(), relative), + }; if tracedecay_code_index::source_walk::has_nested_repository_ancestor( - &self.project_root, - relative, + &source_root, + source_relative, )? || tracedecay_code_index::source_walk::is_source_symlink( - &self.project_root, - relative, + &source_root, + source_relative, )? { return Ok(CapturedFileOutcomeV1::Absent); } - let absolute = self.project_root.join(logical_path); match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), Ok(_) => {} @@ -4048,8 +4110,11 @@ impl CodeIndexWorktreeSchedulerV1 { // Classify committed/staged/unstaged/untracked/deleted/renamed paths // truthfully from gix. Deletions drop out of the present candidate set; // their tombstones flow through `changed_paths`. - let classification = classification::WorktreeChangeClassificationV1::classify(&repository) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let classification = classification::WorktreeChangeClassificationV1::for_policy( + &repository, + self.path_policy.includes_submodules(), + ) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; if self.shutting_down.load(Ordering::Acquire) { return Err(cancelled_code_index_reconcile()); } @@ -4126,6 +4191,7 @@ impl CodeIndexWorktreeSchedulerV1 { // assume-unchanged. A sealed row is reused only while its bytes still // carry its digest; every row they disprove beyond the visible commit // and status deltas is a worktree change stat could not see. + let mut candidate_paths = classification.candidate_paths(); let stat_blind_paths = match reusable_active.as_ref() { Some(active) => { let visible = classification.changed_paths(); @@ -4133,6 +4199,7 @@ impl CodeIndexWorktreeSchedulerV1 { .disputed_sources( &self.project_root, &SourceContentManifestV1::for_snapshot(active.snapshot()), + &candidate_paths, &self.shutting_down, )? .into_iter() @@ -4193,7 +4260,6 @@ impl CodeIndexWorktreeSchedulerV1 { let source_revision = (self.ignored_source_admissions.is_empty() && !has_changes) .then(|| self.identity.head_commit().cloned()) .flatten(); - let mut candidate_paths = classification.candidate_paths(); let mut changed_paths = classification.changed_paths(); changed_paths.extend(tree_delta); changed_paths.extend(stat_blind_paths); @@ -4274,6 +4340,7 @@ impl CodeIndexWorktreeSchedulerV1 { logical_path, control, admitted_paths.contains(logical_path.as_str()), + classification.child_root(logical_path), ) }) }) @@ -4364,6 +4431,7 @@ impl CodeIndexWorktreeSchedulerV1 { snapshot_content_identity(&files, &sanitization_receipts, &omitted_sources); let captured = CapturedSnapshotV1 { repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: self.path_policy.includes_submodules(), tree: self.identity.head_tree().cloned(), dirty, }, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index 06cf0bd3f33..f90c046bd00 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -191,3 +191,216 @@ async fn hook_hints_for_excluded_paths_queue_no_work() { ); registry.shutdown().await; } + +#[test] +fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() { + use super::{git_stdout, run_git_in}; + use crate::code_index_scheduler::{ + branch_generations::BranchGenerationReadControlV1, git_tree_capture::ExactGitTreeSourceV1, + }; + use tracedecay_code_index::grep_search::{GrepSearchQuery, search_tree_with_cancel}; + use tracedecay_domain::{CommitId, RefId, RelationEdgeKindV1, RepositoryDirtyStateV1, TreeId}; + + let parent = GitFixture::new(&[( + "dev/launch.ts", + "import { feature } from '../app/lib';\nexport function launch() { return feature(); }\n", + )]); + let child = GitFixture::new(&[ + ( + "lib.ts", + "export function feature() { return 'pinned_symbol'; }\n", + ), + ("gone.ts", "export function gone() {}\n"), + ("media.bin", "local binary fixture"), + (".gitignore", "*.private.ts\n"), + ]); + run_git_in( + parent.path(), + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + child.path().to_str().unwrap(), + "app", + ], + ); + run_git_in(parent.path(), &["commit", "-qm", "bind application"]); + run_git_in(parent.path(), &["branch", "pinned"]); + let pinned = ExactGitTreeSourceV1 { + reference: RefId::new("refs/heads/pinned").unwrap(), + revision: CommitId::new(git_stdout(parent.path(), &["rev-parse", "HEAD"])).unwrap(), + tree: TreeId::new(git_stdout(parent.path(), &["rev-parse", "HEAD^{tree}"])).unwrap(), + }; + let store = TempDir::new().unwrap(); + let enabled = policy(&[], &[]).with_submodules(true); + let grep = |needle: &str, policy: &IndexPathPolicyV1| { + search_tree_with_cancel( + parent.path(), + &GrepSearchQuery { + pattern: needle.to_owned(), + fixed_strings: true, + case_sensitive: true, + path_glob: Some("app/**".into()), + scope_prefix: None, + context_lines: 0, + max_results: 8, + }, + policy, + || false, + ) + .unwrap() + .hits + .into_iter() + .map(|hit| hit.file.to_string()) + .collect::>() + }; + let mut off = start(&parent, store.path(), policy(&[], &[])); + off.reconcile_now().unwrap(); + assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); + assert!(grep("pinned_symbol", &policy(&[], &[])).is_empty()); + drop(off); + + let mut on = start(&parent, store.path(), enabled.clone()); + on.reconcile_now().unwrap(); + assert_eq!( + indexed_paths(&on), + ["app/gone.ts", "app/lib.ts", "dev/launch.ts"] + ); + assert_eq!(grep("pinned_symbol", &enabled), ["app/lib.ts"]); + let assert_edge = + |generation: &tracedecay_code_index::production::CodeIndexPublishedGenerationV1| { + let symbol = |name: &str| { + generation + .symbols() + .symbols + .iter() + .find(|item| item.simple_name == name) + .unwrap() + .occurrence + .clone() + }; + assert!( + generation + .edges() + .iter() + .any(|edge| edge.kind == RelationEdgeKindV1::Calls + && edge.from_occurrence == symbol("launch") + && edge.to_occurrence == symbol("feature")) + ); + }; + assert_edge(on.latest_complete().unwrap().generation()); + let checkout = parent.path().join("app"); + std::fs::write( + checkout.join("ignored.private.ts"), + "const private_marker = 1;", + ) + .unwrap(); + assert!(grep("private_marker", &enabled).is_empty()); + std::fs::write( + checkout.join("lib.ts"), + "export function feature() { return 'successor_symbol'; }\n", + ) + .unwrap(); + std::fs::remove_file(checkout.join("gone.ts")).unwrap(); + run_git_in(&checkout, &["add", "-A"]); + run_git_in( + &checkout, + &["commit", "-qm", "move child HEAD independently"], + ); + std::fs::write( + checkout.join("pending.ts"), + "export function pending() {}\n", + ) + .unwrap(); + assert!(on.ensure_fresh_for_query().unwrap().is_some()); + assert_eq!( + indexed_paths(&on), + ["app/lib.ts", "app/pending.ts", "dev/launch.ts"] + ); + assert_eq!( + on.latest_complete() + .unwrap() + .generation() + .repository_parse_identity() + .dirty, + RepositoryDirtyStateV1::Dirty + ); + assert_eq!(grep("successor_symbol", &enabled), ["app/lib.ts"]); + run_git_in(parent.path(), &["submodule", "deinit", "-f", "app"]); + assert!(!checkout.join(".git").exists()); + let historical = on + .publish_exact_git_tree_generation( + &pinned, + &BranchGenerationReadControlV1 { + deadline: None, + cancellation: None, + }, + ) + .unwrap(); + let historical = historical.generation(); + assert_edge(historical); + assert!( + !historical + .snapshot() + .files + .iter() + .any(|file| file.logical_path == "app/media.bin") + ); + assert!( + historical + .snapshot() + .files + .iter() + .any(|file| file.logical_path == "app/gone.ts") + ); + assert!( + !historical + .snapshot() + .files + .iter() + .any(|file| file.logical_path == "app/pending.ts") + ); + let pinned_file = historical + .snapshot() + .files + .iter() + .find(|file| file.logical_path == "app/lib.ts") + .unwrap(); + assert_eq!( + pinned_file.content_digest, + tracedecay_code_index::chunks::content_digest( + b"export function feature() { return 'pinned_symbol'; }\n" + ) + ); + drop(on); + let mut off = start(&parent, store.path(), policy(&[], &[])); + off.reconcile_now().unwrap(); + assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); +} + +#[test] +fn parent_filtered_source_does_not_require_submodule_registration_when_disabled() { + use super::run_git_in; + use tracedecay_code_index::chunks::content_digest; + use tracedecay_domain::LanguageId; + + let source = "pub fn parent() {}\n"; + let parent = GitFixture::new(&[ + ("lib.rs", source), + (".gitmodules", "[submodule \"child\"]\npath = child\n"), + ]); + run_git_in(parent.path(), &["config", "core.autocrlf", "true"]); + std::fs::write(parent.path().join("lib.rs"), source.replace('\n', "\r\n")).unwrap(); + std::fs::write(parent.path().join(".gitmodules"), "[unfinished").unwrap(); + let sealed = super::super::sealed_source_bytes( + parent.path(), + "lib.rs", + &LanguageId::new("rust").unwrap(), + false, + &content_digest(source.as_bytes()), + ) + .unwrap(); + assert_eq!(sealed.as_deref(), Some(source.as_bytes())); +} diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs index 35ca13c8c1d..5aa18b0df2a 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs @@ -825,13 +825,11 @@ fn unfollowable_committed_source_links_are_absent_from_disk_capture() { ); } -/// Git stores a committed source symlink as its target path, while the -/// worktree reads through it. Sealing the link text from HEAD's tree left the -/// first post-publication sweep disproving an unchanged checkout, so every -/// mount rebuilt once and served the rebuild as dirty. +/// A readable live symlink stays outside the worktree roster. The remaining +/// files can still establish a current generation without naming the full HEAD tree. #[cfg(unix)] #[test] -fn committed_source_symlink_publishes_a_current_clean_generation() { +fn committed_source_symlink_publishes_current_regular_files_only() { let fixture = GitFixture::new(&[ ("src/lib.rs", "pub mod pricing;\n"), ("src/pricing.rs", "pub fn total() -> u64 { 1 }\n"), @@ -851,16 +849,16 @@ fn committed_source_symlink_publishes_a_current_clean_generation() { !scheduler.request_fresh_now_background(), "the sweep after publication must find the unchanged checkout current" ); + let latest = scheduler.latest_complete().expect("published generation"); + let snapshot = latest.generation.snapshot(); + assert!(snapshot.source_revision.is_none()); assert_eq!( - scheduler - .latest_complete() - .expect("published generation") - .generation - .snapshot() - .source_revision - .as_ref(), - scheduler.identity.head_commit(), - "a clean checkout's generation must carry HEAD's revision" + snapshot + .files + .iter() + .map(|file| file.logical_path.as_str()) + .collect::>(), + ["src/lib.rs", "src/pricing.rs"] ); } diff --git a/crates/tracedecay-code-index/Cargo.toml b/crates/tracedecay-code-index/Cargo.toml index 5404891c52a..cba8706cb2c 100644 --- a/crates/tracedecay-code-index/Cargo.toml +++ b/crates/tracedecay-code-index/Cargo.toml @@ -66,6 +66,7 @@ lang-lean = ["tracedecay-code-extraction/lang-lean"] [dependencies] ast-grep-core = "0.50" flate2.workspace = true +gix.workspace = true miniz_oxide.workspace = true base64.workspace = true hex.workspace = true diff --git a/crates/tracedecay-code-index/benches/partitioned_codec.rs b/crates/tracedecay-code-index/benches/partitioned_codec.rs index 12a962fddf0..c9330ca623b 100644 --- a/crates/tracedecay-code-index/benches/partitioned_codec.rs +++ b/crates/tracedecay-code-index/benches/partitioned_codec.rs @@ -286,6 +286,7 @@ fn build_generation( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -315,6 +316,7 @@ fn build_generation( fn production_config() -> Result> { Ok(CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.partitioned-codec")?, repository: id::("repository.partitioned-codec")?, sanitizer_revision: id::("sanitizer.partitioned-codec.v1")?, diff --git a/crates/tracedecay-code-index/benches/sealed_storage.rs b/crates/tracedecay-code-index/benches/sealed_storage.rs index f3cc2da231d..7e927b88c23 100644 --- a/crates/tracedecay-code-index/benches/sealed_storage.rs +++ b/crates/tracedecay-code-index/benches/sealed_storage.rs @@ -468,6 +468,7 @@ fn request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(id::(tree)?), dirty: RepositoryDirtyStateV1::Clean, }, @@ -482,6 +483,7 @@ fn request( fn config() -> Result> { Ok(CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.sealed-storage")?, repository: id::("repository.sealed-storage")?, sanitizer_revision: id::("sanitizer.sealed-storage.v1")?, diff --git a/crates/tracedecay-code-index/src/production/ignored_sources.rs b/crates/tracedecay-code-index/src/production/ignored_sources.rs index 2a103a5303c..beafc93a382 100644 --- a/crates/tracedecay-code-index/src/production/ignored_sources.rs +++ b/crates/tracedecay-code-index/src/production/ignored_sources.rs @@ -53,6 +53,9 @@ pub struct CodeIndexBuildRequestV1 { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct CodeIndexRepositoryParseIdentityV1 { + /// Older sealed generations used the default parent-only roster. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub include_submodules: bool, pub tree: Option, pub dirty: RepositoryDirtyStateV1, } diff --git a/crates/tracedecay-code-index/src/production/lexical_page_source.rs b/crates/tracedecay-code-index/src/production/lexical_page_source.rs index 76497c639a9..df469bda139 100644 --- a/crates/tracedecay-code-index/src/production/lexical_page_source.rs +++ b/crates/tracedecay-code-index/src/production/lexical_page_source.rs @@ -1332,6 +1332,7 @@ impl VerifiedSealedTextGenerationMetadataV1 { &self.snapshot, config, ); + compatibility.observe_submodule_policy(self.repository_parse_identity(), config); self.chunk_policy.observe(config, &mut compatibility); compatibility } diff --git a/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs b/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs index 6ce31293960..6b2276f9b9e 100644 --- a/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs +++ b/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs @@ -611,6 +611,7 @@ fn fixture_for_source_files( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -624,6 +625,7 @@ fn fixture_for_source_files( }; let mut owner = CodeIndexProductionOwnerV1::new( CodeIndexProductionConfigV1 { + include_submodules: false, project_id: ProjectId::new("project.lexical-page-batch").expect("fixture project ID"), repository: RepositoryId::new("repository.lexical-page-batch") .expect("fixture repository ID"), diff --git a/crates/tracedecay-code-index/src/production/mod.rs b/crates/tracedecay-code-index/src/production/mod.rs index b4b48db5acb..af485652be0 100644 --- a/crates/tracedecay-code-index/src/production/mod.rs +++ b/crates/tracedecay-code-index/src/production/mod.rs @@ -151,6 +151,7 @@ pub const DAEMON_CODE_INDEX_CHUNKER_REVISION: &str = "chunker.daemon.v5"; /// Immutable configuration retained by one production index owner. #[derive(Clone, Debug)] pub struct CodeIndexProductionConfigV1 { + pub include_submodules: bool, pub project_id: ProjectId, pub repository: RepositoryId, pub sanitizer_revision: SanitizerRevision, @@ -192,6 +193,7 @@ pub enum CodeIndexGenerationIncompatibilityV1 { SanitizerRevision, PolicyRevision, MixedPolicyRevisions, + SubmodulePolicy, ChunkerRevision, PrivacyDomain, PrivacyKeyEpoch, @@ -203,6 +205,7 @@ impl CodeIndexGenerationIncompatibilityV1 { Self::Project => "project", Self::LanguageRevisions => "language_revisions", Self::SanitizerRevision => "sanitizer_revision", + Self::SubmodulePolicy => "submodule_policy", Self::PolicyRevision => "policy_revision", Self::MixedPolicyRevisions => "mixed_policy_revisions", Self::ChunkerRevision => "chunker_revision", @@ -226,6 +229,17 @@ pub struct CodeIndexGenerationCompatibilityV1 { } impl CodeIndexGenerationCompatibilityV1 { + fn observe_submodule_policy( + &mut self, + identity: &CodeIndexRepositoryParseIdentityV1, + config: &CodeIndexProductionConfigV1, + ) { + if identity.include_submodules != config.include_submodules { + self.incompatibilities + .insert(CodeIndexGenerationIncompatibilityV1::SubmodulePolicy); + } + } + fn for_metadata( manifest: &CodeGenerationManifestV1, snapshot: &SanitizedCodeSnapshotV1, @@ -1456,6 +1470,7 @@ impl CodeIndexPublishedGenerationV1 { &self.snapshot, config, ); + compatibility.observe_submodule_policy(self.repository_parse_identity(), config); self.chunk_policy_summary() .observe(config, &mut compatibility); compatibility @@ -2364,6 +2379,11 @@ where crate::observe::record_generation_state("building"); crate::observe::record_rebuild_state("unknown"); lexical_page_source::checkpoint(control)?; + if request.repository_parse_identity.include_submodules != self.config.include_submodules { + return Err(CodeIndexProductionErrorV1::Contract( + "snapshot submodule policy differs from owner".to_owned(), + )); + } let ignored_source_roster = IgnoredSourceRosterV1::admit( &request.snapshot, &request.repository_parse_identity, diff --git a/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs b/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs index 01eb1dcb764..66d8af7915c 100644 --- a/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs +++ b/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs @@ -100,6 +100,7 @@ pub(super) fn request(files: &[(&str, &str, &str)], sealed_at: i64) -> CodeIndex invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: tracedecay_domain::RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/src/production/worker_tests.rs b/crates/tracedecay-code-index/src/production/worker_tests.rs index 7f210388b60..71b9eedfccc 100644 --- a/crates/tracedecay-code-index/src/production/worker_tests.rs +++ b/crates/tracedecay-code-index/src/production/worker_tests.rs @@ -39,6 +39,7 @@ where pub(super) fn worker_config() -> CodeIndexProductionConfigV1 { CodeIndexProductionConfigV1 { + include_submodules: false, project_id: worker_id("project.worker"), repository: worker_id("repository.worker"), sanitizer_revision: worker_id::("sanitizer.v1"), @@ -87,6 +88,7 @@ pub(super) fn worker_request_with_source( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: tracedecay_domain::RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index a72a87b77ea..78c682e2971 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -9,6 +9,9 @@ //! walk is therefore public rather than crate-private, the audit in the root //! crate reuses this policy instead of restating it. +pub mod classification; +pub mod submodules; + use std::collections::BTreeSet; use std::path::{Component, Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; @@ -17,6 +20,8 @@ use ignore::overrides::{Override, OverrideBuilder}; use ignore::{Walk, WalkBuilder}; use tracedecay_domain::{IndexPathPolicyV1, forward_slash_path}; +pub type SourceWalk = std::iter::Flatten>; + #[derive(Debug)] pub struct SourceWalkError { pub glob: String, @@ -25,6 +30,7 @@ pub struct SourceWalkError { /// Conservative directory reachability for a positive override. The override /// matcher remains authoritative for files, including directory inheritance. +#[derive(Clone)] struct PathGlobScope { literal_prefix: PathBuf, may_match_descendants: bool, @@ -340,7 +346,7 @@ pub fn source_walk( project_root: &Path, path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, -) -> Result { +) -> Result { source_walk_inner(project_root, path_glob, path_policy, None) } @@ -351,7 +357,7 @@ pub(crate) fn source_walk_with_diagnostics( project_root: &Path, path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, -) -> Result<(Walk, SourceWalkDiagnostics), SourceWalkError> { +) -> Result<(SourceWalk, SourceWalkDiagnostics), SourceWalkError> { let diagnostics = SourceWalkDiagnostics::default(); let walk = source_walk_inner( project_root, @@ -367,7 +373,7 @@ fn source_walk_inner( path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, diagnostics: Option, -) -> Result { +) -> Result { let overrides = build_overrides(project_root, path_glob)?; let has_positive_override = overrides .as_ref() @@ -376,88 +382,163 @@ fn source_walk_inner( let path_glob_scope = path_glob.and_then(PathGlobScope::from_path_glob); let filter_root = project_root.to_path_buf(); let path_policy = path_policy.clone(); - let parent_gitlinks = parent_gitlink_paths(project_root)?; - - let mut builder = WalkBuilder::new(project_root); - builder - .follow_links(false) - .hidden(false) - .git_ignore(true) - .git_global(true) - .git_exclude(true) - .add_custom_ignore_filename(".gitignore") - .filter_entry(move |entry| { - if entry.depth() == 0 { - return true; - } - let segment = entry.file_name().to_string_lossy(); - if segment == ".git" || segment == ".tracedecay" { - return false; - } - let Ok(relative) = entry.path().strip_prefix(&filter_root) else { - return false; - }; - let relative = forward_slash_path(relative); - let is_dir = entry.file_type().is_some_and(|kind| kind.is_dir()); - if is_dir - && path_glob_scope + let scoped_policy = { + let root = filter_root.clone(); + let scope = path_glob_scope.clone(); + move |path: &Path, is_dir: bool| { + has_positive_override + && (generated_dir_overrides .as_ref() - .is_some_and(|scope| !scope.may_contain_match(&filter_root, entry.path())) - { - return false; + .is_some_and(|overrides| overrides.matched(path, is_dir).is_whitelist()) + || (is_dir + && scope + .as_ref() + .is_some_and(|scope| scope.allows(&root, path)))) + } + }; + let mut parent_gitlinks = parent_gitlink_paths(project_root)?; + let mut authenticated = BTreeSet::new(); + let mut child_files = BTreeSet::new(); + if path_policy.includes_submodules() && !parent_gitlinks.is_empty() { + let parent = tracedecay_runtime_core::git_open::open(project_root).map_err(|error| { + SourceWalkError { + glob: String::new(), + message: error.to_string(), } - // An entry the glob names, or a directory it must pass through, - // is judged without the generated-directory defaults: the scope - // is the operator asking for that noise. Every other exclusion - // still applies; only `index.include.v1` lifts those. - let explicitly_requested = has_positive_override - && (generated_dir_overrides.as_ref().is_some_and(|overrides| { - overrides.matched(entry.path(), is_dir).is_whitelist() - }) || (is_dir - && path_glob_scope - .as_ref() - .is_some_and(|scope| scope.allows(&filter_root, entry.path())))); - let path_policy = if explicitly_requested { + })?; + for (path, child) in submodules::live_submodules(&parent)? { + let absolute = project_root.join(&path); + let policy = if scoped_policy(&absolute, true) { path_policy.without_generated_dir_defaults() } else { &path_policy }; - if !is_dir { - return !path_policy.excludes(&relative); + if policy.excludes_directory(&path) + || path_glob_scope + .as_ref() + .is_some_and(|scope| !scope.may_contain_match(project_root, &absolute)) + { + continue; } - // Excluded directories are outside this audit, not missing source - // caused by the repository boundary. Do not probe or name them. - if path_policy.excludes_directory(&relative) { - return false; + let repo = + tracedecay_runtime_core::git_open::open(&child.git_dir).map_err(|error| { + SourceWalkError { + glob: String::new(), + message: error.to_string(), + } + })?; + let inventory = classification::WorktreeChangeClassificationV1::classify(&repo) + .map_err(|error| SourceWalkError { + glob: String::new(), + message: error.to_string(), + })?; + let files = inventory + .regular_child_files(&child.worktree) + .map_err(|error| SourceWalkError { + glob: String::new(), + message: error.to_string(), + })?; + child_files.extend(files.into_iter().map(|file| format!("{path}/{file}"))); + // Only direct registrations belong to this parent. + for nested in inventory.gitlink_paths() { + parent_gitlinks.insert(format!("{path}/{nested}")); + } + authenticated.insert(path); + } + } + + let roots = authenticated + .iter() + .map(|path| project_root.join(path)) + .collect::>(); + let child_dirs = child_files + .iter() + .flat_map(|path| Path::new(path).ancestors().skip(1)) + .map(Path::to_owned) + .collect::>(); + + let accept = Arc::new(move |entry: &ignore::DirEntry, in_child: bool| { + let segment = entry.file_name().to_string_lossy(); + if segment == ".git" || segment == ".tracedecay" { + return false; + } + let Ok(relative) = entry.path().strip_prefix(&filter_root) else { + return false; + }; + let relative = forward_slash_path(relative); + let is_dir = entry.file_type().is_some_and(|kind| kind.is_dir()); + if is_dir + && path_glob_scope + .as_ref() + .is_some_and(|scope| !scope.may_contain_match(&filter_root, entry.path())) + { + return false; + } + // An entry the glob names, or a directory it must pass through, + // is judged without the generated-directory defaults: the scope + // is the operator asking for that noise. Every other exclusion + // still applies; only `index.include.v1` lifts those. + let path_policy = if scoped_policy(entry.path(), is_dir) { + path_policy.without_generated_dir_defaults() + } else { + &path_policy + }; + if !is_dir { + return !path_policy.excludes(&relative) + && (!in_child || child_files.contains(&relative)); + } + // Excluded directories are outside this audit, not missing source + // caused by the repository boundary. Do not probe or name them. + if path_policy.excludes_directory(&relative) { + return false; + } + // Only an independently bound direct gitlink may cross a checkout boundary. + if authenticated.contains(&relative) { + return in_child; + } + if parent_gitlinks.contains(&relative) { + if let Some(diagnostics) = &diagnostics { + diagnostics.record_skipped_nested_repository(relative); } - // Includes select paths, not repository authority. A parent - // gitlink does not capture the child's files or prove their - // freshness, so even registered submodules remain separate - // sources. Linked worktrees and nested clones share this boundary. - // The project root is deliberately exempt at depth zero above. - if parent_gitlinks.contains(&relative) { + return false; + } + match nested_repository_marker(entry.path()) { + Ok(true) => { if let Some(diagnostics) = &diagnostics { diagnostics.record_skipped_nested_repository(relative); } - return false; + false } - match nested_repository_marker(entry.path()) { - Ok(true) => { - if let Some(diagnostics) = &diagnostics { - diagnostics.record_skipped_nested_repository(relative); - } - false - } - Ok(false) => true, - // The walk cannot prove that an unreadable Git marker is - // absent. Never admit files across that boundary. - Err(_) => false, + Ok(false) => !in_child || child_dirs.contains(Path::new(&relative)), + // The walk cannot prove that an unreadable Git marker is + // absent. Never admit files across that boundary. + Err(_) => false, + } + }); + let mut builders = vec![(WalkBuilder::new(project_root), false)]; + builders.extend(roots.into_iter().map(|root| (WalkBuilder::new(root), true))); + let walks = builders + .into_iter() + .map(|(mut builder, in_child)| { + // The child Git roster already applied that checkout's ignores, including + // tracked exceptions. Running parent ignore rules over it would hide those files. + if in_child { + builder.standard_filters(false); + } else { + builder.add_custom_ignore_filename(".gitignore"); } - }); - if let Some(overrides) = overrides { - builder.overrides(overrides); - } - Ok(builder.build()) + let accept = Arc::clone(&accept); + builder + .follow_links(false) + .hidden(false) + .filter_entry(move |entry| accept(entry, in_child)); + if let Some(overrides) = &overrides { + builder.overrides(overrides.clone()); + } + builder.build() + }) + .collect::>(); + Ok(walks.into_iter().flatten()) } /// Project-relative path rendered with forward slashes. @@ -503,7 +584,8 @@ mod tests { use tracedecay_domain::IndexPathPolicyV1; use super::{ - has_nested_repository_ancestor, is_source_symlink, source_walk, source_walk_with_diagnostics, + has_nested_repository_ancestor, is_source_symlink, source_walk, + source_walk_with_diagnostics, }; #[test] @@ -808,11 +890,45 @@ mod tests { ); } + let enabled = IndexPathPolicyV1::new(vec![], vec![]) + .unwrap() + .with_submodules(true); + fixture_git( + &child, + &[ + "worktree", + "add", + "-q", + "--detach", + root.join("other-worktree").to_str().unwrap(), + "HEAD", + ], + ); + fixture_git( + &root, + &["clone", "-q", child.to_str().unwrap(), "other-clone"], + ); + let files = walked_files(&root, &enabled); + assert!(files.contains(&PathBuf::from("app/src/feature.rs"))); + assert!(files.iter().all(|path|!path.starts_with("other-worktree") && !path.starts_with("other-clone"))); + fixture_git( + &child, + &[ + "worktree", + "remove", + root.join("other-worktree").to_str().unwrap(), + ], + ); + fs::remove_dir_all(root.join("other-clone")).unwrap(); + fs::remove_file(root.join(".gitmodules")).unwrap(); + assert!(!walked_files(&root, &enabled).contains(&PathBuf::from("app/src/feature.rs"))); + fixture_git(&root, &["checkout", "--", ".gitmodules"]); // A missing child .git marker must not turn the parent's stage-0 // gitlink into ordinary live source under an explicit include. fs::remove_file(root.join("app/.git")).expect("remove child gitdir marker"); assert!(root.join("app/src/feature.rs").is_file()); let policy = IndexPathPolicyV1::new(vec![], vec!["app/**".into()]).unwrap(); + assert!(!walked_files(&root, &enabled).contains(&PathBuf::from("app/src/feature.rs"))); let (walk, diagnostics) = source_walk_with_diagnostics(&root, None, &policy).expect("source walk"); let files = walk diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index/src/source_walk/classification.rs similarity index 85% rename from crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs rename to crates/tracedecay-code-index/src/source_walk/classification.rs index 0550c4a843c..931bdf5329e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index/src/source_walk/classification.rs @@ -17,10 +17,11 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; -use gix::bstr::ByteSlice; -use tracedecay_code_index::source_walk::{ +use super::{ has_nested_repository_ancestor, is_source_symlink, parent_gitlink_paths, + submodules::live_submodules, }; +use gix::bstr::ByteSlice; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; /// Failure to classify worktree changes through gix. @@ -100,6 +101,9 @@ pub struct WorktreeChangeClassificationV1 { /// or symlinked ancestor. These are tombstones, not indexable candidates. nested_repository_exclusions: BTreeSet, parent_gitlinks: BTreeSet, + /// Direct parent-relative child paths admitted from that checkout's own Git roster. + child_sources: BTreeMap, + child_revision_changed: bool, } impl WorktreeChangeClassificationV1 { @@ -266,9 +270,93 @@ impl WorktreeChangeClassificationV1 { Ok(classification) } - /// Explicitly enrolled files must respect the same Git boundary snapshot - /// as ordinary candidates, including a gitlink staged for removal. - pub(super) fn beneath_gitlink(&self, path: &Path) -> bool { + pub fn for_policy( + repository: &gix::Repository, + include_submodules: bool, + ) -> Result { + let mut parent = Self::classify(repository)?; + if !include_submodules || parent.parent_gitlinks.is_empty() { + return Ok(parent); + } + let children = live_submodules(repository) + .map_err(|error| ClassificationErrorV1::Git(error.message))?; + // A missing checkout must not let the clean parent-tree shortcut import children. + parent.child_revision_changed = parent + .parent_gitlinks + .iter() + .any(|path| !children.contains_key(path)); + for (prefix, binding) in children { + let child = tracedecay_runtime_core::git_open::open(&binding.git_dir) + .map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; + let child_head = child + .head_id() + .map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; + parent.child_revision_changed |= child_head.detach() != binding.pin; + let local = Self::classify(&child)?; + for path in local.regular_child_files(&binding.worktree)? { + let logical = format!("{prefix}/{path}"); + parent.committed_baseline.insert(logical.clone()); + parent.child_sources.insert(logical, prefix.clone()); + } + for change in local.changes { + let path = format!("{prefix}/{}", change.path); + if !change.class.presents_content() || parent.child_sources.contains_key(&path) { + parent.changes.push(ClassifiedChangeV1 { + path, + class: change.class, + }); + } + } + parent.nested_repository_exclusions.extend( + local + .nested_repository_exclusions + .into_iter() + .map(|path| format!("{prefix}/{path}")), + ); + let qualify = |path: &[u8]| [prefix.as_bytes(), b"/", path].concat(); + parent + .non_utf8_paths + .extend(local.non_utf8_paths.into_iter().map(|path| qualify(&path))); + parent + .non_utf8_changes + .extend(local.non_utf8_changes.into_iter().map(|change| { + NonUtf8ClassifiedChangeV1 { + git_path: qualify(&change.git_path), + class: change.class, + } + })); + } + Ok(parent) + } + + pub(super) fn regular_child_files(&self, root: &Path) -> std::io::Result> { + let mut files = BTreeSet::new(); + for path in self.candidate_paths() { + if !super::submodules::is_source_location(Path::new(&path)) { + continue; + } + match root.join(&path).symlink_metadata() { + Ok(metadata) if metadata.is_file() => { + files.insert(path); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(files) + } + + pub fn child_root(&self, logical_path: &str) -> Option<&str> { + self.child_sources.get(logical_path).map(String::as_str) + } + + pub(super) fn gitlink_paths(&self) -> &BTreeSet { + &self.parent_gitlinks + } + + /// Explicit enrollment cannot cross a parent gitlink staged for removal. + pub fn beneath_gitlink(&self, path: &Path) -> bool { path.ancestors() .skip(1) .filter_map(Path::to_str) @@ -315,7 +403,8 @@ impl WorktreeChangeClassificationV1 { pub fn has_changes(&self) -> bool { !(self.changes().is_empty() && self.non_utf8_changes().is_empty() - && self.nested_repository_exclusions.is_empty()) + && self.nested_repository_exclusions.is_empty() + && !self.child_revision_changed) } /// Whether any classified change is a merge conflict. @@ -342,7 +431,7 @@ impl WorktreeChangeClassificationV1 { } /// Paths removed from the present snapshot (staged or unstaged deletions). - #[cfg(test)] + #[cfg(any(test, feature = "test-helpers"))] pub fn deleted_paths(&self) -> BTreeSet { self.changes .iter() @@ -364,7 +453,7 @@ impl WorktreeChangeClassificationV1 { } /// The class recorded for `path`, if any change touched it. - #[cfg(test)] + #[cfg(any(test, feature = "test-helpers"))] pub fn class_of(&self, path: &str) -> Option { self.changes .iter() @@ -533,7 +622,10 @@ mod nested_repository_tests { ); symlink("app/src/lib.rs", parent.join("bridge.rs")).unwrap(); fixture_git(&parent, &["add", ".gitmodules", "app", "bridge.rs"]); - fixture_git(&parent, &["commit", "-qm", "registered child and source alias"]); + fixture_git( + &parent, + &["commit", "-qm", "registered child and source alias"], + ); assert_eq!( fs::read_to_string(parent.join("bridge.rs")).unwrap(), "pub fn child() {}\n", @@ -550,7 +642,10 @@ mod nested_repository_tests { let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); assert!(!classified.candidate_paths().contains("bridge.rs")); assert!(classified.changed_paths().contains("bridge.rs")); - assert!(classified.has_changes(), "do not reuse HEAD tree source bytes"); + assert!( + classified.has_changes(), + "do not reuse HEAD tree source bytes" + ); assert!(!classified.candidate_paths().contains("app/src/lib.rs")); } diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs new file mode 100644 index 00000000000..5c08ebbcd1a --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -0,0 +1,188 @@ +//! A parent gitlink names a commit, not the worktree that may supply its bytes. +//! Bind the registration, object store, and checkout before crossing that boundary. + +use std::collections::BTreeMap; +use std::path::{Component, Path, PathBuf}; + +use gix::bstr::ByteSlice; + +use super::{SourceWalkError, has_nested_repository_ancestor}; + +fn failure(error: impl std::fmt::Display) -> SourceWalkError { + SourceWalkError { + glob: String::new(), + message: format!("cannot inspect registered submodule: {error}"), + } +} + +#[derive(Clone, Debug)] +pub struct AuthenticatedSubmodule { + pub worktree: PathBuf, + pub git_dir: PathBuf, + pub pin: gix::ObjectId, +} + +fn normal_relative(path: &Path) -> bool { + !path.as_os_str().is_empty() + && path + .components() + .all(|component| matches!(component, Component::Normal(_))) +} + +/// Administrative data is outside every parent source surface, even with an explicit glob. +pub fn is_source_location(relative: &Path) -> bool { + normal_relative(relative) + && !relative.components().any(|component| { + component.as_os_str() == ".git" || component.as_os_str() == ".tracedecay" + }) +} + +fn registrations(bytes: &[u8]) -> Result, SourceWalkError> { + let modules = gix::submodule::File::from_bytes(bytes, None, &gix::config::File::default()) + .map_err(failure)?; + let mut registrations = BTreeMap::new(); + for name in modules.names() { + let name = gix::validate::submodule::name(name).map_err(failure)?; + let path = modules.path(name).map_err(failure)?; + let path = path.to_str().map_err(failure)?; + if !is_source_location(Path::new(path)) { + continue; + } + let name = PathBuf::from(name.to_str().map_err(failure)?); + if !normal_relative(&name) + || !normal_relative(Path::new(path)) + || registrations.insert(path.to_owned(), name).is_some() + { + return Err(failure("ambiguous or invalid .gitmodules path")); + } + } + Ok(registrations) +} + +fn module_stores<'a>( + parent: &'a gix::Repository, + name: &'a Path, +) -> impl Iterator + 'a { + [parent.git_dir(), parent.common_dir()] + .into_iter() + .filter_map(move |directory| { + let expected = directory.canonicalize().ok()?.join("modules").join(name); + (expected.canonicalize().ok()? == expected).then_some(expected) + }) +} + +fn open_pinned_store(expected: &Path, pin: gix::ObjectId) -> Option { + let child = tracedecay_runtime_core::git_open::open(expected).ok()?; + if child.git_dir().canonicalize().ok()? != expected + || child.common_dir().canonicalize().ok()? != expected + { + return None; + } + child.find_commit(pin).ok()?; + Some(child) +} + +fn bound_checkout( + parent: &gix::Repository, + path: &str, + name: &Path, + pin: gix::ObjectId, +) -> Option { + let root = parent.workdir()?.canonicalize().ok()?; + let path = Path::new(path); + if has_nested_repository_ancestor(&root, path).ok()? { + return None; + } + let worktree = root.join(path); + let marker = worktree.join(".git"); + if worktree.canonicalize().ok()? != worktree || !marker.symlink_metadata().ok()?.is_file() { + return None; + } + let git_dir = gix::discover::path::from_gitdir_file(&marker) + .ok()? + .canonicalize() + .ok()?; + // Git initializes children of a linked parent under that worktree's + // administrative directory. Check the name-derived, canonical location + // before opening any repository named by the checkout's marker. + let expected = module_stores(parent, name).find(|candidate| *candidate == git_dir)?; + let child = open_pinned_store(&expected, pin)?; + if child.workdir()?.canonicalize().ok()? != worktree { + return None; + } + Some(AuthenticatedSubmodule { + worktree, + git_dir: expected, + pin, + }) +} + +pub fn live_submodules( + parent: &gix::Repository, +) -> Result, SourceWalkError> { + let mut children = BTreeMap::new(); + let index = parent.index_or_empty().map_err(failure)?; + let stage = gix::index::entry::Stage::Unconflicted; + let Some(modules) = index.entry_by_path_and_stage(".gitmodules".into(), stage) else { + return Ok(children); + }; + if !modules + .mode + .to_tree_entry_mode() + .is_some_and(|mode| mode.is_blob()) + { + return Ok(children); + } + let blob = parent.find_blob(modules.id).map_err(failure)?; + let registered = registrations(&blob.data)?; + // An unstaged deletion or replacement of the registration cannot authorize a checkout. + let Some(live_registration) = parent.open_modules_file().map_err(failure)? else { + return Ok(children); + }; + for (path, name) in registered { + let Some(entry) = index.entry_by_path_and_stage(path.as_bytes().as_bstr(), stage) else { + continue; + }; + if !entry.mode.is_submodule() { + continue; + } + if !live_registration + .path(name.as_os_str().as_encoded_bytes().as_bstr()) + .is_ok_and(|current| current.as_slice() == path.as_bytes()) + { + continue; + } + if let Some(child) = bound_checkout(parent, &path, &name, entry.id) { + children.insert(path, child); + } + } + Ok(children) +} + +/// Only the selected parent tree may supply historical registrations and pins. +pub fn pinned_submodules( + parent: &gix::Repository, + pins: &tracedecay_runtime_core::git_repository::GitTreeSubmodulePins, +) -> Result, SourceWalkError> { + let mut children = BTreeMap::new(); + if pins.gitlinks.is_empty() { + return Ok(children); + } + let registrations = registrations(&pins.registration)?; + for (path, pin) in &pins.gitlinks { + let Some(name) = registrations.get(path) else { + continue; + }; + // The checkout can move or be deinitialized after the parent revision. + // Its current bytes and .git marker are never historical authorities. + let repo = module_stores(parent, name) + .find_map(|directory| open_pinned_store(&directory, *pin)) + .ok_or_else(|| { + failure(format!( + "registered object store or pinned commit unavailable for {path}" + )) + })?; + children.insert(path.clone(), (repo, *pin)); + } + Ok(children) +} diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs index 1e1558def46..5ef6dd9e3d2 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs @@ -277,6 +277,7 @@ impl CodeIndexExecutionControlV1 for MutableCancellationControl { pub(super) fn config() -> CodeIndexProductionConfigV1 { CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.production"), repository: id::("repository.production"), sanitizer_revision: id::("sanitizer.v1"), @@ -319,6 +320,7 @@ fn request_in_scope( request.snapshot.worktree = worktree.map(id::); request.snapshot.source_revision = Some(id::(source_revision)); request.repository_parse_identity = CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(id::(&format!("tree.{source_revision}"))), dirty: RepositoryDirtyStateV1::Dirty, }; @@ -365,6 +367,7 @@ fn request_at_path( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -1033,6 +1036,7 @@ fn resumed_parse_quanta_publish_the_same_complete_generation() { invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -3003,6 +3007,7 @@ fn partitioned_request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs index a23f502b509..7f70e4b9f82 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs @@ -92,6 +92,7 @@ fn multi_file_request(file_count: usize, sealed_at: i64) -> CodeIndexBuildReques invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/tests/resident_accounting.rs b/crates/tracedecay-code-index/tests/resident_accounting.rs index 18a6afcf497..05fd76256c1 100644 --- a/crates/tracedecay-code-index/tests/resident_accounting.rs +++ b/crates/tracedecay-code-index/tests/resident_accounting.rs @@ -154,6 +154,7 @@ fn request(files: usize) -> CodeIndexBuildRequestV1 { invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -168,6 +169,7 @@ fn request(files: usize) -> CodeIndexBuildRequestV1 { fn config() -> CodeIndexProductionConfigV1 { CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.accounting"), repository: id::("repository.accounting"), sanitizer_revision: id::("sanitizer.v1"), diff --git a/crates/tracedecay-configuration/src/config/mod.rs b/crates/tracedecay-configuration/src/config/mod.rs index 12628536d73..52d0ca57254 100644 --- a/crates/tracedecay-configuration/src/config/mod.rs +++ b/crates/tracedecay-configuration/src/config/mod.rs @@ -22,9 +22,9 @@ use tracedecay_domain::configuration::{ AUTOMATION_SETTINGS_SETTING_KEY, AutomationSettingsV1, ConfigurationRevisionId, ConfigurationSnapshotV1, ConfigurationValueV1, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, INDEX_INCLUDE_SETTING_KEY, - INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, - INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, - LcmSummarizerExecutablesV1, SYNC_AUTO_INIT_SETTING_KEY, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, + INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutablesV1, SYNC_AUTO_INIT_SETTING_KEY, SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, SYNC_AUTO_WATCH_SETTING_KEY, SYNC_BACKSTOP_INTERVAL_MINS_SETTING_KEY, SYNC_BRANCH_GC_DAYS_SETTING_KEY, SYNC_READ_COOLDOWN_SECS_SETTING_KEY, @@ -277,7 +277,11 @@ fn runtime_config_from_snapshot( required_string_list(snapshot, INDEX_EXCLUDE_SETTING_KEY)?, required_string_list(snapshot, INDEX_INCLUDE_SETTING_KEY)?, ) - .map_err(|error| config_error(format!("resolved index path policy is invalid: {error}")))?, + .map_err(|error| config_error(format!("resolved index path policy is invalid: {error}")))? + .with_submodules(required_bool( + snapshot, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + )?), max_file_size: required_unsigned(snapshot, INDEX_MAX_FILE_SIZE_SETTING_KEY)?, extract_docstrings: required_bool(snapshot, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY)?, track_call_sites: required_bool(snapshot, INDEX_TRACK_CALL_SITES_SETTING_KEY)?, @@ -460,6 +464,28 @@ mod tests { } } + #[test] + fn submodule_policy_is_default_off_and_materializes_the_project_override() { + let default = + PinnedRuntimeConfiguration::new(target(), revision(), resolved(BTreeMap::new())) + .unwrap(); + assert!(!default.config().index_paths.includes_submodules()); + let entries = BTreeMap::from([( + SettingKey::new(super::INDEX_INCLUDE_SUBMODULES_SETTING_KEY).unwrap(), + ConfigurationValueV1::Boolean(true), + )]); + let configured = + PinnedRuntimeConfiguration::new(target(), revision(), resolved(entries)).unwrap(); + assert!(configured.config().index_paths.includes_submodules()); + assert!( + configured + .config() + .index_paths + .without_generated_dir_defaults() + .includes_submodules() + ); + } + #[test] fn pin_rejects_a_snapshot_missing_a_required_setting() { let complete = resolved(BTreeMap::new()); diff --git a/crates/tracedecay-domain/src/configuration.rs b/crates/tracedecay-domain/src/configuration.rs index 601a3eb328f..35d41cb26cd 100644 --- a/crates/tracedecay-domain/src/configuration.rs +++ b/crates/tracedecay-domain/src/configuration.rs @@ -68,6 +68,7 @@ pub fn is_profile_setting_key(key: &SettingKey) -> bool { /// Canonical project-scoped runtime settings. pub const INDEX_EXCLUDE_SETTING_KEY: &str = "index.exclude.v1"; +pub const INDEX_INCLUDE_SUBMODULES_SETTING_KEY: &str = "index.include_submodules.v1"; pub const INDEX_INCLUDE_SETTING_KEY: &str = "index.include.v1"; pub const INDEX_MAX_FILE_SIZE_SETTING_KEY: &str = "index.max_file_size.v1"; pub const INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY: &str = "index.extract_docstrings.v1"; @@ -110,6 +111,7 @@ pub const CONFIGURATION_SETTING_KEYS_V1: &[&str] = &[ USER_WORK_EXPERTISE_CONSENT_SETTING_KEY, INDEX_EXCLUDE_SETTING_KEY, INDEX_INCLUDE_SETTING_KEY, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, diff --git a/crates/tracedecay-domain/src/lib.rs b/crates/tracedecay-domain/src/lib.rs index ffbef3e2d16..764d8cd2317 100644 --- a/crates/tracedecay-domain/src/lib.rs +++ b/crates/tracedecay-domain/src/lib.rs @@ -104,9 +104,10 @@ pub use configuration::{ ContextScoutConfigurationStateV1, ContextScoutConfiguredModelPathV1, ContextScoutSettingsV1, CredentialReferenceId, CrossMergeModeV1, CrossMergePolicyV1, GitHubStackedPullRequestPolicyV1, HistoryRewritePolicyV1, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, - INDEX_INCLUDE_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, - INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, - LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutableV1, LcmSummarizerExecutablesV1, + INDEX_INCLUDE_SETTING_KEY, INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, + LcmSummarizerExecutableV1, LcmSummarizerExecutablesV1, MAX_WORK_EXPERTISE_CONSENT_LIFETIME_MICROS_V1, PROJECT_WORK_EXPERTISE_CONSENT_SETTING_KEY, ProtectedApplyRequest, ProtectedChange, ProtectedChangePlan, ProtectedChangeSnapshotError, ProtectedRefDispositionV1, ProtectedRefRuleV1, ProtectedRefSelectorV1, QueryCollectionId, diff --git a/crates/tracedecay-domain/src/source_path_policy.rs b/crates/tracedecay-domain/src/source_path_policy.rs index 9871687d7e4..9a921d6fb85 100644 --- a/crates/tracedecay-domain/src/source_path_policy.rs +++ b/crates/tracedecay-domain/src/source_path_policy.rs @@ -60,6 +60,7 @@ pub struct IndexPathPatternError { /// files. #[derive(Clone, Debug)] pub struct IndexPathPolicyV1 { + include_submodules: bool, exclude: Vec, include: Vec, exclude_set: GlobSet, @@ -78,7 +79,9 @@ pub struct IndexPathPolicyV1 { impl PartialEq for IndexPathPolicyV1 { fn eq(&self, other: &Self) -> bool { - self.exclude == other.exclude && self.include == other.include + self.include_submodules == other.include_submodules + && self.exclude == other.exclude + && self.include == other.include } } @@ -109,6 +112,7 @@ impl IndexPathPolicyV1 { .then(|| Self::new(configured, include.clone()).map(Box::new)) .transpose()?; Ok(Self { + include_submodules: false, exclude, include, exclude_set, @@ -132,6 +136,18 @@ impl IndexPathPolicyV1 { .unwrap_or(self) } + pub fn with_submodules(mut self, enabled: bool) -> Self { + self.include_submodules = enabled; + if let Some(scoped) = self.without_generated_dir_defaults.as_mut() { + scoped.include_submodules = enabled; + } + self + } + + pub fn includes_submodules(&self) -> bool { + self.include_submodules + } + pub fn exclude_patterns(&self) -> &[String] { &self.exclude } diff --git a/crates/tracedecay-global-db/src/configuration/registry.rs b/crates/tracedecay-global-db/src/configuration/registry.rs index 97d964bf4af..02c3b283cb9 100644 --- a/crates/tracedecay-global-db/src/configuration/registry.rs +++ b/crates/tracedecay-global-db/src/configuration/registry.rs @@ -8,9 +8,10 @@ use tracedecay_domain::configuration::{ AnalyzerSettingsV1, CONFIGURATION_SETTING_KEYS_V1, CONTEXT_SCOUT_SETTINGS_SETTING_KEY, CodeIndexWorkerSelectionV1, ConfigurationValueKindV1, ConfigurationValueV1, ContextScoutSettingsV1, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, - INDEX_INCLUDE_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, - INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, - LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutablesV1, PROFILE_SETTING_KEYS_V1, + INDEX_INCLUDE_SETTING_KEY, INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, + LcmSummarizerExecutablesV1, PROFILE_SETTING_KEYS_V1, PROJECT_WORK_EXPERTISE_CONSENT_SETTING_KEY, RestartRequirementV1, SOURCE_BINDINGS_SETTING_KEY, SYNC_AUTO_INIT_SETTING_KEY, SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, SYNC_AUTO_WATCH_SETTING_KEY, @@ -35,7 +36,7 @@ pub const MAX_PROXIMITY_RISK_THRESHOLD_BASIS_POINTS_V1: u64 = 10_000; /// Registry schema revision. Increment only when setting-definition semantics /// change, not when a setting value changes. -pub const CONFIGURATION_REGISTRY_SCHEMA_REVISION: u16 = 8; +pub const CONFIGURATION_REGISTRY_SCHEMA_REVISION: u16 = 9; #[derive(Debug, Error)] pub enum ConfigurationRegistryError { @@ -523,6 +524,12 @@ fn register_project_settings( SettingSensitivityV1::Sensitive, RestartRequirementV1::DaemonRestart, ), + ( + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + ConfigurationValueV1::Boolean(false), + SettingSensitivityV1::Public, + RestartRequirementV1::DaemonRestart, + ), ( INDEX_MAX_FILE_SIZE_SETTING_KEY, ConfigurationValueV1::Unsigned(defaults.max_file_size), @@ -779,7 +786,10 @@ mod user_profile_settings_tests { ); let definition = registry.definition(&key).expect("definition"); - assert_eq!(definition.schema_revision, 8); + assert_eq!( + definition.schema_revision, + CONFIGURATION_REGISTRY_SCHEMA_REVISION + ); assert_eq!(definition.scope, SettingScopeV1::UserProfile); assert_eq!( definition.value_kind, diff --git a/crates/tracedecay-global-db/src/configuration/store.rs b/crates/tracedecay-global-db/src/configuration/store.rs index f20a0c2bf64..ff751c349a5 100644 --- a/crates/tracedecay-global-db/src/configuration/store.rs +++ b/crates/tracedecay-global-db/src/configuration/store.rs @@ -16,11 +16,11 @@ use tracedecay_domain::configuration::{ ConfigurationAuditEvent, ConfigurationAuditEventId, ConfigurationAuditEventKindV1, ConfigurationCandidateV1, ConfigurationIdempotencyKey, ConfigurationLayerIdV1, ConfigurationReceiptId, ConfigurationRevisionId, ConfigurationSnapshotV1, ConfigurationValueV1, - INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, - ProtectedChange, ProtectedChangePlan, ProtectedChangeSnapshotError, - RedactedConfigurationChangeV1, RollbackModeV1, SOURCE_BINDINGS_SETTING_KEY, - SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, ScopeControlOperationV1, SettingKey, - USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, ProtectedChange, ProtectedChangePlan, + ProtectedChangeSnapshotError, RedactedConfigurationChangeV1, RollbackModeV1, + SOURCE_BINDINGS_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, ScopeControlOperationV1, + SettingKey, USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, UserProfileId, WORK_TOPOLOGY_POLICY_SETTING_KEY, }; @@ -349,6 +349,7 @@ impl<'db> GlobalDbConfigurationControlStore<'db> { occurred_at, ®istry, &[ + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, diff --git a/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs b/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs index 5e1b7f7d9cb..8c6b4ab1288 100644 --- a/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs +++ b/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs @@ -50,13 +50,19 @@ fn incomplete_snapshot_requires_reset_instead_of_default_repair() { } #[asupersync::test] -async fn linked_worktree_default_converges_into_existing_snapshot() { +async fn linked_worktree_and_submodule_defaults_converge_into_existing_snapshot() { let (_directory, runtime, root) = Box::pin(global_setup()).await; - let key = SettingKey::new(SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY).unwrap(); + let keys = [ + SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, + tracedecay_domain::configuration::INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + ] + .map(|name| SettingKey::new(name).unwrap()); let mut effective_values = root.snapshot.effective_values.clone(); let mut provenance = root.snapshot.provenance.clone(); - effective_values.remove(&key); - provenance.remove(&key); + for key in &keys { + effective_values.remove(key); + provenance.remove(key); + } let historical_snapshot = ConfigurationSnapshotV1::new(effective_values, provenance).unwrap(); let db = runtime .registered_database(HostAdmissionScope::Project) @@ -69,13 +75,18 @@ async fn linked_worktree_default_converges_into_existing_snapshot() { ) .await .unwrap(); - transaction - .execute( - "DELETE FROM configuration_entries WHERE revision_id = ?1 AND key = ?2", - tracedecay_runtime_core::db::engine::params![root.revision_id.as_str(), key.as_str()], - ) - .await - .unwrap(); + for key in &keys { + transaction + .execute( + "DELETE FROM configuration_entries WHERE revision_id = ?1 AND key = ?2", + tracedecay_runtime_core::db::engine::params![ + root.revision_id.as_str(), + key.as_str() + ], + ) + .await + .unwrap(); + } point_revision_at_snapshot(&transaction, &root.revision_id, &historical_snapshot).await; transaction.commit().await.unwrap(); let store = GlobalDbConfigurationControlStore::new_registered(db); @@ -86,11 +97,13 @@ async fn linked_worktree_default_converges_into_existing_snapshot() { .unwrap(); assert_ne!(converged.revision_id, root.revision_id); - assert_eq!( - converged.snapshot.effective_values.get(&key), - Some(&ConfigurationValueV1::Boolean(false)) - ); - assert!(converged.snapshot.provenance.contains_key(&key)); + for key in &keys { + assert_eq!( + converged.snapshot.effective_values.get(key), + Some(&ConfigurationValueV1::Boolean(false)) + ); + assert!(converged.snapshot.provenance.contains_key(key)); + } validate_snapshot_registry_completeness(&converged.snapshot).unwrap(); } diff --git a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs index 75f161b692a..d8e2a8c5c1e 100644 --- a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs +++ b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs @@ -13,6 +13,7 @@ pub(super) async fn compute_field_sites( graph: &tracedecay_graph_query::VerifiedGraphQuery, args: Value, scope_prefix: Option<&str>, + include_submodules: bool, ) -> Result { let request: FieldSitesSurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_field_sites")?; @@ -63,6 +64,7 @@ pub(super) async fn compute_field_sites( let needle = format!(".{field_name}"); 'outer: for file in &files { + if tracedecay_code_index::source_walk::is_source_symlink(&project_root,Path::new(file))? { continue; } let abs = project_root.join(file); let Ok(disk_source) = tracedecay_runtime_core::sync::read_source_file(&abs) else { continue; @@ -76,7 +78,7 @@ pub(super) async fn compute_field_sites( if !disk_source.contains(&needle) { continue; } - let source = sealed_field_source(&project_root, file, sealed_files.get(file))?; + let source = sealed_field_source(&project_root, file, sealed_files.get(file), include_submodules)?; let masked = path_is_rust(file).then(|| { tracedecay_code_extraction::source_mask::masked_rust_source_with( &source, @@ -198,6 +200,7 @@ fn sealed_field_source( project_root: &Path, file: &str, sealed: Option<&(LanguageId, ContentDigest)>, + include_submodules: bool, ) -> Result { let Some((language, digest)) = sealed else { return Err(TraceDecayError::project_route( @@ -208,7 +211,7 @@ fn sealed_field_source( ), )); }; - let bytes = sealed_source_bytes(project_root, file, language, digest) + let bytes = sealed_source_bytes(project_root, file, language, include_submodules, digest) .map_err(|error| { TraceDecayError::project_route( "verified-field-source-unavailable", diff --git a/crates/tracedecay-mcp/src/handlers/analysis/mod.rs b/crates/tracedecay-mcp/src/handlers/analysis/mod.rs index a6008e9cfdb..33a5babdad6 100644 --- a/crates/tracedecay-mcp/src/handlers/analysis/mod.rs +++ b/crates/tracedecay-mcp/src/handlers/analysis/mod.rs @@ -100,7 +100,13 @@ pub async fn compute_analysis_report( compute_constructors(&graph, args, scope_prefix).await } ApplicationSurfaceOperation::FieldSites => { - compute_field_sites(&graph, args, scope_prefix).await + compute_field_sites( + &graph, + args, + scope_prefix, + ctx.index_path_policy().includes_submodules(), + ) + .await } operation => Err(unknown_tool_error(operation.mcp_tool_name())), } diff --git a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs index 35f13c19289..8822df7de21 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs @@ -175,6 +175,7 @@ struct RenameReferenceSiteInput { } struct PreviewFileWalk { + include_submodules: bool, project_root: PathBuf, declaration_line: u32, result: RenamePreviewPrimitiveResultV1, @@ -202,6 +203,19 @@ impl PreviewFileWalk { let has_references = reference_inputs .peek() .is_some_and(|input| input.file == file); + if tracedecay_code_index::source_walk::is_source_symlink( + &self.project_root, + std::path::Path::new(&file), + )? { + if has_references { + return Err(TraceDecayError::project_route( + "verified-reference-source-unavailable", + true, + format!("a source symlink replaced '{file}'"), + )); + } + continue; + } let text = match tracedecay_runtime_core::sync::read_source_file( &self.project_root.join(&file), ) { @@ -220,6 +234,7 @@ impl PreviewFileWalk { &self.project_root, &file, language, + self.include_submodules, digest, ) .map_err(|error| { @@ -427,6 +442,7 @@ pub async fn compute_rename_preview( freshness: None, }; let walk = PreviewFileWalk { + include_submodules: ctx.index_path_policy().includes_submodules(), project_root: ctx.project_root().to_path_buf(), declaration_line, result, diff --git a/crates/tracedecay-query/benches/code_lexical_catchup.rs b/crates/tracedecay-query/benches/code_lexical_catchup.rs index 434a75740a9..6b45e53cd60 100644 --- a/crates/tracedecay-query/benches/code_lexical_catchup.rs +++ b/crates/tracedecay-query/benches/code_lexical_catchup.rs @@ -184,6 +184,7 @@ fn build_fixture() -> Fixture { invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -195,6 +196,7 @@ fn build_fixture() -> Fixture { }, }; let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.catchup-benchmark"), repository: repository.clone(), sanitizer_revision, diff --git a/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs b/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs index 64681d78557..78419426251 100644 --- a/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs +++ b/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs @@ -561,6 +561,7 @@ fn run(options: &Options) -> Result { let repository = identity::("repository.index-bench"); let sanitizer_revision = identity::("sanitizer.index-bench.v1"); let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: identity::("project.index-bench"), repository: repository.clone(), sanitizer_revision: sanitizer_revision.clone(), @@ -736,6 +737,7 @@ fn build_request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(identity::(tree)), dirty: RepositoryDirtyStateV1::Clean, }, diff --git a/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs b/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs index d1d6a515c8e..9b9cd38fde5 100644 --- a/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs +++ b/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs @@ -329,6 +329,7 @@ fn run(options: &Options) -> Result { let repository = identity::("repository.search-bench"); let sanitizer_revision = identity::("sanitizer.search-bench.v1"); let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: identity::("project.search-bench"), repository: repository.clone(), sanitizer_revision: sanitizer_revision.clone(), @@ -726,6 +727,7 @@ fn build_request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(identity::("tree.search-bench.clean")), dirty: RepositoryDirtyStateV1::Clean, }, diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index f186edc00ec..aa1def7ed41 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -425,6 +425,7 @@ pub(crate) fn real_lexical_source_fixture_from_sources( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -436,6 +437,7 @@ pub(crate) fn real_lexical_source_fixture_from_sources( }, }; let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.artifact"), repository: repository.clone(), sanitizer_revision, diff --git a/crates/tracedecay-runtime-core/src/git_repository.rs b/crates/tracedecay-runtime-core/src/git_repository.rs index 13c8dadc6d8..bfb65d12122 100644 --- a/crates/tracedecay-runtime-core/src/git_repository.rs +++ b/crates/tracedecay-runtime-core/src/git_repository.rs @@ -33,7 +33,7 @@ pub use history::{ pub use native_integration::{ GitNativeApplyOutcome, GitNativeCandidateTreeV1, GitNativeCandidateTreeVisitError, GitNativeIntegrationMode, GitNativePreflight, GitNativePreflightCaptureError, - GitNativePreflightDisposition, GitNativeUnsupportedReason, + GitNativePreflightDisposition, GitNativeUnsupportedReason, GitTreeSubmodulePins, }; pub use tree_links::resolve_tree_link; diff --git a/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs b/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs index e09324c149f..ada2a0215a1 100644 --- a/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs +++ b/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs @@ -67,6 +67,40 @@ pub struct GitNativeCandidateTreeV1<'repository> { cancellation: &'repository CancellationToken, } +/// Registration and child commits taken from one immutable parent tree. +#[derive(Debug, Default)] +pub struct GitTreeSubmodulePins { + pub registration: Vec, + pub gitlinks: Vec<(String, gix::ObjectId)>, +} + +impl GitTreeSubmodulePins { + pub fn from_tree( + tree: &gix::Tree<'_>, + gitlinks: Vec<(String, gix::ObjectId)>, + ) -> Result { + if gitlinks.is_empty() { + return Ok(Self::default()); + } + let registration = match tree + .find_entry(".gitmodules") + .filter(|entry| entry.mode().is_blob()) + { + Some(entry) => tree + .repo + .find_blob(entry.object_id()) + .map_err(|error| operation("read pinned .gitmodules", error))? + .data + .clone(), + None => Vec::new(), + }; + Ok(Self { + registration, + gitlinks, + }) + } +} + impl GitNativeCandidateTreeV1<'_> { pub fn tree(&self) -> Result { oid(self.tree) @@ -78,9 +112,10 @@ impl GitNativeCandidateTreeV1<'_> { /// when it names none; every other symlink is visited with its link text. pub fn visit_blobs( &self, + include_submodules: bool, read_through: impl Fn(&[u8]) -> bool, mut visitor: impl FnMut(&[u8], &[u8]) -> Result<(), E>, - ) -> Result<(), GitNativeCandidateTreeVisitError> { + ) -> Result> { let tree = self .repository .find_tree(self.tree) @@ -90,6 +125,7 @@ impl GitNativeCandidateTreeV1<'_> { repository_visit_error("native candidate tree traversal", error) })?; entries.sort_by(|left, right| left.filepath.cmp(&right.filepath)); + let mut gitlinks = Vec::new(); for entry in entries { if self.cancellation.is_cancelled() { return Err(GitNativeCandidateTreeVisitError::Repository( @@ -99,6 +135,12 @@ impl GitNativeCandidateTreeV1<'_> { }, )); } + if include_submodules && entry.mode.is_commit() { + let path = entry.filepath.to_str().map_err(|error| { + repository_visit_error("native candidate gitlink path", error) + })?; + gitlinks.push((path.to_owned(), entry.oid)); + } if entry.mode.is_tree() || entry.mode.is_commit() { continue; } @@ -120,7 +162,8 @@ impl GitNativeCandidateTreeV1<'_> { visitor(entry.filepath.as_ref(), &blob.data) .map_err(GitNativeCandidateTreeVisitError::Visitor)?; } - Ok(()) + GitTreeSubmodulePins::from_tree(&tree, gitlinks) + .map_err(GitNativeCandidateTreeVisitError::Repository) } } diff --git a/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs b/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs index a1dc0be714e..2f8d6c85f09 100644 --- a/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs +++ b/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs @@ -268,6 +268,7 @@ fn synthetic_candidate_tree_is_streamed_in_order_before_object_memory_drops() { let mut paths = Vec::new(); candidate .visit_blobs( + false, |_| false, |path, _bytes| { paths.push(String::from_utf8(path.to_vec()).expect("UTF-8 test path")); @@ -306,6 +307,7 @@ fn synthetic_candidate_tree_is_streamed_in_order_before_object_memory_drops() { |_, candidate| { candidate .visit_blobs( + false, |_| false, |_, _| { cancelled.cancel(); @@ -358,6 +360,7 @@ fn candidate_tree_reads_accepted_symlinks_through_to_their_blobs() { let mut visited = Vec::new(); candidate .visit_blobs( + false, |path| path.ends_with(b".rs"), |path, bytes| { visited.push(( diff --git a/crates/tracedecay-search-eval/src/candidate_output.rs b/crates/tracedecay-search-eval/src/candidate_output.rs index dcb3c2bb70b..4bb11bb8d56 100644 --- a/crates/tracedecay-search-eval/src/candidate_output.rs +++ b/crates/tracedecay-search-eval/src/candidate_output.rs @@ -1324,6 +1324,7 @@ fn publish_scope_generation( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -1331,6 +1332,7 @@ fn publish_scope_generation( target_projection_key, }; let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.candidate.fixture")?, repository: id::("repository.candidate.fixture")?, sanitizer_revision: id::("sanitizer.candidate.v1")?, diff --git a/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs b/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs index eff54a990f0..b258849ab29 100644 --- a/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs +++ b/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs @@ -72,6 +72,7 @@ pub(super) fn prove_cancellation( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -84,6 +85,7 @@ pub(super) fn prove_cancellation( }; let generation_scope = CodeIndexGenerationScopeV1::for_snapshot(&request.snapshot); let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.candidate.cancel")?, repository: id::("repository.candidate.cancel")?, sanitizer_revision: id::("sanitizer.candidate.v1")?, diff --git a/docs/USER-GUIDE.md b/docs/USER-GUIDE.md index d7eb98fc29d..15a3ee90140 100644 --- a/docs/USER-GUIDE.md +++ b/docs/USER-GUIDE.md @@ -209,6 +209,22 @@ The value replaces the whole list, so start from the current effective value malformed pattern is refused before anything is written. There is no per-run folder exclusion; `init` and `sync` take no folder flags. +Direct Git submodules are excluded by default. To index their checked-out +files under parent-relative paths, set the project’s Boolean +`index.include_submodules.v1` setting and restart the daemon. + +```bash +tracedecay tool configuration_set --args '{"layer":{"kind":"project","project_id":""},"key":"index.include_submodules.v1","value":{"kind":"boolean","value":true},"expected_revision":"","idempotency_key":""}' +``` + +The parent index and `.gitmodules` must identify the child, and its checkout +must use the matching repository under the parent's Git modules directory. +Live indexing includes child worktree changes. Historical indexing reads the +child commit recorded by the selected parent revision from the local object +store. Initialize the submodule locally before requesting that history. +Linked worktrees, unrelated clones, and deeper nested repositories stay +outside the parent index. + ### Seeing what changed Use the daemon's status/coverage result to see the selected generation, exact From b96724957d79e242838c22b14b48c9d7c17e64b1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:42:02 -0800 Subject: [PATCH 26/42] fix(graph): bind foreground reads to current checkout sources --- .../code_index_scheduler/freshness_witness.rs | 32 +---- .../tests/index_path_policy_tests.rs | 20 +++- .../tracedecay-code-index/src/source_walk.rs | 74 +++++------- .../src/source_walk/submodules.rs | 111 +++++++++++++++++- .../src/handlers/analysis/field_sites.rs | 18 ++- .../src/handlers/graph/rename_preview.rs | 27 +++-- 6 files changed, 188 insertions(+), 94 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index b98634731bb..00228326aab 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -996,43 +996,21 @@ fn filter_mismatches_in_repository<'a>( /// disk first, then through the repository's own filter pipeline, the same /// two readings [`clean_filter_mismatches`] accepts as current. pub fn sealed_source_bytes( - project_root: &Path, + boundaries: &tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries, logical_path: &str, language: &LanguageId, - include_submodules: bool, sealed: &ContentDigest, ) -> Result>, CodeIndexSchedulerErrorV1> { - if tracedecay_code_index::source_walk::is_source_symlink(project_root, Path::new(logical_path))? - { + let Some((root, relative)) = boundaries.resolve(Path::new(logical_path))? else { return Ok(None); - } - let raw = - ignored_dependencies::read_bounded_snapshot_source(&project_root.join(logical_path), None)?; + }; + let raw = ignored_dependencies::read_bounded_snapshot_source(&root.join(relative), None)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &raw)?; if content_digest(&sanitized) == *sealed { return Ok(Some(sanitized)); } - // ponytail: one repository open and filter pipeline per filtered file; - // hoist them into a caller-held reader if a caller ever needs hundreds. - let repository = tracedecay_runtime_core::git_open::open(project_root) + let repository = tracedecay_runtime_core::git_open::open(root) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - let mut repository = repository; - let mut relative = Path::new(logical_path); - if include_submodules { - let children = tracedecay_code_index::source_walk::submodules::live_submodules(&repository) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; - if let Some((prefix, binding)) = children - .iter() - .find(|(prefix, _)| relative.starts_with(prefix)) - { - let child_path = relative - .strip_prefix(prefix) - .map_err(|_| CodeIndexSchedulerErrorV1::Git("invalid child filter path".into()))?; - repository = tracedecay_runtime_core::git_open::open(&binding.git_dir) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - relative = child_path; - } - } let (mut pipeline, index) = repository .filter_pipeline(None) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index f90c046bd00..c0e00751f89 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -256,6 +256,17 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() .map(|hit| hit.file.to_string()) .collect::>() }; + let child_path = std::path::Path::new("app/lib.ts"); + let direct_reads = |enabled| { + tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries::open( + parent.path(), + enabled, + ) + .unwrap() + }; + assert!(direct_reads(false).resolve(child_path).unwrap().is_none()); + let active_reads = direct_reads(true); + assert!(active_reads.resolve(child_path).unwrap().is_some()); let mut off = start(&parent, store.path(), policy(&[], &[])); off.reconcile_now().unwrap(); assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); @@ -330,6 +341,7 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() assert_eq!(grep("successor_symbol", &enabled), ["app/lib.ts"]); run_git_in(parent.path(), &["submodule", "deinit", "-f", "app"]); assert!(!checkout.join(".git").exists()); + assert!(active_reads.resolve(child_path).unwrap().is_none()); let historical = on .publish_exact_git_tree_generation( &pinned, @@ -394,11 +406,15 @@ fn parent_filtered_source_does_not_require_submodule_registration_when_disabled( run_git_in(parent.path(), &["config", "core.autocrlf", "true"]); std::fs::write(parent.path().join("lib.rs"), source.replace('\n', "\r\n")).unwrap(); std::fs::write(parent.path().join(".gitmodules"), "[unfinished").unwrap(); - let sealed = super::super::sealed_source_bytes( + let boundaries = tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries::open( parent.path(), + false, + ) + .unwrap(); + let sealed = super::super::sealed_source_bytes( + &boundaries, "lib.rs", &LanguageId::new("rust").unwrap(), - false, &content_digest(source.as_bytes()), ) .unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 78c682e2971..3701db9d5f8 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -396,55 +396,37 @@ fn source_walk_inner( .is_some_and(|scope| scope.allows(&root, path)))) } }; - let mut parent_gitlinks = parent_gitlink_paths(project_root)?; + let boundaries = + submodules::CheckoutBoundaries::open(project_root, path_policy.includes_submodules())?; + let mut parent_gitlinks = boundaries.gitlinks; let mut authenticated = BTreeSet::new(); let mut child_files = BTreeSet::new(); - if path_policy.includes_submodules() && !parent_gitlinks.is_empty() { - let parent = tracedecay_runtime_core::git_open::open(project_root).map_err(|error| { - SourceWalkError { - glob: String::new(), - message: error.to_string(), - } - })?; - for (path, child) in submodules::live_submodules(&parent)? { - let absolute = project_root.join(&path); - let policy = if scoped_policy(&absolute, true) { - path_policy.without_generated_dir_defaults() - } else { - &path_policy - }; - if policy.excludes_directory(&path) - || path_glob_scope - .as_ref() - .is_some_and(|scope| !scope.may_contain_match(project_root, &absolute)) - { - continue; - } - let repo = - tracedecay_runtime_core::git_open::open(&child.git_dir).map_err(|error| { - SourceWalkError { - glob: String::new(), - message: error.to_string(), - } - })?; - let inventory = classification::WorktreeChangeClassificationV1::classify(&repo) - .map_err(|error| SourceWalkError { - glob: String::new(), - message: error.to_string(), - })?; - let files = inventory - .regular_child_files(&child.worktree) - .map_err(|error| SourceWalkError { - glob: String::new(), - message: error.to_string(), - })?; - child_files.extend(files.into_iter().map(|file| format!("{path}/{file}"))); - // Only direct registrations belong to this parent. - for nested in inventory.gitlink_paths() { - parent_gitlinks.insert(format!("{path}/{nested}")); - } - authenticated.insert(path); + for (path, child) in boundaries.children { + let absolute = project_root.join(&path); + let policy = if scoped_policy(&absolute, true) { + path_policy.without_generated_dir_defaults() + } else { + &path_policy + }; + if policy.excludes_directory(&path) + || path_glob_scope + .as_ref() + .is_some_and(|scope| !scope.may_contain_match(project_root, &absolute)) + { + continue; + } + let inventory = child.sources()?; + child_files.extend( + inventory + .files + .into_iter() + .map(|file| format!("{path}/{file}")), + ); + // Only direct registrations belong to this parent. + for nested in inventory.gitlinks { + parent_gitlinks.insert(format!("{path}/{nested}")); } + authenticated.insert(path); } let roots = authenticated diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 5c08ebbcd1a..461f79aff41 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -1,7 +1,7 @@ //! A parent gitlink names a commit, not the worktree that may supply its bytes. //! Bind the registration, object store, and checkout before crossing that boundary. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::{Component, Path, PathBuf}; use gix::bstr::ByteSlice; @@ -22,6 +22,44 @@ pub struct AuthenticatedSubmodule { pub pin: gix::ObjectId, } +pub(super) struct ChildSources { + pub files: BTreeSet, + pub gitlinks: BTreeSet, +} + +impl AuthenticatedSubmodule { + pub(super) fn sources(&self) -> Result { + let repository = tracedecay_runtime_core::git_open::open(&self.git_dir).map_err(failure)?; + let inventory = + super::classification::WorktreeChangeClassificationV1::classify(&repository) + .map_err(failure)?; + Ok(ChildSources { + files: inventory + .regular_child_files(&self.worktree) + .map_err(failure)?, + gitlinks: inventory.gitlink_paths().clone(), + }) + } +} + +pub(super) struct CheckoutBoundaries { + pub gitlinks: BTreeSet, + pub children: BTreeMap, +} + +impl CheckoutBoundaries { + pub(super) fn open(root: &Path, include_submodules: bool) -> Result { + let gitlinks = super::parent_gitlink_paths(root)?; + let children = if include_submodules && !gitlinks.is_empty() { + let parent = tracedecay_runtime_core::git_open::open(root).map_err(failure)?; + live_submodules(&parent)? + } else { + BTreeMap::new() + }; + Ok(Self { gitlinks, children }) + } +} + fn normal_relative(path: &Path) -> bool { !path.as_os_str().is_empty() && path @@ -159,6 +197,77 @@ pub fn live_submodules( Ok(children) } +/// Batch the same Git inventory for foreground graph readers. A graph path can +/// outlive its checkout binding, even when a replacement file has equal bytes. +pub struct LiveSourceBoundaries { + project_root: PathBuf, + gitlinks: BTreeSet, + children: BTreeMap)>, +} + +impl LiveSourceBoundaries { + pub fn open(root: &Path, include_submodules: bool) -> Result { + let boundaries = CheckoutBoundaries::open(root, include_submodules)?; + let mut children = BTreeMap::new(); + for (prefix, child) in boundaries.children { + let files = child.sources()?.files; + children.insert(prefix, (child, files)); + } + Ok(Self { + project_root: root.to_owned(), + gitlinks: boundaries.gitlinks, + children, + }) + } + + /// The owning checkout and its relative file path, after checking the + /// current filesystem boundary before reading any bytes. + pub fn resolve<'a, 'p>( + &'a self, + path: &'p Path, + ) -> std::io::Result> { + if !is_source_location(path) { + return Ok(None); + } + let mut root = self.project_root.as_path(); + let mut relative = path; + for ancestor in path.ancestors().skip(1).filter_map(Path::to_str) { + if !self.gitlinks.contains(ancestor) { + continue; + } + let Some((child, files)) = self.children.get(ancestor) else { + return Ok(None); + }; + relative = path.strip_prefix(ancestor).map_err(std::io::Error::other)?; + if !relative.to_str().is_some_and(|file| files.contains(file)) + || child.worktree.canonicalize().ok().as_ref() != Some(&child.worktree) + { + return Ok(None); + } + let marker = child.worktree.join(".git"); + if !marker + .symlink_metadata() + .is_ok_and(|metadata| metadata.is_file()) + || gix::discover::path::from_gitdir_file(&marker) + .ok() + .and_then(|target| target.canonicalize().ok()) + .as_ref() + != Some(&child.git_dir) + { + return Ok(None); + } + root = &child.worktree; + break; + } + if has_nested_repository_ancestor(root, relative)? + || super::is_source_symlink(root, relative)? + { + return Ok(None); + } + Ok(Some((root, relative))) + } +} + /// Only the selected parent tree may supply historical registrations and pins. pub fn pinned_submodules( parent: &gix::Repository, diff --git a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs index d8e2a8c5c1e..d3df5f2f09e 100644 --- a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs +++ b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs @@ -1,5 +1,6 @@ //! `tracedecay_field_sites`, read and write references to a named field. +use tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries; use tracedecay_code_index_runtime::code_index_scheduler::sealed_source_bytes; use tracedecay_contracts::retrieval::{ FieldSiteV1, FieldSitesResultV1, FieldSitesSurfaceRequestV1, @@ -62,10 +63,16 @@ pub(super) async fn compute_field_sites( let mut reads: Vec = Vec::new(); let mut touched: Vec = Vec::new(); + let boundaries = LiveSourceBoundaries::open(&project_root, include_submodules) + .map_err(|error| TraceDecayError::project_route( + "verified-field-source-unavailable", true, error.message, + ))?; let needle = format!(".{field_name}"); 'outer: for file in &files { - if tracedecay_code_index::source_walk::is_source_symlink(&project_root,Path::new(file))? { continue; } - let abs = project_root.join(file); + let Some((root, relative)) = boundaries.resolve(Path::new(file))? else { + continue; + }; + let abs = root.join(relative); let Ok(disk_source) = tracedecay_runtime_core::sync::read_source_file(&abs) else { continue; }; @@ -78,7 +85,7 @@ pub(super) async fn compute_field_sites( if !disk_source.contains(&needle) { continue; } - let source = sealed_field_source(&project_root, file, sealed_files.get(file), include_submodules)?; + let source = sealed_field_source(&boundaries, file, sealed_files.get(file))?; let masked = path_is_rust(file).then(|| { tracedecay_code_extraction::source_mask::masked_rust_source_with( &source, @@ -197,10 +204,9 @@ pub(super) async fn compute_field_sites( /// the generation sealed, which differ from the checkout when git's clean /// filters rewrite it (a CRLF checkout under `core.autocrlf=true`). fn sealed_field_source( - project_root: &Path, + boundaries: &LiveSourceBoundaries, file: &str, sealed: Option<&(LanguageId, ContentDigest)>, - include_submodules: bool, ) -> Result { let Some((language, digest)) = sealed else { return Err(TraceDecayError::project_route( @@ -211,7 +217,7 @@ fn sealed_field_source( ), )); }; - let bytes = sealed_source_bytes(project_root, file, language, include_submodules, digest) + let bytes = sealed_source_bytes(boundaries, file, language, digest) .map_err(|error| { TraceDecayError::project_route( "verified-field-source-unavailable", diff --git a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs index 8822df7de21..f7ab389acf1 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs @@ -1,12 +1,13 @@ //! `tracedecay_rename_preview`: what a rename of one symbol would touch. use std::collections::HashMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use serde_json::Value; +use tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries; use tracedecay_code_index_runtime::code_index_scheduler::sealed_source_bytes; use tracedecay_contracts::graph_tool::{GraphToolCompletionV1, GraphToolResultV1}; use tracedecay_contracts::retrieval::{ @@ -198,27 +199,30 @@ impl PreviewFileWalk { let mut text_only_matches = HashMap::new(); // Only the current file's decoded text and line index are resident. // Ordinals retain the graph's reference order across the grouped walk. + let boundaries = LiveSourceBoundaries::open(&self.project_root, self.include_submodules) + .map_err(|error| { + TraceDecayError::project_route( + "verified-reference-source-unavailable", + true, + error.message, + ) + })?; for file in files { self.control.check()?; let has_references = reference_inputs .peek() .is_some_and(|input| input.file == file); - if tracedecay_code_index::source_walk::is_source_symlink( - &self.project_root, - std::path::Path::new(&file), - )? { + let Some((root, relative)) = boundaries.resolve(Path::new(&file))? else { if has_references { return Err(TraceDecayError::project_route( "verified-reference-source-unavailable", true, - format!("a source symlink replaced '{file}'"), + format!("'{file}' is outside the current checkout source boundary"), )); } continue; - } - let text = match tracedecay_runtime_core::sync::read_source_file( - &self.project_root.join(&file), - ) { + }; + let text = match tracedecay_runtime_core::sync::read_source_file(&root.join(relative)) { Ok(text) => text, Err(error) if has_references => return Err(error.into()), // Declaration snippets are optional; reference evidence is not. @@ -231,10 +235,9 @@ impl PreviewFileWalk { // after the seal only its current text remains to map against. let sealed = match self.sealed_files.get(&file) { Some((language, digest)) => sealed_source_bytes( - &self.project_root, + &boundaries, &file, language, - self.include_submodules, digest, ) .map_err(|error| { From b25c3f54da4482046eb9e173757234b14d590044 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:24:38 -0800 Subject: [PATCH 27/42] fix(code-index): reject failed submodule freshness witnesses Refs #4593. Encode a failed authenticated probe in the persisted metadata signature, and never treat repeated failed probes as retained-generation freshness evidence. Add positive and negative behavioral regressions. --- .../src/code_index_scheduler/identity.rs | 64 +++++++++++++++++-- .../src/code_index_scheduler/reconcile.rs | 9 ++- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 728e42d7b45..39f0813488b 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -240,12 +240,18 @@ impl GitMetadataFingerprintV1 { && self.head_contents == other.head_contents } + /// Whether an authenticated submodule probe can establish a freshness witness. + pub(super) fn can_prove_freshness(&self) -> bool { + !self.submodule_probe_failed + } + /// A stable, order-independent textual signature of this fingerprint. /// /// Used to persist a restore-time freshness witness: two fingerprints share - /// a signature iff they are structurally equal (`PartialEq`), so comparing - /// the recomputed signature against a witness recorded at seal time is a - /// faithful stand-in for `differs_from`. `SystemTime` mtimes are rendered as + /// a signature iff they are structurally equal (`PartialEq`). A failed + /// submodule probe is explicitly encoded, but even matching failed probes + /// cannot prove freshness: callers must reject them, as `differs_from` does. + /// `SystemTime` mtimes are rendered as /// nanos-since-epoch so the encoding is deterministic across processes. pub fn stable_signature(&self) -> String { fn render_time(time: Option) -> String { @@ -263,10 +269,14 @@ impl GitMetadataFingerprintV1 { |value| super::sha256_hex(value.as_bytes()) ), ); - if let (Some(refs), Some(indexes)) = (&self.submodule_refs, &self.submodule_indexes) { - signature.push_str(&format!( - "|submodule_refs={refs}|submodule_indexes={indexes}" - )); + if let Some(refs) = &self.submodule_refs { + signature.push_str(&format!("|submodule_refs={refs}")); + } + if let Some(indexes) = &self.submodule_indexes { + signature.push_str(&format!("|submodule_indexes={indexes}")); + } + if self.submodule_probe_failed { + signature.push_str("|submodule_probe_failed=true"); } signature } @@ -496,6 +506,46 @@ mod tests { ); } + #[test] + fn failed_submodule_probe_cannot_reuse_a_persisted_parent_only_witness() { + let root = TempDir::new().expect("non-repository root"); + let parent_only = GitMetadataFingerprintV1::capture(root.path()); + let failed = GitMetadataFingerprintV1::for_policy(root.path(), true); + + assert!(failed.submodule_probe_failed, "opt-in probe must fail closed"); + assert!(!failed.can_prove_freshness()); + assert!(failed.differs_from(&parent_only)); + assert_ne!( + failed.stable_signature(), + parent_only.stable_signature(), + "restore cannot mistake failed authentication for parent-only freshness" + ); + assert_eq!( + GitMetadataFingerprintV1::for_policy(root.path(), false).stable_signature(), + parent_only.stable_signature(), + "the default-off persisted witness remains stable" + ); + let failed_again = GitMetadataFingerprintV1::for_policy(root.path(), true); + assert_eq!(failed.stable_signature(), failed_again.stable_signature()); + assert!( + !failed_again.can_prove_freshness(), + "repeated matching failures cannot authorize retained reuse" + ); + } + + #[test] + fn successful_submodule_probe_has_a_distinct_stable_witness() { + let repo = init_repo(&[("src/lib.rs", "pub fn example() {}\n")]); + let disabled = GitMetadataFingerprintV1::for_policy(repo.path(), false); + let enabled = GitMetadataFingerprintV1::for_policy(repo.path(), true); + + assert!(enabled.can_prove_freshness()); + assert!(!disabled.submodule_probe_failed); + assert!(enabled.submodule_refs.is_some()); + assert!(enabled.submodule_indexes.is_some()); + assert_ne!(enabled.stable_signature(), disabled.stable_signature()); + } + #[test] fn git_metadata_fingerprint_detects_a_commit() { let repo = init_repo(&[("src/lib.rs", "pub fn a() {}\n")]); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 4113a4ff273..c9c21b66661 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -2007,7 +2007,9 @@ impl CodeIndexWorktreeSchedulerV1 { return Ok(None); } let metadata = self.freshness_fence.git_metadata(&self.project_root); - if witness.git_metadata_signature != metadata.stable_signature() { + if !metadata.can_prove_freshness() + || witness.git_metadata_signature != metadata.stable_signature() + { return Ok(None); } // Re-swept under the adopted ignored-source roster. Equal metadata is @@ -2063,7 +2065,9 @@ impl CodeIndexWorktreeSchedulerV1 { return false; } let metadata = self.freshness_fence.git_metadata(&self.project_root); - if witness.git_metadata_signature != metadata.stable_signature() { + if !metadata.can_prove_freshness() + || witness.git_metadata_signature != metadata.stable_signature() + { return false; } self.worktree_stat_sweep() @@ -2266,6 +2270,7 @@ impl CodeIndexWorktreeSchedulerV1 { hook(); } let quiet_witness = sealed_bytes_match + && sampled_metadata.can_prove_freshness() && witness.as_ref().is_some_and(|witness| { witness.git_metadata_signature == sampled_metadata.stable_signature() && witness.stat_signature == sampled_sweep.signature From ebd52fe3e494b0cfb03a1bd488c1db280a407dcf Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:36:57 -0800 Subject: [PATCH 28/42] fix(code-index): require valid probes for retained graph reads --- .../src/code_index_scheduler/reconcile.rs | 15 +++-- .../tests/index_path_policy_tests.rs | 65 +++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index c9c21b66661..a820832fb4f 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -663,7 +663,7 @@ impl SourceFreshnessFenceV1 { state.source_witness = source_witness; state.source_roster = source_roster.to_vec(); state.last_reconciled_at = Instant::now(); - state.verified_against_source = true; + state.verified_against_source = state.git_metadata.can_prove_freshness(); state.reconciled_without_generation = reconciled_without_generation; state.reconciled_source_epoch = source_epoch; state.successor_build_in_flight = false; @@ -845,7 +845,7 @@ impl SourceFreshnessFenceV1 { let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); state.git_metadata = git_metadata; state.last_reconciled_at = Instant::now(); - state.verified_against_source = true; + state.verified_against_source = state.git_metadata.can_prove_freshness(); self.last_reconciled_at_micros .store(micros, Ordering::Release); } @@ -883,6 +883,10 @@ impl SourceFreshnessFenceV1 { manifest: &SourceContentManifestV1, shutting_down: &AtomicBool, ) -> (bool, freshness_witness::SourceSweepStatsV1) { + // Equal source bytes cannot authenticate a previously selected child checkout. + if !git_metadata.can_prove_freshness() { + return (false, freshness_witness::SourceSweepStatsV1::default()); + } self.sweep_cache .lock() .unwrap_or_else(PoisonError::into_inner) @@ -2263,14 +2267,15 @@ impl CodeIndexWorktreeSchedulerV1 { // generation's sealed file digests; its matching stat signature is // the negative cache that lets a moved tree skip the byte comparison. let source_manifest = SourceContentManifestV1::for_snapshot(metadata.snapshot()); - let sealed_bytes_match = - retained_is_reusable && !has_hints && self.sources_match_manifest(&source_manifest); + let sealed_bytes_match = retained_is_reusable + && !has_hints + && sampled_metadata.can_prove_freshness() + && self.sources_match_manifest(&source_manifest); #[cfg(test)] if let Some(hook) = self.after_retained_source_check.clone() { hook(); } let quiet_witness = sealed_bytes_match - && sampled_metadata.can_prove_freshness() && witness.as_ref().is_some_and(|witness| { witness.git_metadata_signature == sampled_metadata.stable_signature() && witness.stat_signature == sampled_sweep.signature diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index c0e00751f89..57e30b6b5e0 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -420,3 +420,68 @@ fn parent_filtered_source_does_not_require_submodule_registration_when_disabled( .unwrap(); assert_eq!(sealed.as_deref(), Some(source.as_bytes())); } + +#[test] +fn retained_graph_checks_require_a_successful_submodule_probe_each_time() { + let registration = "[submodule \"child\"]\npath = child\n"; + let parent = GitFixture::new(&[ + ("lib.rs", "pub fn first() {}\n"), + (".gitmodules", registration), + ]); + let store = TempDir::new().unwrap(); + // Keep the sealed attribution worktree-relative across the configuration edit. + std::fs::write(parent.path().join("lib.rs"), "pub fn latest() {}\n").unwrap(); + let mut scheduler = start( + &parent, + store.path(), + policy(&[], &[]).with_submodules(true), + ); + published(scheduler.reconcile_now().unwrap()); + let metadata = scheduler + .publication + .load_active_manifest() + .unwrap() + .unwrap() + .metadata; + // A partially staged edit leaves the worktree and its source bytes readable, + // while the index registration still cannot establish submodule identity. + let registration_path = parent.path().join(".gitmodules"); + std::fs::write(®istration_path, "[unfinished").unwrap(); + super::run_git_in(parent.path(), &["add", ".gitmodules"]); + std::fs::write(®istration_path, registration).unwrap(); + for _ in 0..2 { + assert!( + !scheduler + .freshness_fence() + .git_metadata(parent.path()) + .can_prove_freshness() + ); + let serving_witness = scheduler.currency_witness_for_sealed_snapshot( + &metadata.manifest().generation_id, + &metadata.snapshot().content_identity, + ); + let retained = scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .unwrap(); + assert_eq!( + (serving_witness.is_some(), retained.is_some()), + (false, false), + "unchanged source bytes cannot replace a failed submodule probe" + ); + } + super::run_git_in(parent.path(), &["add", ".gitmodules"]); + assert!(matches!( + scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .unwrap(), + Some(CodeIndexReconcileOutcomeV1::Noop(_)) + )); + assert!( + scheduler + .currency_witness_for_sealed_snapshot( + &metadata.manifest().generation_id, + &metadata.snapshot().content_identity, + ) + .is_some() + ); +} From 478da430ede75cb22cb1f7ffc97a5f14364a0f46 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:37:44 -0800 Subject: [PATCH 29/42] style(code-index): format submodule witness assertion --- .../src/code_index_scheduler/identity.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 39f0813488b..71d215b6b83 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -512,7 +512,10 @@ mod tests { let parent_only = GitMetadataFingerprintV1::capture(root.path()); let failed = GitMetadataFingerprintV1::for_policy(root.path(), true); - assert!(failed.submodule_probe_failed, "opt-in probe must fail closed"); + assert!( + failed.submodule_probe_failed, + "opt-in probe must fail closed" + ); assert!(!failed.can_prove_freshness()); assert!(failed.differs_from(&parent_only)); assert_ne!( From 2b752c786481b9d4e4fa3d6d837459325b7f526f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:31:12 -0800 Subject: [PATCH 30/42] fix(code-index): clear submodule indexing Clippy diagnostics --- .../src/code_index_scheduler/git_tree_capture.rs | 9 +++++---- .../src/code_index_scheduler/identity.rs | 6 ++++-- .../src/code_index_scheduler/ignored_dependencies.rs | 7 +++---- .../tests/index_path_policy_tests.rs | 7 +++++-- crates/tracedecay-code-index/src/source_walk.rs | 4 +++- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs index 922d0c889e9..26b798f880b 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs @@ -35,6 +35,9 @@ use tracedecay_code_index::production::{ CodeIndexRepositoryParseIdentityV1, }; +type TreeBlobVisitor<'a> = + dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1> + 'a; + #[derive(Clone, Debug, PartialEq, Eq)] pub struct ExactGitTreeSourceV1 { pub reference: tracedecay_domain::RefId, @@ -500,7 +503,7 @@ impl CodeIndexWorktreeSchedulerV1 { parent: &gix::Repository, pins: &GitTreeSubmodulePins, control: &branch_generations::BranchGenerationReadControlV1, - visitor: &mut dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1>, + visitor: &mut TreeBlobVisitor<'_>, ) -> Result<(), CodeIndexSearchUnavailableReasonV1> { let unavailable = CodeIndexSearchUnavailableReasonV1::GenerationUnavailable; let children = @@ -596,9 +599,7 @@ impl CodeIndexWorktreeSchedulerV1 { source_revision: Option, tree: tracedecay_domain::TreeId, control: &branch_generations::BranchGenerationReadControlV1, - visit: impl FnOnce( - &mut dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1>, - ) -> Result<(), CodeIndexSearchUnavailableReasonV1>, + visit: impl FnOnce(&mut TreeBlobVisitor<'_>) -> Result<(), CodeIndexSearchUnavailableReasonV1>, ) -> Result { let registry = StaticLanguageRegistry::new(); let mut roster = CapturedFileRosterV1::default(); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 71d215b6b83..ed1378ee977 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -270,10 +270,12 @@ impl GitMetadataFingerprintV1 { ), ); if let Some(refs) = &self.submodule_refs { - signature.push_str(&format!("|submodule_refs={refs}")); + signature.push_str("|submodule_refs="); + signature.push_str(refs); } if let Some(indexes) = &self.submodule_indexes { - signature.push_str(&format!("|submodule_indexes={indexes}")); + signature.push_str("|submodule_indexes="); + signature.push_str(indexes); } if self.submodule_probe_failed { signature.push_str("|submodule_probe_failed=true"); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index 23dacd76f56..f0d6c56fa00 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -1055,10 +1055,9 @@ mod tests { fixture_git(&parent, &["add", ".gitmodules", ".gitignore"]); fixture_git(&parent, &["commit", "-qm", "parent with submodule"]); fixture_git(&parent, &["rm", "--cached", "-f", "node_modules/pkg"]); - assert!( - fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]) - .stdout - .is_empty() + assert_eq!( + fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]).stdout, + b"" ); assert!( fixture_git(&parent, &["ls-tree", "HEAD", "--", "node_modules/pkg"]) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index 57e30b6b5e0..9ec5037f423 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -270,7 +270,10 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() let mut off = start(&parent, store.path(), policy(&[], &[])); off.reconcile_now().unwrap(); assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); - assert!(grep("pinned_symbol", &policy(&[], &[])).is_empty()); + assert_eq!( + grep("pinned_symbol", &policy(&[], &[])), + Vec::::new() + ); drop(off); let mut on = start(&parent, store.path(), enabled.clone()); @@ -308,7 +311,7 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() "const private_marker = 1;", ) .unwrap(); - assert!(grep("private_marker", &enabled).is_empty()); + assert_eq!(grep("private_marker", &enabled), Vec::::new()); std::fs::write( checkout.join("lib.ts"), "export function feature() { return 'successor_symbol'; }\n", diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 3701db9d5f8..e227f3032d4 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -847,7 +847,9 @@ mod tests { &IndexPathPolicyV1::new(vec![], vec![]).unwrap(), ) .expect("source walk with nested repository diagnostics"); - let _visited = walk.map(|entry| entry.expect("source entry")).count(); + for entry in walk { + entry.expect("source entry"); + } assert_eq!( diagnostics.skipped_nested_repositories_note().as_deref(), Some("Skipped nested repositories outside parent index: \"app\"") From 26d7171bf87eebd8cd9a75abc24d9e3107670f44 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:51:17 -0800 Subject: [PATCH 31/42] fix(code-index): preserve retained source boundary refusals --- .../src/code_index_scheduler/reconcile.rs | 19 ++++++++++++++++--- crates/tracedecay-mcp/src/handlers/grep.rs | 16 +++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index a820832fb4f..6e0a7feb15e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -4050,14 +4050,27 @@ impl CodeIndexWorktreeSchedulerV1 { } _ => (self.project_root.clone(), relative), }; - if tracedecay_code_index::source_walk::has_nested_repository_ancestor( + let refusal = if tracedecay_code_index::source_walk::has_nested_repository_ancestor( &source_root, source_relative, - )? || tracedecay_code_index::source_walk::is_source_symlink( + )? { + Some(CodeIndexIgnoredDependencyRefusalV1::NotIgnored) + } else if tracedecay_code_index::source_walk::is_source_symlink( &source_root, source_relative, )? { - return Ok(CapturedFileOutcomeV1::Absent); + Some(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) + } else { + None + }; + if let Some(reason) = refusal { + // A retained admission names required bytes. Omitting them silently + // would turn a source refusal into a broken snapshot contract. + return if explicitly_admitted { + Err(reason.into()) + } else { + Ok(CapturedFileOutcomeV1::Absent) + }; } match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), diff --git a/crates/tracedecay-mcp/src/handlers/grep.rs b/crates/tracedecay-mcp/src/handlers/grep.rs index 1cb09655889..1d09dc452ee 100644 --- a/crates/tracedecay-mcp/src/handlers/grep.rs +++ b/crates/tracedecay-mcp/src/handlers/grep.rs @@ -477,11 +477,17 @@ mod tests { "// UNIQUE_GENERATED_DIR_TOKEN\n", ) .expect("source fixture"); - std::fs::write( - project.path().join(".git"), - "gitdir: UNIQUE_GENERATED_DIR_TOKEN\n", - ) - .expect("linked-worktree git file fixture"); + let administration = tempfile::tempdir().expect("isolated Git administration"); + let git = tracedecay_runtime_core::git::try_git_program().expect("Git program"); + let initialized = std::process::Command::new(git) + .args(["init", "--quiet", "--template=", "--separate-git-dir"]) + .arg(administration.path().join("UNIQUE_GENERATED_DIR_TOKEN")) + .arg(project.path()) + .env("HOME", administration.path()) + .env("GIT_CONFIG_NOSYSTEM", "1") + .status() + .expect("initialize checkout with a Git pointer file"); + assert!(initialized.success()); std::fs::write( project.path().join(".tracedecay/internal.txt"), "UNIQUE_GENERATED_DIR_TOKEN\n", From da69c12c2a3b511cdf33d02c2377e8ff5dc0b294 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:18:56 -0800 Subject: [PATCH 32/42] fix(code-index): retain source aliases within the parent checkout --- .../code_index_scheduler/freshness_witness.rs | 41 ++++---- .../code_index_scheduler/git_tree_capture.rs | 31 ++++-- .../src/code_index_scheduler/reconcile.rs | 35 +++---- .../tracedecay-code-index/src/source_walk.rs | 8 +- .../src/source_walk/classification.rs | 60 +++++++++--- .../src/source_walk/submodules.rs | 15 ++- .../src/source_walk/tracked_links.rs | 94 +++++++++++++++++++ .../src/git_repository.rs | 2 +- .../src/git_repository/tree_links.rs | 67 +++++++++---- docs/USER-GUIDE.md | 4 +- 10 files changed, 269 insertions(+), 88 deletions(-) create mode 100644 crates/tracedecay-code-index/src/source_walk/tracked_links.rs diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 00228326aab..fe936ae0327 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -248,18 +248,6 @@ fn read_candidate( project_root: &Path, candidate: &StatCandidateV1, ) -> Result, CodeIndexSchedulerErrorV1> { - // A link installed after candidate classification must not turn a - // freshness proof into a read of a different repository's source bytes. - if tracedecay_code_index::source_walk::is_source_symlink( - project_root, - Path::new(&candidate.logical_path), - )? { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "source symlink is outside the parent source inventory", - ) - .into()); - } if candidate.explicitly_admitted { ignored_dependencies::read_explicitly_admitted_source( project_root, @@ -267,13 +255,22 @@ fn read_candidate( None, ) } else { - ignored_dependencies::read_bounded_snapshot_source( - &project_root.join(&candidate.logical_path), - None, - ) + let path = resolved_candidate_path(project_root, &candidate.logical_path)?; + ignored_dependencies::read_bounded_snapshot_source(&path, None) } } +fn resolved_candidate_path(root: &Path, logical_path: &str) -> std::io::Result { + tracedecay_code_index::source_walk::resolve_parent_source(root, Path::new(logical_path))? + .map(|relative| root.join(relative)) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "source alias is outside the parent Git inventory", + ) + }) +} + /// The canonical content digest capture would seal for `raw`: the digest of /// the sanitized bytes. fn sanitized_digest( @@ -956,13 +953,17 @@ fn filter_mismatches_in_repository<'a>( let Ok((mut pipeline, index)) = repository.filter_pipeline(None) else { return disputed; }; + let repository_root = project_root.join(prefix); disputed .into_iter() .filter(|candidate| { let Some(expected) = manifest.files.get(&candidate.logical_path) else { return true; }; - let Ok(rela_path) = Path::new(&candidate.logical_path).strip_prefix(prefix) else { + let Ok(path) = resolved_candidate_path(project_root, &candidate.logical_path) else { + return true; + }; + let Ok(rela_path) = path.strip_prefix(&repository_root) else { return true; }; if candidate.explicitly_admitted @@ -972,7 +973,7 @@ fn filter_mismatches_in_repository<'a>( { return true; } - let Ok(file) = std::fs::File::open(project_root.join(&candidate.logical_path)) else { + let Ok(file) = std::fs::File::open(&path) else { return true; }; let Ok(mut converted) = pipeline.convert_to_git(file, rela_path, &index) else { @@ -1004,7 +1005,7 @@ pub fn sealed_source_bytes( let Some((root, relative)) = boundaries.resolve(Path::new(logical_path))? else { return Ok(None); }; - let raw = ignored_dependencies::read_bounded_snapshot_source(&root.join(relative), None)?; + let raw = ignored_dependencies::read_bounded_snapshot_source(&root.join(&relative), None)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &raw)?; if content_digest(&sanitized) == *sealed { return Ok(Some(sanitized)); @@ -1022,7 +1023,7 @@ pub fn sealed_source_bytes( } let mut filtered = Vec::new(); pipeline - .convert_to_git(raw.as_slice(), relative, &index) + .convert_to_git(raw.as_slice(), relative.as_ref(), &index) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? .read_to_end(&mut filtered)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &filtered)?; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs index 26b798f880b..ef4a82f1335 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs @@ -1403,15 +1403,32 @@ mod tests { .snapshot, ); assert_eq!( - live.iter() - .map(|(path, _)| path.as_str()) - .collect::>(), - ["AGENTS.md", "src/real.rs"] + branch, live, + "same-parent source aliases resolve to the live checkout's bytes" ); - for regular in live { + let changed = b"pub fn linked_target() -> u32 { 19 }\n"; + std::fs::write(root.join("src/real.rs"), changed).expect("edit only the alias target"); + let edited = present( + &scheduler + .capture_authoritative_snapshot_without_active_generation_reuse(None) + .expect("capture unstaged target through each tracked alias") + .snapshot, + ); + let expected = tracedecay_code_index::chunks::content_digest(changed); + for alias in [ + "src/alias.rs", + "src/nested/chained.rs", + "src/through_directory.rs", + ] { + assert!( + edited + .iter() + .any(|(path, digest)| path == alias && *digest == expected) + ); assert!( - branch.contains(®ular), - "historical and live regular files must carry the same source digest" + branch + .iter() + .any(|(path, digest)| path == alias && *digest != expected) ); } let notes = captured diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 6e0a7feb15e..56e5ea5a844 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -4050,28 +4050,29 @@ impl CodeIndexWorktreeSchedulerV1 { } _ => (self.project_root.clone(), relative), }; - let refusal = if tracedecay_code_index::source_walk::has_nested_repository_ancestor( + let resolved = if tracedecay_code_index::source_walk::has_nested_repository_ancestor( &source_root, source_relative, )? { - Some(CodeIndexIgnoredDependencyRefusalV1::NotIgnored) - } else if tracedecay_code_index::source_walk::is_source_symlink( - &source_root, - source_relative, - )? { - Some(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) + Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored) + } else if explicitly_admitted + && tracedecay_code_index::source_walk::is_source_symlink(&source_root, source_relative)? + { + Err(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) } else { - None + tracedecay_code_index::source_walk::resolve_parent_source( + &source_root, + source_relative, + )? + .ok_or(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) }; - if let Some(reason) = refusal { - // A retained admission names required bytes. Omitting them silently - // would turn a source refusal into a broken snapshot contract. - return if explicitly_admitted { - Err(reason.into()) - } else { - Ok(CapturedFileOutcomeV1::Absent) - }; - } + let resolved = match resolved { + Ok(path) => path, + // Retained admissions name required bytes. A refusal must remain visible. + Err(reason) if explicitly_admitted => return Err(reason.into()), + Err(_) => return Ok(CapturedFileOutcomeV1::Absent), + }; + let absolute = source_root.join(resolved); match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), Ok(_) => {} diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index e227f3032d4..93b6b3847e3 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -11,6 +11,8 @@ pub mod classification; pub mod submodules; +mod tracked_links; +pub use tracked_links::resolve_parent_source; use std::collections::BTreeSet; use std::path::{Component, Path, PathBuf}; @@ -253,9 +255,7 @@ pub fn has_nested_repository_ancestor( Ok(false) } -/// A worktree source entry must be a real file, not a link to bytes outside -/// the parent's Git authority. The shared live walker does not follow leaf -/// symlinks; classification, capture, and freshness must make the same choice. +/// Distinguish leaf links before selecting their source authority. /// Missing paths remain possible tombstones and are not symlink exclusions. pub fn is_source_symlink(project_root: &Path, relative_path: &Path) -> std::io::Result { if relative_path.as_os_str().is_empty() @@ -616,7 +616,7 @@ mod tests { #[cfg(unix)] #[test] - fn readable_symlink_leaf_is_not_an_admissible_source() { + fn leaf_link_detection_preserves_missing_paths() { use std::os::unix::fs::symlink; let root = TempDir::new().unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk/classification.rs b/crates/tracedecay-code-index/src/source_walk/classification.rs index 931bdf5329e..2042361ca32 100644 --- a/crates/tracedecay-code-index/src/source_walk/classification.rs +++ b/crates/tracedecay-code-index/src/source_walk/classification.rs @@ -18,7 +18,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use super::{ - has_nested_repository_ancestor, is_source_symlink, parent_gitlink_paths, + has_nested_repository_ancestor, parent_gitlink_paths, resolve_parent_source, submodules::live_submodules, }; use gix::bstr::ByteSlice; @@ -241,12 +241,8 @@ impl WorktreeChangeClassificationV1 { { let relative = Path::new(path); let parent = relative.parent().unwrap_or_else(|| Path::new("")); - // The live walker never follows a symlink file, even when - // the lexical parent is ordinary and the target is inside a - // registered submodule. Do not capture its target through a - // clean-tree or worktree path. let excluded = if classification.beneath_gitlink(relative) - || is_source_symlink(project_root, relative)? + || resolve_parent_source(project_root, relative)?.is_none() { true } else { @@ -621,7 +617,19 @@ mod nested_repository_tests { ], ); symlink("app/src/lib.rs", parent.join("bridge.rs")).unwrap(); - fixture_git(&parent, &["add", ".gitmodules", "app", "bridge.rs"]); + fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); + symlink("parent.rs", parent.join("local.rs")).unwrap(); + fixture_git( + &parent, + &[ + "add", + ".gitmodules", + "app", + "bridge.rs", + "parent.rs", + "local.rs", + ], + ); fixture_git( &parent, &["commit", "-qm", "registered child and source alias"], @@ -639,14 +647,40 @@ mod nested_repository_tests { ); let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); - let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); - assert!(!classified.candidate_paths().contains("bridge.rs")); - assert!(classified.changed_paths().contains("bridge.rs")); + for include_submodules in [false, true] { + let classified = + WorktreeChangeClassificationV1::for_policy(&repository, include_submodules) + .unwrap(); + let candidates = classified.candidate_paths(); + let boundaries = crate::source_walk::submodules::LiveSourceBoundaries::open( + &parent, + include_submodules, + ) + .unwrap(); + let (_, relative) = boundaries.resolve(Path::new("local.rs")).unwrap().unwrap(); + assert_eq!(relative.as_ref(), Path::new("parent.rs")); + assert!( + boundaries + .resolve(Path::new("bridge.rs")) + .unwrap() + .is_none() + ); + assert!(candidates.contains("local.rs")); + assert!(!candidates.contains("bridge.rs")); + assert!(classified.changed_paths().contains("bridge.rs")); + assert!( + classified.has_changes(), + "do not reuse HEAD tree source bytes" + ); + assert_eq!(candidates.contains("app/src/lib.rs"), include_submodules); + } + symlink("parent.rs", parent.join("untracked_alias.rs")).unwrap(); + let classification = WorktreeChangeClassificationV1::classify(&repository).unwrap(); assert!( - classified.has_changes(), - "do not reuse HEAD tree source bytes" + !classification + .candidate_paths() + .contains("untracked_alias.rs") ); - assert!(!classified.candidate_paths().contains("app/src/lib.rs")); } #[test] diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 461f79aff41..0c068e8d328 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -1,6 +1,7 @@ //! A parent gitlink names a commit, not the worktree that may supply its bytes. //! Bind the registration, object store, and checkout before crossing that boundary. +use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Component, Path, PathBuf}; @@ -225,7 +226,7 @@ impl LiveSourceBoundaries { pub fn resolve<'a, 'p>( &'a self, path: &'p Path, - ) -> std::io::Result> { + ) -> std::io::Result)>> { if !is_source_location(path) { return Ok(None); } @@ -259,12 +260,16 @@ impl LiveSourceBoundaries { root = &child.worktree; break; } - if has_nested_repository_ancestor(root, relative)? - || super::is_source_symlink(root, relative)? - { + if has_nested_repository_ancestor(root, relative)? { return Ok(None); } - Ok(Some((root, relative))) + if root == self.project_root { + Ok(super::resolve_parent_source(root, relative)?.map(|path| (root, path))) + } else if super::is_source_symlink(root, relative)? { + Ok(None) + } else { + Ok(Some((root, Cow::Borrowed(relative)))) + } } } diff --git a/crates/tracedecay-code-index/src/source_walk/tracked_links.rs b/crates/tracedecay-code-index/src/source_walk/tracked_links.rs new file mode 100644 index 00000000000..35d47fa9149 --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/tracked_links.rs @@ -0,0 +1,94 @@ +//! Resolve a tracked source alias using the parent Git index and live worktree. + +use std::borrow::Cow; +use std::io; +use std::path::{Component, Path}; + +use gix::bstr::ByteSlice; +use gix::index::entry::{Mode, Stage}; +use tracedecay_runtime_core::git_repository::{RepositoryLinkEntry, resolve_repository_link}; + +use super::{is_source_symlink, nested_repository_marker, submodules::is_source_location}; + +/// Ordinary files keep their original path. A live parent alias may only read +/// a tracked regular file through tracked links and parent-owned directories. +pub fn resolve_parent_source<'a>( + root: &Path, + relative: &'a Path, +) -> io::Result>> { + if !is_source_symlink(root, relative)? { + return Ok(Some(Cow::Borrowed(relative))); + } + if !nested_repository_marker(root)? { + return Ok(None); + } + let repo = tracedecay_runtime_core::git_open::open(root).map_err(io::Error::other)?; + let index = repo.index_or_empty().map_err(io::Error::other)?; + let head = repo.head().map_err(io::Error::other)?; + let tree = if head.id().is_some() { + Some(repo.head_tree().map_err(io::Error::other)?) + } else { + None + }; + let alias = gix::path::into_bstr(relative).map_err(io::Error::other)?; + let alias = gix::path::to_unix_separators_on_windows(alias); + if !index + .entry_by_path_and_stage(&alias, Stage::Unconflicted) + .is_some_and(|entry| entry.mode == Mode::SYMLINK) + { + return Ok(None); + } + resolve_repository_link(b"", &alias, |components| { + let bytes = components.join(&b'/'); + let path = gix::path::from_byte_slice(&bytes).map_err(io::Error::other)?; + if !is_source_location(path) { + return Ok(None); + } + let absolute = root.join(path); + let metadata = match absolute.symlink_metadata() { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let indexed = index.entry_by_path_and_stage(bytes.as_bstr(), Stage::Unconflicted); + if metadata.is_dir() { + // A gitlink staged for removal still bounds the old checkout even + // after its marker disappears. Check HEAD before descending. + let head_entry = tree + .as_ref() + .map(|tree| tree.lookup_entry(components.iter().map(Vec::as_slice))) + .transpose() + .map_err(io::Error::other)? + .flatten(); + if index.entry_by_path(bytes.as_bstr()).is_some() + || head_entry.is_some_and(|entry| entry.mode().is_commit()) + || nested_repository_marker(&absolute)? + { + return Ok(None); + } + return Ok(Some(RepositoryLinkEntry::Directory)); + } + let Some(entry) = indexed else { + return Ok(None); + }; + if metadata.file_type().is_symlink() && entry.mode == Mode::SYMLINK { + let target = std::fs::read_link(&absolute)?; + if target + .components() + .any(|component| matches!(component, Component::Prefix(_) | Component::RootDir)) + { + return Ok(None); + } + let target = gix::path::into_bstr(target).map_err(io::Error::other)?; + return Ok(Some(RepositoryLinkEntry::Link( + gix::path::to_unix_separators_on_windows(target).to_vec(), + ))); + } + Ok((metadata.is_file() + && entry + .mode + .to_tree_entry_mode() + .is_some_and(|mode| mode.is_blob())) + .then(|| RepositoryLinkEntry::File(Cow::Owned(path.to_owned())))) + }) +} diff --git a/crates/tracedecay-runtime-core/src/git_repository.rs b/crates/tracedecay-runtime-core/src/git_repository.rs index bfb65d12122..e978a2cce04 100644 --- a/crates/tracedecay-runtime-core/src/git_repository.rs +++ b/crates/tracedecay-runtime-core/src/git_repository.rs @@ -35,7 +35,7 @@ pub use native_integration::{ GitNativeIntegrationMode, GitNativePreflight, GitNativePreflightCaptureError, GitNativePreflightDisposition, GitNativeUnsupportedReason, GitTreeSubmodulePins, }; -pub use tree_links::resolve_tree_link; +pub use tree_links::{RepositoryLinkEntry, resolve_repository_link, resolve_tree_link}; /// A typed failure from the in-process Git repository authority. #[derive(Clone, Debug, thiserror::Error)] diff --git a/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs b/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs index ccee3e049f4..31e81fe6525 100644 --- a/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs +++ b/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs @@ -15,6 +15,42 @@ pub fn resolve_tree_link( link_path: &[u8], target: &[u8], ) -> Result, GitRepositoryError> { + resolve_repository_link(link_path, target, |path| { + let Some(entry) = tree + .lookup_entry(path.iter().map(Vec::as_slice)) + .map_err(|error| operation("tree link lookup", error))? + else { + return Ok(None); + }; + let mode = entry.mode(); + Ok(if mode.is_tree() { + Some(RepositoryLinkEntry::Directory) + } else if mode.is_link() { + let link = entry + .object() + .map_err(|error| operation("tree link target", error))?; + Some(RepositoryLinkEntry::Link(link.data.clone())) + } else if mode.is_blob() { + Some(RepositoryLinkEntry::File(entry.object_id())) + } else { + None + }) + }) +} + +pub enum RepositoryLinkEntry { + Directory, + Link(Vec), + File(T), +} + +/// Walk relative link components with the caller's repository authority. Each +/// directory is checked before descending, including one followed by `..`. +pub fn resolve_repository_link( + link_path: &[u8], + target: &[u8], + mut lookup: impl FnMut(&[Vec]) -> Result>, E>, +) -> Result, E> { let mut resolved = components(link_path).collect::>(); resolved.pop(); let mut pending = Vec::new(); @@ -33,26 +69,17 @@ pub fn resolve_tree_link( } _ => resolved.push(component), } - let Some(entry) = tree - .lookup_entry(resolved.iter().map(Vec::as_slice)) - .map_err(|error| operation("tree link lookup", error))? - else { - return Ok(None); - }; - let mode = entry.mode(); - if mode.is_tree() { - continue; - } - if !mode.is_link() { - return Ok((pending.is_empty() && mode.is_blob()).then(|| entry.object_id())); - } - hops += 1; - resolved.pop(); - let link = entry - .object() - .map_err(|error| operation("tree link target", error))?; - if hops > MAX_LINK_HOPS || !push_target(&mut pending, &link.data) { - return Ok(None); + match lookup(&resolved)? { + Some(RepositoryLinkEntry::Directory) => {} + Some(RepositoryLinkEntry::Link(target)) => { + hops += 1; + resolved.pop(); + if hops > MAX_LINK_HOPS || !push_target(&mut pending, &target) { + return Ok(None); + } + } + Some(RepositoryLinkEntry::File(file)) if pending.is_empty() => return Ok(Some(file)), + _ => return Ok(None), } } Ok(None) diff --git a/docs/USER-GUIDE.md b/docs/USER-GUIDE.md index 15a3ee90140..5ef9797bc9d 100644 --- a/docs/USER-GUIDE.md +++ b/docs/USER-GUIDE.md @@ -219,7 +219,9 @@ tracedecay tool configuration_set --args '{"layer":{"kind":"project","project_id The parent index and `.gitmodules` must identify the child, and its checkout must use the matching repository under the parent's Git modules directory. -Live indexing includes child worktree changes. Historical indexing reads the +Live indexing includes child worktree changes. The parent graph retains tracked +source aliases that resolve through tracked links to a tracked parent file. A +parent alias cannot cross into a child checkout. Historical indexing reads the child commit recorded by the selected parent revision from the local object store. Initialize the submodule locally before requesting that history. Linked worktrees, unrelated clones, and deeper nested repositories stay From f36fbb3841056efbef51057d6e6fdc9573bc81b2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:38:44 -0800 Subject: [PATCH 33/42] test(code-index): require clean revisions for tracked aliases --- .../tests/code_index_scheduler/reconcile.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs index 5aa18b0df2a..318dc166f93 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs @@ -825,11 +825,11 @@ fn unfollowable_committed_source_links_are_absent_from_disk_capture() { ); } -/// A readable live symlink stays outside the worktree roster. The remaining -/// files can still establish a current generation without naming the full HEAD tree. +/// Publishing the link target bytes under both tracked paths must preserve +/// the clean HEAD revision and avoid a redundant post-publication rebuild. #[cfg(unix)] #[test] -fn committed_source_symlink_publishes_current_regular_files_only() { +fn committed_source_symlink_publishes_a_current_clean_generation() { let fixture = GitFixture::new(&[ ("src/lib.rs", "pub mod pricing;\n"), ("src/pricing.rs", "pub fn total() -> u64 { 1 }\n"), @@ -851,14 +851,17 @@ fn committed_source_symlink_publishes_current_regular_files_only() { ); let latest = scheduler.latest_complete().expect("published generation"); let snapshot = latest.generation.snapshot(); - assert!(snapshot.source_revision.is_none()); + assert_eq!( + snapshot.source_revision.as_ref(), + scheduler.identity.head_commit() + ); assert_eq!( snapshot .files .iter() .map(|file| file.logical_path.as_str()) .collect::>(), - ["src/lib.rs", "src/pricing.rs"] + ["src/lib.rs", "src/pricing.rs", "src/pricing_alias.rs"] ); } From 88ca6c1dee8bb9cea75aaea30370834226d4f04d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:46:37 -0800 Subject: [PATCH 34/42] fix(code-index): preserve authenticated child capture on Windows Refs #4593 Normalize the existing child-root canonical identity before comparing it to the scheduler's plain path. Windows canonicalize returns a verbatim disk path even for the same checkout; the raw comparison silently dropped live child candidates after a child HEAD advance. Keep the strict existing-root/no-symlink boundary. The already-failing native Windows submodule graph regression exercises this path; run it on the new head with Bazel before accepting. --- .../src/code_index_scheduler/reconcile.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 56e5ea5a844..c2028cd1243 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -4036,7 +4036,14 @@ impl CodeIndexWorktreeSchedulerV1 { return Ok(CapturedFileOutcomeV1::Absent); } let root = self.project_root.join(prefix); - if root.canonicalize().ok().as_ref() != Some(&root) + // The scheduler stores a plain canonical path. Windows + // canonicalize() returns a verbatim \\?\\ disk path, so compare + // through the same existing-root identity without relaxing + // the no-symlink/no-escape boundary. + if tracedecay_runtime_core::path_safety::canonical_existing_identity(&root) + .ok() + .as_ref() + != Some(&root) || !absolute .symlink_metadata() .is_ok_and(|metadata| metadata.is_file()) From f0874897e1d49b9542fe12d2c3735b32e68b53d6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:27:12 -0800 Subject: [PATCH 35/42] fix(code-index): revalidate lazy submodule source walks Retain each authenticated child binding while its source iterator is alive. Reuse the graph reader's checkout/marker liveness proof before advancing the child walk, including the depth-zero root. Return a typed walk error and permanently stop that child if its binding changes. Add real-Git regressions for marker removal, retargeting, mid-iteration binding loss and a replaced root, with positive parent/child controls. Rust regression execution and native verification remain required. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 23 ++- .../src/source_walk/submodules.rs | 76 +++++-- .../src/source_walk/submodules/tests.rs | 194 ++++++++++++++++++ 3 files changed, 270 insertions(+), 23 deletions(-) create mode 100644 crates/tracedecay-code-index/src/source_walk/submodules/tests.rs diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 93b6b3847e3..aa71768a395 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -18,11 +18,11 @@ use std::collections::BTreeSet; use std::path::{Component, Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; +use ignore::WalkBuilder; use ignore::overrides::{Override, OverrideBuilder}; -use ignore::{Walk, WalkBuilder}; use tracedecay_domain::{IndexPathPolicyV1, forward_slash_path}; -pub type SourceWalk = std::iter::Flatten>; +pub type SourceWalk = std::iter::Flatten>; #[derive(Debug)] pub struct SourceWalkError { @@ -401,6 +401,7 @@ fn source_walk_inner( let mut parent_gitlinks = boundaries.gitlinks; let mut authenticated = BTreeSet::new(); let mut child_files = BTreeSet::new(); + let mut roots = Vec::new(); for (path, child) in boundaries.children { let absolute = project_root.join(&path); let policy = if scoped_policy(&absolute, true) { @@ -427,12 +428,9 @@ fn source_walk_inner( parent_gitlinks.insert(format!("{path}/{nested}")); } authenticated.insert(path); + roots.push((absolute, child)); } - let roots = authenticated - .iter() - .map(|path| project_root.join(path)) - .collect::>(); let child_dirs = child_files .iter() .flat_map(|path| Path::new(path).ancestors().skip(1)) @@ -497,11 +495,16 @@ fn source_walk_inner( Err(_) => false, } }); - let mut builders = vec![(WalkBuilder::new(project_root), false)]; - builders.extend(roots.into_iter().map(|root| (WalkBuilder::new(root), true))); + let mut builders = vec![(WalkBuilder::new(project_root), None)]; + builders.extend( + roots + .into_iter() + .map(|(root, child)| (WalkBuilder::new(root), Some(child))), + ); let walks = builders .into_iter() - .map(|(mut builder, in_child)| { + .map(|(mut builder, child)| { + let in_child = child.is_some(); // The child Git roster already applied that checkout's ignores, including // tracked exceptions. Running parent ignore rules over it would hide those files. if in_child { @@ -517,7 +520,7 @@ fn source_walk_inner( if let Some(overrides) = &overrides { builder.overrides(overrides.clone()); } - builder.build() + submodules::CheckoutWalk::new(builder.build(), child) }) .collect::>(); Ok(walks.into_iter().flatten()) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 0c068e8d328..e7811a6dabc 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -3,9 +3,11 @@ use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; +use std::io; use std::path::{Component, Path, PathBuf}; use gix::bstr::ByteSlice; +use ignore::{DirEntry, Error as WalkError, Walk}; use super::{SourceWalkError, has_nested_repository_ancestor}; @@ -29,6 +31,19 @@ pub(super) struct ChildSources { } impl AuthenticatedSubmodule { + fn checkout_is_live(&self) -> bool { + let marker = self.worktree.join(".git"); + self.worktree.canonicalize().ok().as_ref() == Some(&self.worktree) + && marker + .symlink_metadata() + .is_ok_and(|metadata| metadata.is_file()) + && gix::discover::path::from_gitdir_file(&marker) + .ok() + .and_then(|target| target.canonicalize().ok()) + .as_ref() + == Some(&self.git_dir) + } + pub(super) fn sources(&self) -> Result { let repository = tracedecay_runtime_core::git_open::open(&self.git_dir).map_err(failure)?; let inventory = @@ -43,6 +58,50 @@ impl AuthenticatedSubmodule { } } +/// A lazy walk that cannot outlive its authenticated child checkout binding. +/// The ordinary parent walk has no child binding and retains its own policy. +pub struct CheckoutWalk { + walk: Option, + child: Option, +} + +impl CheckoutWalk { + pub(super) fn new(walk: Walk, child: Option) -> Self { + Self { + walk: Some(walk), + child, + } + } +} + +impl Iterator for CheckoutWalk { + type Item = Result; + + fn next(&mut self) -> Option { + self.walk.as_ref()?; + if let Some(child) = &self.child + && !child.checkout_is_live() + { + // Check before polling, including the root entry: ignore does not + // apply filter_entry to a walk's root. Stop permanently after a + // binding change, even if the original marker is later restored. + self.walk = None; + return Some(Err(WalkError::WithPath { + path: child.worktree.clone(), + err: Box::new(WalkError::Io(io::Error::new( + io::ErrorKind::InvalidData, + "authenticated submodule checkout binding changed during source walk", + ))), + })); + } + let next = self.walk.as_mut()?.next(); + if next.is_none() { + self.walk = None; + } + next + } +} + pub(super) struct CheckoutBoundaries { pub gitlinks: BTreeSet, pub children: BTreeMap, @@ -241,19 +300,7 @@ impl LiveSourceBoundaries { }; relative = path.strip_prefix(ancestor).map_err(std::io::Error::other)?; if !relative.to_str().is_some_and(|file| files.contains(file)) - || child.worktree.canonicalize().ok().as_ref() != Some(&child.worktree) - { - return Ok(None); - } - let marker = child.worktree.join(".git"); - if !marker - .symlink_metadata() - .is_ok_and(|metadata| metadata.is_file()) - || gix::discover::path::from_gitdir_file(&marker) - .ok() - .and_then(|target| target.canonicalize().ok()) - .as_ref() - != Some(&child.git_dir) + || !child.checkout_is_live() { return Ok(None); } @@ -300,3 +347,6 @@ pub fn pinned_submodules( } Ok(children) } + +#[cfg(test)] +mod tests; diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs new file mode 100644 index 00000000000..2766aeb4913 --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -0,0 +1,194 @@ +use std::collections::BTreeSet; +use std::fs; +use std::io; +use std::path::Path; +use std::process::Command; + +use ignore::Error as WalkError; +use tempfile::TempDir; +use tracedecay_domain::IndexPathPolicyV1; + +use super::LiveSourceBoundaries; +use crate::source_walk::{SourceWalk, source_walk}; + +fn git(root: &Path, args: &[&str]) { + let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git"); + let mut command = Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run fixture Git"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +fn fixture() -> TempDir { + let fixture = TempDir::new().unwrap(); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(child.join("nested")).unwrap(); + git(&parent, &["init", "-q"]); + git(&child, &["init", "-q"]); + fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); + fs::write(child.join("first.rs"), "pub fn first() {}\n").unwrap(); + fs::write(child.join("nested/second.rs"), "pub fn second() {}\n").unwrap(); + git(&child, &["add", "."]); + git(&child, &["commit", "-qm", "child source"]); + git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + git(&parent, &["add", "."]); + git(&parent, &["commit", "-qm", "parent with child"]); + fixture +} + +fn walk(root: &Path) -> SourceWalk { + let policy = IndexPathPolicyV1::new(vec![], vec![]) + .unwrap() + .with_submodules(true); + source_walk(root, None, &policy).expect("authenticated source walk") +} + +fn assert_live_control(root: &Path) { + let files = walk(root) + .map(|entry| entry.expect("unchanged checkout")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .map(|entry| entry.path().strip_prefix(root).unwrap().to_path_buf()) + .collect::>(); + for source in ["parent.rs", "app/first.rs", "app/nested/second.rs"] { + assert!(files.contains(Path::new(source)), "{source}: {files:?}"); + } +} + +fn assert_binding_error(error: &WalkError) { + assert_eq!( + error.io_error().expect("typed binding refusal").kind(), + io::ErrorKind::InvalidData + ); + assert!( + error.to_string().contains("checkout binding changed"), + "{error}" + ); +} + +fn assert_refused_walk(root: &Path, walk: SourceWalk) { + let mut files = BTreeSet::new(); + let mut errors = 0; + for entry in walk { + match entry { + Ok(entry) if entry.file_type().is_some_and(|kind| kind.is_file()) => { + files.insert(entry.path().strip_prefix(root).unwrap().to_path_buf()); + } + Ok(_) => {} + Err(error) => { + assert_binding_error(&error); + errors += 1; + } + } + } + assert_eq!(errors, 1, "binding loss must not be a successful empty walk"); + assert!(files.contains(Path::new("parent.rs"))); + assert!(files.iter().all(|path| !path.starts_with("app"))); +} + +fn marker_change_before_iteration(replacement: Option<&str>) { + let fixture = fixture(); + let root = fixture.path().join("parent"); + assert_live_control(&root); + let boundaries = LiveSourceBoundaries::open(&root, true).unwrap(); + let path = Path::new("app/first.rs"); + assert!(boundaries.resolve(path).unwrap().is_some()); + let pending = walk(&root); + let marker = root.join("app/.git"); + match replacement { + Some(contents) => fs::write(&marker, contents).unwrap(), + None => fs::remove_file(&marker).unwrap(), + } + assert!(root.join(path).is_file(), "the child bytes are still present"); + assert!(boundaries.resolve(path).unwrap().is_none()); + assert_refused_walk(&root, pending); +} + +#[test] +fn lazy_submodule_walk_refuses_removed_marker() { + marker_change_before_iteration(None); +} + +#[test] +fn lazy_submodule_walk_refuses_retargeted_marker() { + // The source repository has identical tracked files but is not the + // parent's previously authenticated module store. + marker_change_before_iteration(Some("gitdir: ../../child/.git\n")); +} + +#[test] +fn lazy_submodule_walk_stops_after_mid_iteration_binding_loss() { + let fixture = fixture(); + let root = fixture.path().join("parent"); + assert_live_control(&root); + let mut pending = walk(&root); + let first = pending + .by_ref() + .map(|entry| entry.expect("unchanged checkout")) + .find(|entry| { + entry.file_type().is_some_and(|kind| kind.is_file()) + && entry.path().starts_with(root.join("app")) + }) + .expect("at least one child source was yielded"); + assert!(first.path().is_file()); + let marker = root.join("app/.git"); + let original = fs::read(&marker).unwrap(); + fs::remove_file(&marker).unwrap(); + let error = pending + .next() + .expect("binding refusal after a yielded child file") + .expect_err("must revalidate before advancing the child walk"); + assert_binding_error(&error); + fs::write(&marker, original).unwrap(); + assert!(pending.next().is_none(), "a failed walk cannot resume"); +} + +#[cfg(unix)] +#[test] +fn lazy_submodule_walk_refuses_replaced_root_before_root_entry() { + let fixture = fixture(); + let root = fixture.path().join("parent"); + assert_live_control(&root); + let pending = walk(&root); + let app = root.join("app"); + fs::rename(&app, fixture.path().join("detached-child")).unwrap(); + std::os::unix::fs::symlink(fixture.path().join("child"), &app).unwrap(); + assert!(app.join("first.rs").is_file()); + // ignore does not filter its depth-zero entry. The guard must run + // before polling this separate child root, not only in filter_entry. + assert_refused_walk(&root, pending); +} From 7473407babeb56d5d2eb7916e98c190b17f871ca Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:32:26 -0800 Subject: [PATCH 36/42] fix(code-index): require explicit sealed submodule policy Refs #4593 Reject older generation parse identities without an include_submodules field rather than silently treating them as default-off. Seal both true and false explicitly, with a regression for round-trip and omitted-field refusal. Apply the two exact Bazel rustfmt corrections reported for the new authenticated submodule source-walk regressions. Native Bazel behavioral tests and independent review remain required. --- .../src/production/ignored_sources.rs | 43 ++++++++++++++++++- .../src/source_walk/submodules/tests.rs | 10 ++++- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-code-index/src/production/ignored_sources.rs b/crates/tracedecay-code-index/src/production/ignored_sources.rs index beafc93a382..438ec1c1d72 100644 --- a/crates/tracedecay-code-index/src/production/ignored_sources.rs +++ b/crates/tracedecay-code-index/src/production/ignored_sources.rs @@ -53,8 +53,8 @@ pub struct CodeIndexBuildRequestV1 { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct CodeIndexRepositoryParseIdentityV1 { - /// Older sealed generations used the default parent-only roster. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] + /// Every sealed generation records its admission policy explicitly. + /// Missing policy evidence is an incompatible persisted shape, not default-off. pub include_submodules: bool, pub tree: Option, pub dirty: RepositoryDirtyStateV1, @@ -185,3 +185,42 @@ impl CodeIndexPublishedGenerationV1 { self.ignored_source_roster.digest() } } + +#[cfg(test)] +mod parse_identity_tests { + use super::CodeIndexRepositoryParseIdentityV1; + use tracedecay_domain::RepositoryDirtyStateV1; + + #[test] + fn sealed_parse_identity_requires_an_explicit_submodule_policy() { + for include_submodules in [false, true] { + let identity = CodeIndexRepositoryParseIdentityV1 { + include_submodules, + tree: None, + dirty: RepositoryDirtyStateV1::Clean, + }; + let serialized = serde_json::to_value(&identity).expect("serialize parse identity"); + assert_eq!( + serialized + .get("include_submodules") + .and_then(serde_json::Value::as_bool), + Some(include_submodules) + ); + let restored: CodeIndexRepositoryParseIdentityV1 = + serde_json::from_value(serialized.clone()).expect("explicit policy round trip"); + assert_eq!(restored, identity); + + let mut missing_policy = serialized; + missing_policy + .as_object_mut() + .expect("parse identity object") + .remove("include_submodules"); + let refused: Result = + serde_json::from_value(missing_policy); + assert!( + refused.unwrap_err().to_string().contains("include_submodules"), + "missing policy must be rejected" + ); + } + } +} diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index 2766aeb4913..41f3cf7e74f 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -115,7 +115,10 @@ fn assert_refused_walk(root: &Path, walk: SourceWalk) { } } } - assert_eq!(errors, 1, "binding loss must not be a successful empty walk"); + assert_eq!( + errors, 1, + "binding loss must not be a successful empty walk" + ); assert!(files.contains(Path::new("parent.rs"))); assert!(files.iter().all(|path| !path.starts_with("app"))); } @@ -133,7 +136,10 @@ fn marker_change_before_iteration(replacement: Option<&str>) { Some(contents) => fs::write(&marker, contents).unwrap(), None => fs::remove_file(&marker).unwrap(), } - assert!(root.join(path).is_file(), "the child bytes are still present"); + assert!( + root.join(path).is_file(), + "the child bytes are still present" + ); assert!(boundaries.resolve(path).unwrap().is_none()); assert_refused_walk(&root, pending); } From 71602073d01bfdda3971da98c6ceef79fa73d8bc Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:02:07 -0800 Subject: [PATCH 37/42] perf(code-index): restrict scoped Git boundary traversal Avoid scanning every parent index entry and HEAD-tree object before a literal-prefix source search. Probe index ancestors and the sorted prefix range, and resolve only HEAD ancestors and the matching subtree. Broad or basename scopes keep the complete fail-closed boundary inventory. Add a real-Git regression where an unrelated committed subtree is unreadable: its absence must not block an exact-file scope, while a full source walk must still refuse the incomplete HEAD authority. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 199 ++++++++++++++++-- .../src/source_walk/submodules.rs | 10 +- 2 files changed, 190 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index aa71768a395..f62ae0d1ede 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -128,6 +128,21 @@ fn nested_repository_marker(directory: &Path) -> std::io::Result { /// not an ordinary directory the live source walker can authenticate. /// This is deny-only; it never enables submodule admission. pub fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { + parent_gitlink_paths_scoped(project_root, None) +} + +/// A positive source glob with a literal prefix cannot inspect paths outside +/// that prefix. Keep ancestor gitlinks and any gitlinks below a named directory. +pub(super) fn parent_gitlink_paths_scoped( + project_root: &Path, + scope: Option<&Path>, +) -> Result, SourceWalkError> { + let scope = scope.filter(|scope| { + !scope.as_os_str().is_empty() + && scope + .components() + .all(|component| matches!(component, Component::Normal(_))) + }); let repository = match tracedecay_runtime_core::git_open::open(project_root) { Ok(repository) => repository, Err(error) => { @@ -163,14 +178,56 @@ pub fn parent_gitlink_paths(project_root: &Path) -> Result, Sou message: format!("cannot inspect parent Git index: {error}"), })?; let mut paths = BTreeSet::new(); - for entry in index.entries() { - if entry.mode.is_submodule() { - let path: &[u8] = entry.path(&index).as_ref(); + { + let mut record = |path: &[u8]| -> Result<(), SourceWalkError> { let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { glob: String::new(), message: "non-UTF-8 parent gitlink cannot be safely scoped".to_owned(), })?; paths.insert(relative.to_owned()); + Ok(()) + }; + let entries = index.entries(); + if let Some(scope) = scope { + // Git index entries are sorted by byte-exact path. Probe every + // ancestor, including unmerged stages, then only the prefix range. + for ancestor in scope.ancestors().filter(|path| !path.as_os_str().is_empty()) { + let name = forward_slash_path(ancestor); + let bytes = name.as_bytes(); + let start = entries.partition_point(|entry| { + let path: &[u8] = entry.path(&index).as_ref(); + path < bytes + }); + for entry in entries[start..].iter().take_while(|entry| { + let path: &[u8] = entry.path(&index).as_ref(); + path == bytes + }) { + if entry.mode.is_submodule() { + record(bytes)?; + } + } + } + let prefix = format!("{}/", forward_slash_path(scope)); + let start = entries.partition_point(|entry| { + let path: &[u8] = entry.path(&index).as_ref(); + path < prefix.as_bytes() + }); + for entry in entries[start..].iter().take_while(|entry| { + let path: &[u8] = entry.path(&index).as_ref(); + path.starts_with(prefix.as_bytes()) + }) { + if entry.mode.is_submodule() { + let path: &[u8] = entry.path(&index).as_ref(); + record(path)?; + } + } + } else { + for entry in entries { + if entry.mode.is_submodule() { + let path: &[u8] = entry.path(&index).as_ref(); + record(path)?; + } + } } } // A staged gitlink removal does not erase the checkout boundary from @@ -189,24 +246,78 @@ pub fn parent_gitlink_paths(project_root: &Path) -> Result, Sou glob: String::new(), message: format!("cannot read parent Git HEAD tree: {error}"), })?; - let entries = tree - .traverse() - .breadthfirst - .files() - .map_err(|error| SourceWalkError { - glob: String::new(), - message: format!("cannot walk parent Git HEAD tree: {error}"), - })?; - for entry in entries { - if !entry.mode.is_commit() { - continue; - } - let path: &[u8] = entry.filepath.as_ref(); + let mut record = |path: &[u8]| -> Result<(), SourceWalkError> { let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { glob: String::new(), message: "non-UTF-8 HEAD gitlink cannot be safely scoped".to_owned(), })?; paths.insert(relative.to_owned()); + Ok(()) + }; + if let Some(scope) = scope { + // A staged gitlink removal survives in HEAD even without a child + // marker. Follow only the scope's ancestors and committed subtree. + let ancestors = scope + .ancestors() + .filter(|path| !path.as_os_str().is_empty()) + .collect::>(); + for ancestor in ancestors.into_iter().rev() { + let Some(entry) = tree + .lookup_entry_by_path(ancestor) + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect scoped parent HEAD tree: {error}"), + })? + else { + break; + }; + if entry.mode().is_commit() { + record(forward_slash_path(ancestor).as_bytes())?; + break; + } + if !entry.mode().is_tree() { + break; + } + if ancestor == scope { + let subtree = repository + .find_tree(entry.object_id()) + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot read scoped parent HEAD tree: {error}"), + })?; + let entries = subtree + .traverse() + .breadthfirst + .files() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot walk scoped parent HEAD tree: {error}"), + })?; + let prefix = forward_slash_path(scope); + for entry in entries { + if entry.mode.is_commit() { + let mut path = prefix.as_bytes().to_vec(); + path.push(b'/'); + path.extend_from_slice(entry.filepath.as_ref()); + record(&path)?; + } + } + } + } + } else { + let entries = tree + .traverse() + .breadthfirst + .files() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot walk parent Git HEAD tree: {error}"), + })?; + for entry in entries { + if entry.mode.is_commit() { + record(entry.filepath.as_ref())?; + } + } } } else if head.referent_name().is_none() { return Err(SourceWalkError { @@ -396,8 +507,14 @@ fn source_walk_inner( .is_some_and(|scope| scope.allows(&root, path)))) } }; - let boundaries = - submodules::CheckoutBoundaries::open(project_root, path_policy.includes_submodules())?; + let literal_scope = path_glob_scope.as_ref().and_then(|scope| { + (!scope.literal_prefix.as_os_str().is_empty()).then_some(scope.literal_prefix.as_path()) + }); + let boundaries = submodules::CheckoutBoundaries::open_scoped( + project_root, + path_policy.includes_submodules(), + literal_scope, + )?; let mut parent_gitlinks = boundaries.gitlinks; let mut authenticated = BTreeSet::new(); let mut child_files = BTreeSet::new(); @@ -814,6 +931,52 @@ mod tests { output } + #[test] + fn a_literal_scope_does_not_read_unrelated_committed_trees() { + let root = TempDir::new().expect("parent checkout"); + fixture_git(root.path(), &["init", "-q"]); + for (name, contents) in [ + ("src/keep.rs", "pub fn kept() {}\n"), + ("unrelated/deep/noise.rs", "pub fn noise() {}\n"), + ] { + let path = root.path().join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, contents).unwrap(); + } + fixture_git(root.path(), &["add", "."]); + fixture_git(root.path(), &["commit", "-qm", "two independent trees"]); + let output = fixture_git(root.path(), &["rev-parse", "HEAD:unrelated"]); + let oid = String::from_utf8(output.stdout).unwrap(); + let oid = oid.trim(); + assert!(oid.len() > 2 && oid.bytes().all(|byte| byte.is_ascii_hexdigit())); + let object = root + .path() + .join(".git/objects") + .join(&oid[..2]) + .join(&oid[2..]); + assert!(object.is_file(), "fixture must have a loose unrelated tree"); + fs::remove_file(&object).unwrap(); + + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + let files = source_walk(root.path(), Some("src/keep.rs"), &policy) + .expect("literal scope must not read unrelated HEAD tree") + .map(|entry| entry.expect("scoped source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .map(|entry| { + entry + .into_path() + .strip_prefix(root.path()) + .unwrap() + .to_path_buf() + }) + .collect::>(); + assert_eq!(files, vec![PathBuf::from("src/keep.rs")]); + assert!( + source_walk(root.path(), None, &policy).is_err(), + "an unscoped walk must still refuse unreadable HEAD tree authority" + ); + } + #[test] fn includes_cannot_admit_registered_submodule_files_as_parent_source() { let fixture = TempDir::new().expect("repository fixture"); diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index e7811a6dabc..57c49b81635 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -109,7 +109,15 @@ pub(super) struct CheckoutBoundaries { impl CheckoutBoundaries { pub(super) fn open(root: &Path, include_submodules: bool) -> Result { - let gitlinks = super::parent_gitlink_paths(root)?; + Self::open_scoped(root, include_submodules, None) + } + + pub(super) fn open_scoped( + root: &Path, + include_submodules: bool, + scope: Option<&Path>, + ) -> Result { + let gitlinks = super::parent_gitlink_paths_scoped(root, scope)?; let children = if include_submodules && !gitlinks.is_empty() { let parent = tracedecay_runtime_core::git_open::open(root).map_err(failure)?; live_submodules(&parent)? From 967d9d19d0081703d92a25f9abc392e1744f1aeb Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:14:33 -0800 Subject: [PATCH 38/42] style(code-index): apply Bazel rustfmt to scoped Git boundaries Apply the five precise formatting corrections from repository-gates run 42740034779 for source_walk.rs. No behavior or policy changes. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index f62ae0d1ede..a2bb37cadaa 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -191,7 +191,10 @@ pub(super) fn parent_gitlink_paths_scoped( if let Some(scope) = scope { // Git index entries are sorted by byte-exact path. Probe every // ancestor, including unmerged stages, then only the prefix range. - for ancestor in scope.ancestors().filter(|path| !path.as_os_str().is_empty()) { + for ancestor in scope + .ancestors() + .filter(|path| !path.as_os_str().is_empty()) + { let name = forward_slash_path(ancestor); let bytes = name.as_bytes(); let start = entries.partition_point(|entry| { @@ -262,12 +265,12 @@ pub(super) fn parent_gitlink_paths_scoped( .filter(|path| !path.as_os_str().is_empty()) .collect::>(); for ancestor in ancestors.into_iter().rev() { - let Some(entry) = tree - .lookup_entry_by_path(ancestor) - .map_err(|error| SourceWalkError { - glob: String::new(), - message: format!("cannot inspect scoped parent HEAD tree: {error}"), - })? + let Some(entry) = + tree.lookup_entry_by_path(ancestor) + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect scoped parent HEAD tree: {error}"), + })? else { break; }; @@ -279,20 +282,18 @@ pub(super) fn parent_gitlink_paths_scoped( break; } if ancestor == scope { - let subtree = repository - .find_tree(entry.object_id()) - .map_err(|error| SourceWalkError { + let subtree = repository.find_tree(entry.object_id()).map_err(|error| { + SourceWalkError { glob: String::new(), message: format!("cannot read scoped parent HEAD tree: {error}"), - })?; - let entries = subtree - .traverse() - .breadthfirst - .files() - .map_err(|error| SourceWalkError { + } + })?; + let entries = subtree.traverse().breadthfirst.files().map_err(|error| { + SourceWalkError { glob: String::new(), message: format!("cannot walk scoped parent HEAD tree: {error}"), - })?; + } + })?; let prefix = forward_slash_path(scope); for entry in entries { if entry.mode.is_commit() { @@ -305,14 +306,14 @@ pub(super) fn parent_gitlink_paths_scoped( } } } else { - let entries = tree - .traverse() - .breadthfirst - .files() - .map_err(|error| SourceWalkError { - glob: String::new(), - message: format!("cannot walk parent Git HEAD tree: {error}"), - })?; + let entries = + tree.traverse() + .breadthfirst + .files() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot walk parent Git HEAD tree: {error}"), + })?; for entry in entries { if entry.mode.is_commit() { record(entry.filepath.as_ref())?; From 2647906c2ac4de4a8fea89e11f928f0858f80116 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:22:02 -0800 Subject: [PATCH 39/42] style(code-index): format explicit submodule policy rejection --- .../tracedecay-code-index/src/production/ignored_sources.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/src/production/ignored_sources.rs b/crates/tracedecay-code-index/src/production/ignored_sources.rs index 438ec1c1d72..fba65f7ed18 100644 --- a/crates/tracedecay-code-index/src/production/ignored_sources.rs +++ b/crates/tracedecay-code-index/src/production/ignored_sources.rs @@ -218,7 +218,10 @@ mod parse_identity_tests { let refused: Result = serde_json::from_value(missing_policy); assert!( - refused.unwrap_err().to_string().contains("include_submodules"), + refused + .unwrap_err() + .to_string() + .contains("include_submodules"), "missing policy must be rejected" ); } From bfe14a24e51106c061dbc163b20e2817ac6adfd7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:07:32 -0800 Subject: [PATCH 40/42] fix(code-index): revalidate parent submodule authority during reads Refs #4593 Recheck the parent's exact staged mode-160000 gitlink and pin, staged .gitmodules identity, live registration and owned module store on existing child source and graph reads. The child marker alone is not sufficient authority. Add real-Git regressions for working and staged registration removal, staged gitlink removal and pin replacement after reader construction. These new Rust tests still require exact-head nonzero Bazel validation. --- .../src/source_walk/submodules.rs | 64 ++++++++++++++++++- .../src/source_walk/submodules/tests.rs | 47 ++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 57c49b81635..cd633f2b32f 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -23,6 +23,10 @@ pub struct AuthenticatedSubmodule { pub worktree: PathBuf, pub git_dir: PathBuf, pub pin: gix::ObjectId, + parent_root: PathBuf, + module_path: String, + module_name: PathBuf, + staged_modules_id: gix::ObjectId, } pub(super) struct ChildSources { @@ -31,6 +35,57 @@ pub(super) struct ChildSources { } impl AuthenticatedSubmodule { + fn parent_registration_is_live(&self) -> bool { + // An already-open iterator or graph reader cannot continue to use a + // child after the parent revokes its staged gitlink or registration. + // Inspect exact index entries, not the entire committed Git tree. + let Ok(parent) = tracedecay_runtime_core::git_open::open(&self.parent_root) else { + return false; + }; + if parent + .workdir() + .and_then(|root| root.canonicalize().ok()) + .as_ref() + != Some(&self.parent_root) + { + return false; + } + // A replaced parent Git directory cannot inherit the old child's + // authorization merely by advertising the same staged paths. + if !module_stores(&parent, &self.module_name).any(|store| store == self.git_dir) { + return false; + } + let Ok(index) = parent.index_or_empty() else { + return false; + }; + let stage = gix::index::entry::Stage::Unconflicted; + let Some(modules) = index.entry_by_path_and_stage(".gitmodules".into(), stage) else { + return false; + }; + if modules.id != self.staged_modules_id + || !modules + .mode + .to_tree_entry_mode() + .is_some_and(|mode| mode.is_blob()) + { + return false; + } + let Some(gitlink) = + index.entry_by_path_and_stage(self.module_path.as_bytes().as_bstr(), stage) + else { + return false; + }; + if !gitlink.mode.is_submodule() || gitlink.id != self.pin { + return false; + } + let Ok(Some(working)) = parent.open_modules_file() else { + return false; + }; + working + .path(self.module_name.as_os_str().as_encoded_bytes().as_bstr()) + .is_ok_and(|path| path.as_slice() == self.module_path.as_bytes()) + } + fn checkout_is_live(&self) -> bool { let marker = self.worktree.join(".git"); self.worktree.canonicalize().ok().as_ref() == Some(&self.worktree) @@ -42,6 +97,7 @@ impl AuthenticatedSubmodule { .and_then(|target| target.canonicalize().ok()) .as_ref() == Some(&self.git_dir) + && self.parent_registration_is_live() } pub(super) fn sources(&self) -> Result { @@ -193,8 +249,10 @@ fn bound_checkout( path: &str, name: &Path, pin: gix::ObjectId, + staged_modules_id: gix::ObjectId, ) -> Option { let root = parent.workdir()?.canonicalize().ok()?; + let module_path = path.to_owned(); let path = Path::new(path); if has_nested_repository_ancestor(&root, path).ok()? { return None; @@ -220,6 +278,10 @@ fn bound_checkout( worktree, git_dir: expected, pin, + parent_root: root, + module_path, + module_name: name.to_path_buf(), + staged_modules_id, }) } @@ -258,7 +320,7 @@ pub fn live_submodules( { continue; } - if let Some(child) = bound_checkout(parent, &path, &name, entry.id) { + if let Some(child) = bound_checkout(parent, &path, &name, entry.id, modules.id) { children.insert(path, child); } } diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index 41f3cf7e74f..bc13ac8cd38 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -144,6 +144,53 @@ fn marker_change_before_iteration(replacement: Option<&str>) { assert_refused_walk(&root, pending); } +fn parent_authority_change_before_iteration(change: impl FnOnce(&Path)) { + let fixture = fixture(); + let root = fixture.path().join("parent"); + assert_live_control(&root); + let boundaries = LiveSourceBoundaries::open(&root, true).unwrap(); + let path = Path::new("app/first.rs"); + assert!(boundaries.resolve(path).unwrap().is_some()); + let pending = walk(&root); + change(&root); + assert!(root.join(path).is_file(), "the same child bytes still exist"); + assert!(root.join("app/.git").is_file(), "child marker is unchanged"); + assert!(boundaries.resolve(path).unwrap().is_none()); + assert_refused_walk(&root, pending); +} + +#[test] +fn lazy_submodule_walk_refuses_removed_worktree_registration() { + parent_authority_change_before_iteration(|root| { + fs::remove_file(root.join(".gitmodules")).unwrap(); + }); +} + +#[test] +fn lazy_submodule_walk_refuses_removed_staged_gitlink() { + parent_authority_change_before_iteration(|root| { + git(root, &["rm", "-q", "--cached", "app"]); + }); +} + +#[test] +fn lazy_submodule_walk_refuses_removed_staged_registration() { + parent_authority_change_before_iteration(|root| { + git(root, &["rm", "-q", "--cached", ".gitmodules"]); + }); +} + +#[test] +fn lazy_submodule_walk_refuses_updated_staged_child_pin() { + parent_authority_change_before_iteration(|root| { + let child = root.join("app"); + fs::write(child.join("first.rs"), "pub fn updated() {}\n").unwrap(); + git(&child, &["add", "first.rs"]); + git(&child, &["commit", "-qm", "updated child"]); + git(root, &["add", "app"]); + }); +} + #[test] fn lazy_submodule_walk_refuses_removed_marker() { marker_change_before_iteration(None); From 5f2d9f307ad0b83724702ba56c84dd6a3b715a89 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:31:07 -0800 Subject: [PATCH 41/42] style(code-index): apply Bazel rustfmt to submodule authority test Refs #4593 Apply the single exact formatter hunk reported by the current-head hosted Bazel rustfmt_check_2024 failure (run 42761466450). Retain all four new behavioral regression assertions unchanged. --- .../src/source_walk/submodules/tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index bc13ac8cd38..e2f479a5fb8 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -153,7 +153,10 @@ fn parent_authority_change_before_iteration(change: impl FnOnce(&Path)) { assert!(boundaries.resolve(path).unwrap().is_some()); let pending = walk(&root); change(&root); - assert!(root.join(path).is_file(), "the same child bytes still exist"); + assert!( + root.join(path).is_file(), + "the same child bytes still exist" + ); assert!(root.join("app/.git").is_file(), "child marker is unchanged"); assert!(boundaries.resolve(path).unwrap().is_none()); assert_refused_walk(&root, pending); From d245259e78419ca1c401d6b606465c5f37c60f4f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:56:47 -0800 Subject: [PATCH 42/42] fix(code-index): recheck child repository worktree binding Refs #4593 Reject existing child source walks and graph reads when the child Git administration changes core.worktree without changing the .git marker. Reuse the initial pinned-store topology proof on every bound read. Add a real-Git regression that changes only the child repository worktree configuration after source and graph readers were created. The native Git fixture demonstrates the mismatch; Rust acceptance still needs nonzero Bazel tests on this exact commit. --- .../src/source_walk/submodules.rs | 7 +++++ .../src/source_walk/submodules/tests.rs | 28 +++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index cd633f2b32f..f2fe2a256c4 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -98,6 +98,13 @@ impl AuthenticatedSubmodule { .as_ref() == Some(&self.git_dir) && self.parent_registration_is_live() + // The child gitdir marker is unchanged if its repository config + // redirects core.worktree. A fresh binding would refuse that store; + // an existing graph reader or lazy walk must do the same. + && open_pinned_store(&self.git_dir, self.pin) + .and_then(|child| child.workdir()?.canonicalize().ok()) + .as_ref() + == Some(&self.worktree) } pub(super) fn sources(&self) -> Result { diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index e2f479a5fb8..0152a5cb35a 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -144,7 +144,7 @@ fn marker_change_before_iteration(replacement: Option<&str>) { assert_refused_walk(&root, pending); } -fn parent_authority_change_before_iteration(change: impl FnOnce(&Path)) { +fn binding_change_before_iteration(change: impl FnOnce(&Path)) { let fixture = fixture(); let root = fixture.path().join("parent"); assert_live_control(&root); @@ -164,28 +164,28 @@ fn parent_authority_change_before_iteration(change: impl FnOnce(&Path)) { #[test] fn lazy_submodule_walk_refuses_removed_worktree_registration() { - parent_authority_change_before_iteration(|root| { + binding_change_before_iteration(|root| { fs::remove_file(root.join(".gitmodules")).unwrap(); }); } #[test] fn lazy_submodule_walk_refuses_removed_staged_gitlink() { - parent_authority_change_before_iteration(|root| { + binding_change_before_iteration(|root| { git(root, &["rm", "-q", "--cached", "app"]); }); } #[test] fn lazy_submodule_walk_refuses_removed_staged_registration() { - parent_authority_change_before_iteration(|root| { + binding_change_before_iteration(|root| { git(root, &["rm", "-q", "--cached", ".gitmodules"]); }); } #[test] fn lazy_submodule_walk_refuses_updated_staged_child_pin() { - parent_authority_change_before_iteration(|root| { + binding_change_before_iteration(|root| { let child = root.join("app"); fs::write(child.join("first.rs"), "pub fn updated() {}\n").unwrap(); git(&child, &["add", "first.rs"]); @@ -194,6 +194,24 @@ fn lazy_submodule_walk_refuses_updated_staged_child_pin() { }); } +#[test] +fn lazy_submodule_walk_refuses_changed_child_repository_worktree() { + binding_change_before_iteration(|root| { + // The marker stays at app/.git and the source bytes remain in app, + // but the parent-owned child Git admin now advertises another worktree. + let other = root.parent().unwrap().join("child"); + git( + &root.join("app"), + &["config", "core.worktree", other.to_str().unwrap()], + ); + let parent = tracedecay_runtime_core::git_open::open(root).unwrap(); + assert!( + !super::live_submodules(&parent).unwrap().contains_key("app"), + "fresh admission must reject a retargeted child repository worktree" + ); + }); +} + #[test] fn lazy_submodule_walk_refuses_removed_marker() { marker_change_before_iteration(None);