From 3cfe5a68216a4dcf59a6ed2b7e454345e21d6881 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Sun, 8 Nov 2026 17:39:40 -0500 Subject: [PATCH 1/4] perf(workspace): cut Git process count in checkout verification Checkout verification spawned one Git process per question. A parent execution root test started 5,211 Git processes and took 85 s on macOS. - Batch the repository-layout, shallow-state, HEAD, and branch queries in verify_git_checkout into two rev-parse calls, and the integration checkout root, common dir, and HEAD into one. - Return the verified Git facts to parent preparation and retained-child validation so they no longer re-run status, branch, and rev-parse HEAD. - Resolve the real Git binary once on macOS when git on PATH is the /usr/bin/git xcrun trampoline, removing about 10 ms of developer-dir lookup from every workspace and code-index Git process. --- crates/opensymphony-memory/src/code_graph.rs | 21 +- .../opensymphony-workspace/src/git_program.rs | 81 +++++++ crates/opensymphony-workspace/src/lib.rs | 2 + crates/opensymphony-workspace/src/manager.rs | 207 ++++++++++-------- docs/workspace-and-lifecycle.md | 8 +- 5 files changed, 214 insertions(+), 105 deletions(-) create mode 100644 crates/opensymphony-workspace/src/git_program.rs diff --git a/crates/opensymphony-memory/src/code_graph.rs b/crates/opensymphony-memory/src/code_graph.rs index 45e76f392..b053bddf1 100644 --- a/crates/opensymphony-memory/src/code_graph.rs +++ b/crates/opensymphony-memory/src/code_graph.rs @@ -17,6 +17,7 @@ use crate::opensymphony_gateway_schema::{ CodeSymbolProvenance, }, }; +use crate::opensymphony_workspace::git_program; use duckdb::OptionalExt; use url::Url; use std::sync::{Mutex, OnceLock}; @@ -1363,7 +1364,7 @@ fn unindexed_code_repo_summary(config: &MemoryConfig) -> CodeRepoSummary { } fn configured_code_repo_id(config: &MemoryConfig) -> String { - let remote = Command::new("git") + let remote = Command::new(git_program()) .args(["-C"]) .arg(&config.repo_root) .args(["remote", "get-url", "origin"]) @@ -3170,7 +3171,7 @@ fn workspace_git_bytes( workspace_path: &Path, args: [&str; N], ) -> Result, CodeGraphProjectionError> { - let output = Command::new("git") + let output = Command::new(git_program()) .arg("-C") .arg(workspace_path) .args(args) @@ -3931,7 +3932,7 @@ pub fn code_index_target( } pub fn code_index_repository_is_git(config: &MemoryConfig) -> bool { - Command::new("git") + Command::new(git_program()) .args(["-C"]) .arg(&config.repo_root) .args(["rev-parse", "--git-dir"]) @@ -5035,7 +5036,7 @@ fn validate_code_index_branch( "configured target branch is invalid".to_string(), )); } - let branch_check = Command::new("git") + let branch_check = Command::new(git_program()) .args(["check-ref-format", "--branch", branch]) .output() .map_err(|source| CodeGraphProjectionError::Memory(MemoryError::ResolvePath { @@ -5058,7 +5059,7 @@ fn git_target_commit( format!("refs/remotes/origin/{branch}"), format!("refs/heads/{branch}"), ] { - let output = Command::new("git") + let output = Command::new(git_program()) .args(["-C"]) .arg(root) .args(["rev-parse", "--verify", "--quiet"]) @@ -5087,7 +5088,7 @@ fn git_tree_paths( } else { format!("{directory}/") }; - let output = Command::new("git") + let output = Command::new(git_program()) .args(["-C"]) .arg(root) .args(["ls-tree", "-z"]) @@ -5139,7 +5140,7 @@ fn git_tree_paths( } fn git_repo_prefix(root: &Path) -> Result { - let output = Command::new("git") + let output = Command::new(git_program()) .args(["-C"]) .arg(root) .args(["rev-parse", "--show-prefix"]) @@ -5162,7 +5163,7 @@ fn git_changed_tree_paths( head: &str, repo_prefix: &Path, ) -> Result, CodeGraphProjectionError> { - let output = Command::new("git") + let output = Command::new(git_program()) .args(["-C"]) .arg(root) .args(["diff", "--name-only", "-z", base, head, "--", "."]) @@ -5197,7 +5198,7 @@ fn git_blob( path: &Path, ) -> Result, CodeGraphProjectionError> { let object = format!("{}:{}", commit, repo_prefix.join(path).to_string_lossy()); - let output = Command::new("git") + let output = Command::new(git_program()) .args(["-C"]) .arg(root) .args(["cat-file", "blob"]) @@ -5223,7 +5224,7 @@ fn git_blob_size( path: &Path, ) -> Result { let object = format!("{}:{}", commit, repo_prefix.join(path).to_string_lossy()); - let output = Command::new("git") + let output = Command::new(git_program()) .args(["-C"]) .arg(root) .args(["cat-file", "-s"]) diff --git a/crates/opensymphony-workspace/src/git_program.rs b/crates/opensymphony-workspace/src/git_program.rs new file mode 100644 index 000000000..96387e587 --- /dev/null +++ b/crates/opensymphony-workspace/src/git_program.rs @@ -0,0 +1,81 @@ +//! Resolves the Git executable that checkout and memory code spawn. +//! +//! On macOS, `/usr/bin/git` is an `xcrun` trampoline: every invocation looks +//! up the active developer directory before it execs the real binary, which +//! adds roughly 10 ms to each Git process. Checkout verification and code +//! indexing start thousands of short Git processes, so the trampoline cost +//! dominates their wall time. When `git` on `PATH` is that trampoline, the +//! real binary is resolved once with `xcrun --find git`, the same lookup the +//! trampoline performs, and used directly. Every other platform and PATH +//! layout keeps spawning `git` through `PATH`. + +use std::{ffi::OsStr, sync::OnceLock}; + +/// Returns the program to pass to `Command::new` for Git. +pub fn git_program() -> &'static OsStr { + static PROGRAM: OnceLock = OnceLock::new(); + PROGRAM.get_or_init(resolve_git_program).as_os_str() +} + +#[cfg(target_os = "macos")] +fn resolve_git_program() -> std::ffi::OsString { + use std::path::{Path, PathBuf}; + + const XCRUN_TRAMPOLINE: &str = "/usr/bin/git"; + + fn first_git_on_path() -> Option { + let path = std::env::var_os("PATH")?; + std::env::split_paths(&path) + .map(|directory| directory.join("git")) + .find(|candidate| candidate.is_file()) + } + + let default = std::ffi::OsString::from("git"); + let Some(on_path) = first_git_on_path() else { + return default; + }; + if on_path.canonicalize().ok().as_deref() != Some(Path::new(XCRUN_TRAMPOLINE)) { + return default; + } + let Ok(output) = std::process::Command::new("/usr/bin/xcrun") + .args(["--find", "git"]) + .stdin(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .output() + else { + return default; + }; + if !output.status.success() { + return default; + } + let Ok(resolved) = String::from_utf8(output.stdout) else { + return default; + }; + let resolved = PathBuf::from(resolved.trim()); + if resolved.is_absolute() && resolved.is_file() && resolved != Path::new(XCRUN_TRAMPOLINE) { + tracing::debug!(git = %resolved.display(), "resolved Git behind the xcrun trampoline"); + resolved.into_os_string() + } else { + default + } +} + +#[cfg(not(target_os = "macos"))] +fn resolve_git_program() -> std::ffi::OsString { + std::ffi::OsString::from("git") +} + +#[cfg(test)] +mod tests { + use super::git_program; + + #[test] + fn git_program_runs_git() { + let output = std::process::Command::new(git_program()) + .arg("--version") + .output() + .expect("spawn the resolved Git program"); + assert!(output.status.success()); + assert!(String::from_utf8_lossy(&output.stdout).starts_with("git version")); + } +} diff --git a/crates/opensymphony-workspace/src/lib.rs b/crates/opensymphony-workspace/src/lib.rs index 2cb3aed62..cdb5a8089 100644 --- a/crates/opensymphony-workspace/src/lib.rs +++ b/crates/opensymphony-workspace/src/lib.rs @@ -1,5 +1,6 @@ mod environment; mod error; +mod git_program; mod manager; mod models; mod paths; @@ -10,6 +11,7 @@ pub(crate) use sensitive_fields::{normalize_secret_field_name, runtime_field_is_ pub use environment::environment_variable_names_equal; pub(crate) use environment::{has_environment_name_collision, insert_environment_value}; pub use error::{WorkspaceError, WorkspaceOwnershipConflictDetails}; +pub use git_program::git_program; pub use manager::{ WorkspaceManager, compose_parent_continuation_prompt, compose_parent_prompt, compose_terminal_prompt, diff --git a/crates/opensymphony-workspace/src/manager.rs b/crates/opensymphony-workspace/src/manager.rs index 440ab02bf..248660175 100644 --- a/crates/opensymphony-workspace/src/manager.rs +++ b/crates/opensymphony-workspace/src/manager.rs @@ -38,6 +38,7 @@ use super::{ ParentRuntimeDescriptor, ParentRuntimeEnvelope, PromptCaptureDescriptor, PromptCaptureManifest, RunDescriptor, RunManifest, RunStatus, SessionContextArtifact, TerminalRuntimeEnvelope, WorkspaceError, WorkspaceHandle, WorkspaceManagerConfig, WorkspaceOwnershipConflictDetails, + git_program, models::{ AfterCreateBootstrapReceipt, InstructionProvenance, SSH_AUTH_SOCK_ENV, redact_runtime_diagnostic, @@ -407,8 +408,8 @@ impl WorkspaceManager { ), ) .await?; - let checkout = self - .verify_checkout_with_worker_changes_timeout( + let (checkout, facts) = self + .verify_checkout_facts_with_worker_changes_timeout( &child_handle, true, true, @@ -421,36 +422,17 @@ impl WorkspaceManager { "retained child checkout has the wrong canonical repository", )); } - let status = checkout_operation_with_timeout( - checkout_time_remaining(deadline), - child_handle.workspace_path(), - "verify retained child cleanliness", - self.git( - child_handle.workspace_path(), - &["status", "--porcelain", "--untracked-files=all"], - ), - ) - .await?; - if !status.is_empty() { + if !facts.clean { return Err(checkout_verification( child_handle.workspace_path(), "retained child checkout is dirty", )); } - let child_branch = checkout_operation_with_timeout( - checkout_time_remaining(deadline), - child_handle.workspace_path(), - "verify retained child branch", - self.git(child_handle.workspace_path(), &["branch", "--show-current"]), - ) - .await?; - let child_head = checkout_operation_with_timeout( - checkout_time_remaining(deadline), - child_handle.workspace_path(), - "verify retained child head", - self.git(child_handle.workspace_path(), &["rev-parse", "HEAD"]), - ) - .await?; + let GitFacts { + branch: child_branch, + head: child_head, + .. + } = facts; source.get_or_insert((child_handle.clone(), checkout.clone())); required_merge_commits.extend( request @@ -1777,7 +1759,7 @@ impl WorkspaceManager { })?; let mut hooks_config = OsString::from("core.hooksPath="); hooks_config.push(hooks_directory.path()); - let mut command = Command::new("git"); + let mut command = Command::new(git_program()); command .arg("-c") .arg("credential.helper=") @@ -1953,18 +1935,24 @@ impl WorkspaceManager { } self.reject_checkout_controlled_process_filters(&integration) .await?; - let worktree_root = self - .git(&integration, &["rev-parse", "--show-toplevel"]) + let layout = self + .git( + &integration, + &["rev-parse", "--show-toplevel", "--git-common-dir", "HEAD"], + ) .await?; - if self.canonicalize_path(Path::new(&worktree_root)).await? != canonical_integration { + let [worktree_root, common, head] = rev_parse_lines(&layout).ok_or_else(|| { + checkout_verification( + &integration, + "integration checkout Git root is inconsistent", + ) + })?; + if self.canonicalize_path(Path::new(worktree_root)).await? != canonical_integration { return Err(checkout_verification( &integration, "integration checkout Git root is inconsistent", )); } - let common = self - .git(&integration, &["rev-parse", "--git-common-dir"]) - .await?; let common = PathBuf::from(common); let common = if common.is_absolute() { common @@ -1986,7 +1974,6 @@ impl WorkspaceManager { "integration checkout does not share retained child Git storage", )); } - let head = self.git(&integration, &["rev-parse", "HEAD"]).await?; for required in &record.required_merge_commits { if !self .git_is_ancestor( @@ -2013,7 +2000,7 @@ impl WorkspaceManager { && !self .git_is_ancestor( &integration, - &["merge-base", "--is-ancestor", &record.target_commit, &head], + &["merge-base", "--is-ancestor", &record.target_commit, head], ) .await?) { @@ -2153,39 +2140,16 @@ impl WorkspaceManager { self.reject_checkout_controlled_process_filters(child.workspace_path()), ) .await?; - self.verify_checkout_with_worker_changes_timeout(&child, true, true, deadline) + let (_, facts) = self + .verify_checkout_facts_with_worker_changes_timeout(&child, true, true, deadline) .await?; - let status = checkout_operation_with_timeout( - checkout_time_remaining(deadline), - child.workspace_path(), - "verify retained child cleanliness", - self.git( - child.workspace_path(), - &["status", "--porcelain", "--untracked-files=all"], - ), - ) - .await?; - if !status.is_empty() { + if !facts.clean { return Err(checkout_verification( child.workspace_path(), "retained child checkout is dirty", )); } - let child_head = checkout_operation_with_timeout( - checkout_time_remaining(deadline), - child.workspace_path(), - "verify retained child head", - self.git(child.workspace_path(), &["rev-parse", "HEAD"]), - ) - .await?; - let child_branch = checkout_operation_with_timeout( - checkout_time_remaining(deadline), - child.workspace_path(), - "verify retained child branch", - self.git(child.workspace_path(), &["branch", "--show-current"]), - ) - .await?; - if child_head != retained.child_head || child_branch != retained.child_branch { + if facts.head != retained.child_head || facts.branch != retained.child_branch { return Err(checkout_verification( child.workspace_path(), "retained child evidence no longer matches the pinned parent map", @@ -2793,6 +2757,26 @@ impl WorkspaceManager { allow_shallow: bool, checkout_deadline: Option, ) -> Result { + self.verify_checkout_facts_with_worker_changes_timeout( + workspace, + allow_worker_changes, + allow_shallow, + checkout_deadline, + ) + .await + .map(|(manifest, _)| manifest) + } + + /// Verifies a retained checkout and also returns the Git state observed + /// during verification, so callers do not query branch, HEAD, and + /// cleanliness a second time. + async fn verify_checkout_facts_with_worker_changes_timeout( + &self, + workspace: &WorkspaceHandle, + allow_worker_changes: bool, + allow_shallow: bool, + checkout_deadline: Option, + ) -> Result<(CheckoutManifest, GitFacts), WorkspaceError> { self.validate_workspace_handle(workspace).await?; let manifest = self .load_manifest::(workspace, &workspace.checkout_manifest_path()) @@ -2921,7 +2905,7 @@ impl WorkspaceManager { reason: "instruction provenance does not match the recorded commit".to_owned(), }); } - Ok(manifest) + Ok((manifest, facts)) } pub async fn read_checkout_instructions( @@ -3766,7 +3750,7 @@ impl WorkspaceManager { } else { None }; - let mut command = Command::new("git"); + let mut command = Command::new(git_program()); command .arg("clone") .arg("--single-branch") @@ -3916,10 +3900,37 @@ impl WorkspaceManager { require_remote_head: bool, allow_shallow: bool, ) -> Result { - let inside = self - .git(checkout, &["rev-parse", "--is-inside-work-tree"]) + // One rev-parse answers every repository-layout question. Outside a + // work tree `--show-toplevel` fails, so any failure means the same + // thing as `--is-inside-work-tree` not printing `true`. + let layout = self + .git( + checkout, + &[ + "rev-parse", + "--is-inside-work-tree", + "--show-toplevel", + "--git-dir", + "--git-common-dir", + "--git-path", + "objects", + "--git-path", + "info/grafts", + "--is-shallow-repository", + ], + ) .await .map_err(|_| checkout_verification(checkout, "not a Git worktree"))?; + let [ + inside, + worktree_root, + git_dir, + git_common_dir, + object_dir, + grafts_path, + shallow, + ] = rev_parse_lines(&layout) + .ok_or_else(|| checkout_verification(checkout, "not a Git worktree"))?; if inside != "true" { return Err(checkout_verification(checkout, "not a Git worktree")); } @@ -3927,12 +3938,8 @@ impl WorkspaceManager { .canonicalize_path(checkout) .await .map_err(|error| checkout_verification(checkout, &error.to_string()))?; - let worktree_root = self - .git(checkout, &["rev-parse", "--show-toplevel"]) - .await - .map_err(|_| checkout_verification(checkout, "Git worktree root is unavailable"))?; let canonical_worktree_root = self - .canonicalize_path(Path::new(&worktree_root)) + .canonicalize_path(Path::new(worktree_root)) .await .map_err(|error| checkout_verification(checkout, &error.to_string()))?; if canonical_worktree_root != canonical_checkout { @@ -3941,9 +3948,8 @@ impl WorkspaceManager { "Git worktree root does not equal the checkout generation", )); } - for git_path in ["--git-dir", "--git-common-dir"] { - let git_path = self.git(checkout, &["rev-parse", git_path]).await?; - let git_path = PathBuf::from(&git_path); + for git_path in [git_dir, git_common_dir] { + let git_path = PathBuf::from(git_path); let git_path = if git_path.is_absolute() { git_path } else { @@ -3960,12 +3966,9 @@ impl WorkspaceManager { )); } } - self.verify_git_object_alternates(checkout, &canonical_checkout) - .await?; - let grafts_path = self - .git(checkout, &["rev-parse", "--git-path", "info/grafts"]) + self.verify_git_object_alternates(checkout, &canonical_checkout, object_dir) .await?; - let grafts_path = PathBuf::from(&grafts_path); + let grafts_path = PathBuf::from(grafts_path); let grafts_path = if grafts_path.is_absolute() { grafts_path } else { @@ -4039,7 +4042,21 @@ impl WorkspaceManager { )); } } - let branch = self.git(checkout, &["branch", "--show-current"]).await?; + // `--symbolic-full-name HEAD` prints `refs/heads/` on a branch + // and `HEAD` when detached, matching `git branch --show-current`. + let head_state = self + .git( + checkout, + &["rev-parse", "HEAD", "--symbolic-full-name", "HEAD"], + ) + .await?; + let [head, symbolic_head] = rev_parse_lines(&head_state) + .ok_or_else(|| checkout_verification(checkout, "HEAD is unavailable"))?; + let head = head.to_owned(); + let branch = symbolic_head + .strip_prefix("refs/heads/") + .unwrap_or_default() + .to_owned(); if enforce_worktree_state && branch != repository.target_branch { return Err(checkout_verification(checkout, "target branch mismatch")); } @@ -4049,7 +4066,6 @@ impl WorkspaceManager { .map_err(|_| { checkout_verification(checkout, "target branch is not present on origin") })?; - let head = self.git(checkout, &["rev-parse", "HEAD"]).await?; if require_remote_head { let remote_head = self.git(checkout, &["rev-parse", &remote_ref]).await?; if head != remote_head { @@ -4059,10 +4075,7 @@ impl WorkspaceManager { )); } } - let shallow = self - .git(checkout, &["rev-parse", "--is-shallow-repository"]) - .await? - == "true"; + let shallow = shallow == "true"; if shallow && !allow_shallow { return Err(checkout_verification(checkout, "history is shallow")); } @@ -4088,10 +4101,8 @@ impl WorkspaceManager { &self, checkout: &Path, canonical_checkout: &Path, + object_dir: &str, ) -> Result<(), WorkspaceError> { - let object_dir = self - .git(checkout, &["rev-parse", "--git-path", "objects"]) - .await?; let object_dir = PathBuf::from(object_dir); let object_dir = if object_dir.is_absolute() { object_dir @@ -4172,7 +4183,7 @@ impl WorkspaceManager { })?; let mut hooks_config = OsString::from("core.hooksPath="); hooks_config.push(isolated_config.path()); - let mut command = Command::new("git"); + let mut command = Command::new(git_program()); command .arg("-c") .arg("core.fsmonitor=") @@ -4239,7 +4250,7 @@ impl WorkspaceManager { })?; let mut hooks_config = OsString::from("core.hooksPath="); hooks_config.push(hooks_directory.path()); - let mut command = Command::new("git"); + let mut command = Command::new(git_program()); command .arg("-c") .arg("core.fsmonitor=") @@ -4300,7 +4311,7 @@ impl WorkspaceManager { })?; let mut hooks_config = OsString::from("core.hooksPath="); hooks_config.push(hooks_directory.path()); - let mut command = Command::new("git"); + let mut command = Command::new(git_program()); command .arg("-c") .arg("core.fsmonitor=") @@ -4374,7 +4385,7 @@ impl WorkspaceManager { })?; let mut hooks_config = OsString::from("core.hooksPath="); hooks_config.push(hooks_directory.path()); - let mut command = Command::new("git"); + let mut command = Command::new(git_program()); command .arg("-c") .arg("core.fsmonitor=") @@ -7369,6 +7380,14 @@ struct GitFacts { clean: bool, } +/// Splits the output of a multi-option `git rev-parse` into exactly `N` +/// answers, one per line. Returns `None` when the line count differs, for +/// example because a path contains a newline, so callers fail closed. +fn rev_parse_lines(output: &str) -> Option<[&str; N]> { + let lines = output.lines().collect::>(); + lines.try_into().ok() +} + fn checkout_verification(path: &Path, reason: &str) -> WorkspaceError { WorkspaceError::CheckoutVerification { path: path.to_path_buf(), @@ -7721,7 +7740,7 @@ async fn tracked_instruction_paths( .map_err(|_| checkout_verification(root, "tracked instruction probe failed"))?; let mut hooks_config = OsString::from("core.hooksPath="); hooks_config.push(hooks_directory.path()); - let mut command = Command::new("git"); + let mut command = Command::new(git_program()); command .arg("-c") .arg("core.fsmonitor=") diff --git a/docs/workspace-and-lifecycle.md b/docs/workspace-and-lifecycle.md index c2229bc75..0bc9b0580 100644 --- a/docs/workspace-and-lifecycle.md +++ b/docs/workspace-and-lifecycle.md @@ -94,7 +94,13 @@ checkout manifests form a mutually consistent ownership set (or a completed partial, or malformed generation-shaped directories are preserved unless the matching staging intent proves them to be abandoned. Git verification and clone helpers clear inherited repository-selection variables such as `GIT_DIR` -and `GIT_WORK_TREE` before using an explicit checkout path. +and `GIT_WORK_TREE` before using an explicit checkout path. On macOS, when +`git` on `PATH` is the `/usr/bin/git` `xcrun` trampoline, the workspace manager +and code indexer resolve the real binary once with `xcrun --find git` and spawn +it directly, which removes about 10 ms of developer-directory lookup from every +Git process. Checkout verification reads the repository layout, shallow state, +HEAD, and current branch with batched `git rev-parse` calls, and parent +preparation reuses that verified state rather than querying it again. Checkout retention and deletion remain `WorkspaceManager` decisions. Generated tree instruction pruning asks Git only for tracked `AGENTS.md` paths, so a large vendored tree with unrelated tracked files does not become an unbounded From b32d5eb1f61f41eb80be9d975b09577ae662b9ec Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Sun, 8 Nov 2026 17:44:35 -0500 Subject: [PATCH 2/4] ci(rust-test): re-download DuckDB when the restored cache lacks it rust-cache prunes target/duckdb-download before saving, so the cache-directories entry never reached the cache and restored builds failed to link against -lduckdb. Remove the libduckdb-sys build-script output when the download directory is missing so the build script downloads the library again. --- .github/workflows/ci.yml | 14 ++++++++------ docs/testing-and-operations.md | 9 +++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d481e9b32..1b5215bd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,12 +55,14 @@ jobs: - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.ref == 'refs/heads/main' }} - # rust-cache keeps the duckdb build-script output but prunes the - # library it downloaded, so a cache hit skips the download and the - # test binaries fail to link with `-lduckdb`. Cache the library too. - cache-directories: target/duckdb-download - # Bumped with cache-directories so restores carry the library. - key: duckdb-download + # rust-cache prunes target/duckdb-download before it saves, but keeps the + # libduckdb-sys build-script output that links against it. Drop that + # output when the library is missing so the build script downloads it. + - name: restore downloaded DuckDB library + run: | + if [ ! -d target/duckdb-download ]; then + rm -rf target/debug/build/libduckdb-sys-* target/debug/.fingerprint/libduckdb-sys-* + fi - uses: taiki-e/install-action@v2 with: tool: cargo-nextest diff --git a/docs/testing-and-operations.md b/docs/testing-and-operations.md index 7131bb3d1..e4d9dd3bc 100644 --- a/docs/testing-and-operations.md +++ b/docs/testing-and-operations.md @@ -647,10 +647,11 @@ Only `main` saves the Rust caches (`Swatinem/rust-cache` with `save-if`). A pull request can read the `main` cache but not another pull request's cache, so per-PR caches only consumed the 10 GB repository quota and evicted the shared one. -The `rust-test` cache also holds `target/duckdb-download`. rust-cache keeps the -`libduckdb-sys` build-script output but prunes directories it does not know, so -without that entry a cache hit skips the download and the test binaries fail to -link against `-lduckdb`. +rust-cache prunes `target/duckdb-download` before it saves, but keeps the +`libduckdb-sys` build-script output that links against it, so a cache hit would +skip the download and the test binaries would fail to link against `-lduckdb`. +The `rust-test` job removes that build-script output whenever the download +directory is missing, and the build script downloads the library again. Timing evidence is attached to every run: From bc464c6984a014e049ef6e86090a144c18a1a4c6 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Sun, 8 Nov 2026 18:07:10 -0500 Subject: [PATCH 3/4] ci(rust-test): check for the DuckDB library file, not its directory The restored cache keeps target/duckdb-download but not libduckdb.so, so the directory check never triggered the rebuild. --- .github/workflows/ci.yml | 10 ++++++---- docs/testing-and-operations.md | 11 ++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b5215bd5..a84f73f68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,12 +55,14 @@ jobs: - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.ref == 'refs/heads/main' }} - # rust-cache prunes target/duckdb-download before it saves, but keeps the - # libduckdb-sys build-script output that links against it. Drop that - # output when the library is missing so the build script downloads it. + # rust-cache prunes the library under target/duckdb-download before it + # saves, but keeps the libduckdb-sys build-script output that links + # against it. Drop that output when the library is missing so the build + # script downloads it again. - name: restore downloaded DuckDB library run: | - if [ ! -d target/duckdb-download ]; then + if ! ls target/duckdb-download/*/*/libduckdb.so >/dev/null 2>&1; then + echo "libduckdb.so is missing from the restored cache; rebuilding libduckdb-sys" rm -rf target/debug/build/libduckdb-sys-* target/debug/.fingerprint/libduckdb-sys-* fi - uses: taiki-e/install-action@v2 diff --git a/docs/testing-and-operations.md b/docs/testing-and-operations.md index e4d9dd3bc..f012a4d6a 100644 --- a/docs/testing-and-operations.md +++ b/docs/testing-and-operations.md @@ -647,11 +647,12 @@ Only `main` saves the Rust caches (`Swatinem/rust-cache` with `save-if`). A pull request can read the `main` cache but not another pull request's cache, so per-PR caches only consumed the 10 GB repository quota and evicted the shared one. -rust-cache prunes `target/duckdb-download` before it saves, but keeps the -`libduckdb-sys` build-script output that links against it, so a cache hit would -skip the download and the test binaries would fail to link against `-lduckdb`. -The `rust-test` job removes that build-script output whenever the download -directory is missing, and the build script downloads the library again. +rust-cache prunes the library under `target/duckdb-download` before it saves, +but keeps the `libduckdb-sys` build-script output that links against it, so a +cache hit would skip the download and the test binaries would fail to link +against `-lduckdb`. The `rust-test` job removes that build-script output +whenever `libduckdb.so` is missing, and the build script downloads the library +again. Timing evidence is attached to every run: From 2ab658b0370332330fb6b04d79b6bfb112c1f870 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Sun, 8 Nov 2026 18:07:28 -0500 Subject: [PATCH 4/4] ci(rust-test): always rebuild libduckdb-sys after restoring the cache A restored cache never holds the downloaded library, so the check only risked missing a stale version directory. --- .github/workflows/ci.yml | 10 +++------- docs/testing-and-operations.md | 5 ++--- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a84f73f68..c8334b17d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,14 +57,10 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} # rust-cache prunes the library under target/duckdb-download before it # saves, but keeps the libduckdb-sys build-script output that links - # against it. Drop that output when the library is missing so the build - # script downloads it again. + # against it. Drop that output so the build script downloads the library + # again; a cache miss has nothing to drop. - name: restore downloaded DuckDB library - run: | - if ! ls target/duckdb-download/*/*/libduckdb.so >/dev/null 2>&1; then - echo "libduckdb.so is missing from the restored cache; rebuilding libduckdb-sys" - rm -rf target/debug/build/libduckdb-sys-* target/debug/.fingerprint/libduckdb-sys-* - fi + run: rm -rf target/debug/build/libduckdb-sys-* target/debug/.fingerprint/libduckdb-sys-* - uses: taiki-e/install-action@v2 with: tool: cargo-nextest diff --git a/docs/testing-and-operations.md b/docs/testing-and-operations.md index f012a4d6a..2113b9b5e 100644 --- a/docs/testing-and-operations.md +++ b/docs/testing-and-operations.md @@ -650,9 +650,8 @@ one. rust-cache prunes the library under `target/duckdb-download` before it saves, but keeps the `libduckdb-sys` build-script output that links against it, so a cache hit would skip the download and the test binaries would fail to link -against `-lduckdb`. The `rust-test` job removes that build-script output -whenever `libduckdb.so` is missing, and the build script downloads the library -again. +against `-lduckdb`. The `rust-test` job removes that build-script output after +restoring the cache, and the build script downloads the library again. Timing evidence is attached to every run: