From 9340ff4b71e18267877f3b314a1ca805352e9d78 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Sun, 8 Nov 2026 15:09:01 -0500 Subject: [PATCH 1/6] test(run): configure running-state refresh cadence for merging interrupt tests The two Merging-supersede run tests waited out the scheduler's hardcoded 30s running-state refresh before observing the tracker transition. SchedulerConfig now carries the cadence (default 30s) and OPENSYMPHONY_RUNNING_STATE_REFRESH_MS overrides it for opensymphony run; the tests set it to 50ms. --- .../src/orchestrator_run/mod.rs | 46 +++++++++++++++++ crates/opensymphony-cli/tests/run.rs | 49 +++++++++++++------ .../src/scheduler.rs | 10 +++- .../tests/scheduler.rs | 1 + 4 files changed, 91 insertions(+), 15 deletions(-) diff --git a/crates/opensymphony-cli/src/orchestrator_run/mod.rs b/crates/opensymphony-cli/src/orchestrator_run/mod.rs index 3d8c27eb8..d3fae68ef 100644 --- a/crates/opensymphony-cli/src/orchestrator_run/mod.rs +++ b/crates/opensymphony-cli/src/orchestrator_run/mod.rs @@ -68,6 +68,25 @@ use self::{ const MEMORY_SERVER_BIND_STATE: &str = ".opensymphony-memory-bind.json"; +/// Overrides how often `opensymphony run` refreshes tracker states for running +/// issues. Intended for local diagnostics and process-level tests; production +/// runs keep the scheduler's 30-second default to bound Linear reads. +const RUNNING_STATE_REFRESH_INTERVAL_ENV: &str = "OPENSYMPHONY_RUNNING_STATE_REFRESH_MS"; + +fn running_state_refresh_interval_override(value: Option<&str>) -> Option { + let value = value?.trim(); + match value.parse::() { + Ok(interval_ms) if interval_ms > 0 => Some(interval_ms), + _ => { + tracing::warn!( + value, + "ignoring invalid {RUNNING_STATE_REFRESH_INTERVAL_ENV}; expected a positive integer of milliseconds" + ); + None + } + } +} + enum RunWake { Shutdown, Server(Result, tokio::task::JoinError>), @@ -1236,6 +1255,17 @@ async fn run_orchestrator(args: RunArgs) -> Result<(), RunCommandError> { let mut scheduler_config = SchedulerConfig::from_workflow(&runtime.workflow)?; scheduler_config.max_retry_attempts = runtime.retry_max_attempts; scheduler_config.repository_routing = runtime.repository_routing.clone(); + if let Some(interval_ms) = running_state_refresh_interval_override( + std::env::var(RUNNING_STATE_REFRESH_INTERVAL_ENV) + .ok() + .as_deref(), + ) { + tracing::info!( + interval_ms, + "using {RUNNING_STATE_REFRESH_INTERVAL_ENV} running-state refresh cadence" + ); + scheduler_config.running_state_refresh_interval_ms = interval_ms; + } let mut scheduler = Scheduler::new(tracker, workspace, worker, scheduler_config); let mut recent_events = VecDeque::new(); @@ -2240,6 +2270,22 @@ mod tests { use super::*; use crate::opensymphony_memory::MemoryError; + #[test] + fn running_state_refresh_override_accepts_only_positive_milliseconds() { + assert_eq!(running_state_refresh_interval_override(None), None); + assert_eq!( + running_state_refresh_interval_override(Some("250")), + Some(250) + ); + assert_eq!( + running_state_refresh_interval_override(Some(" 50 ")), + Some(50) + ); + assert_eq!(running_state_refresh_interval_override(Some("0")), None); + assert_eq!(running_state_refresh_interval_override(Some("-5")), None); + assert_eq!(running_state_refresh_interval_override(Some("soon")), None); + } + #[tokio::test] async fn pending_operator_flush_allows_worker_wake_and_shutdown() { use crate::opensymphony_gateway_schema::approval::{ diff --git a/crates/opensymphony-cli/tests/run.rs b/crates/opensymphony-cli/tests/run.rs index 10f168b28..1b3185bfa 100644 --- a/crates/opensymphony-cli/tests/run.rs +++ b/crates/opensymphony-cli/tests/run.rs @@ -632,7 +632,7 @@ async fn run_recovers_human_review_worker_and_interrupts_when_tracker_reports_me .await; seed_fake_openhands_conversation(&openhands, conversation_id, project.path()).await; - let mut child = spawn_run_child(project.path(), &[]); + let mut child = spawn_run_child_with_fast_running_refresh(project.path(), &[], None); let snapshot = wait_for_merging_supersede_event(&format!("http://{bind_addr}/api/v1/snapshot")) .await @@ -661,12 +661,14 @@ async fn run_interrupts_active_codex_stdio_worker_when_tracker_reports_merging() write_codex_merging_supersede_project_files(project.path(), linear.base_url(), bind_addr); write_memory_config(project.path()); - let mut child = spawn_run_child_with_codex_bin( + let mut child = spawn_run_child_with_fast_running_refresh( project.path(), &[], - fake_codex - .to_str() - .expect("fake codex path should be utf-8"), + Some( + fake_codex + .to_str() + .expect("fake codex path should be utf-8"), + ), ); wait_for_codex_merging_interrupt_ack(&format!("http://{bind_addr}/api/v1/snapshot")) @@ -814,23 +816,34 @@ Run the scheduler. } fn spawn_run_child(project_root: &std::path::Path, extra_args: &[&str]) -> Child { - spawn_run_child_configured(project_root, extra_args, None, None) + spawn_run_child_configured(project_root, extra_args, None, None, None) } -fn spawn_run_child_with_worker_stack( +/// Running-state refresh cadence for tests that wait for the scheduler to +/// observe a tracker transition on an already-running issue. The production +/// default is 30 seconds; the mock tracker is local, so poll it every tick. +const FAST_RUNNING_STATE_REFRESH_MS: u64 = 50; + +fn spawn_run_child_with_fast_running_refresh( project_root: &std::path::Path, extra_args: &[&str], - stack_bytes: usize, + codex_bin: Option<&str>, ) -> Child { - spawn_run_child_configured(project_root, extra_args, None, Some(stack_bytes)) + spawn_run_child_configured( + project_root, + extra_args, + codex_bin, + None, + Some(FAST_RUNNING_STATE_REFRESH_MS), + ) } -fn spawn_run_child_with_codex_bin( +fn spawn_run_child_with_worker_stack( project_root: &std::path::Path, extra_args: &[&str], - codex_bin: &str, + stack_bytes: usize, ) -> Child { - spawn_run_child_configured(project_root, extra_args, Some(codex_bin), None) + spawn_run_child_configured(project_root, extra_args, None, Some(stack_bytes), None) } fn spawn_run_child_configured( @@ -838,6 +851,7 @@ fn spawn_run_child_configured( extra_args: &[&str], codex_bin: Option<&str>, worker_stack_bytes: Option, + running_state_refresh_ms: Option, ) -> Child { let mut command = Command::new(env!("CARGO_BIN_EXE_opensymphony")); command @@ -852,12 +866,19 @@ fn spawn_run_child_configured( .env_remove("LLM_MODEL") .env_remove("LLM_API_KEY") .env_remove("LLM_BASE_URL") + .env_remove("OPENSYMPHONY_RUNNING_STATE_REFRESH_MS") .stdout(Stdio::null()) .stderr(Stdio::piped()) .kill_on_drop(true); if let Some(codex_bin) = codex_bin { command.env("OPENSYMPHONY_CODEX_BIN", codex_bin); } + if let Some(interval_ms) = running_state_refresh_ms { + command.env( + "OPENSYMPHONY_RUNNING_STATE_REFRESH_MS", + interval_ms.to_string(), + ); + } if cfg!(unix) && let Some(stack_bytes) = worker_stack_bytes { @@ -1190,7 +1211,7 @@ async fn wait_for_merging_supersede_event(url: &str) -> Result { } last_snapshot = Some(snapshot); } - sleep(Duration::from_millis(250)).await; + sleep(Duration::from_millis(50)).await; } Err(format!( "timed out waiting for Merging supersede event at {url}; last snapshot: {}", @@ -1212,7 +1233,7 @@ async fn wait_for_codex_merging_interrupt_ack(url: &str) -> Result, pub routing: RoutingConfig, pub repository_routing: Option, + /// How often running issues have their tracker state refreshed between + /// full-detail snapshots. `from_workflow` uses the 30-second default. + pub running_state_refresh_interval_ms: u64, } impl SchedulerConfig { @@ -272,6 +278,7 @@ impl SchedulerConfig { tracker_project_slugs: workflow.config.tracker.project_slugs.clone(), routing: workflow.config.routing.clone(), repository_routing: None, + running_state_refresh_interval_ms: RUNNING_STATE_REFRESH_INTERVAL_MS, }) } @@ -2573,7 +2580,7 @@ where && pre_update_full_snapshot.is_none() && due( self.last_running_state_refresh_at, - RUNNING_STATE_REFRESH_INTERVAL_MS, + self.config.running_state_refresh_interval_ms, observed_at, ) && self.has_reconcilable_executions() @@ -10774,6 +10781,7 @@ mod tests { dry_run: false, }, repository_routing: None, + running_state_refresh_interval_ms: 30_000, }; assert!(!project_belongs_to_configured_project( diff --git a/crates/opensymphony-orchestrator/tests/scheduler.rs b/crates/opensymphony-orchestrator/tests/scheduler.rs index 7e7a2ed8d..e0d71a0f6 100644 --- a/crates/opensymphony-orchestrator/tests/scheduler.rs +++ b/crates/opensymphony-orchestrator/tests/scheduler.rs @@ -80,6 +80,7 @@ fn scheduler_config() -> SchedulerConfig { dry_run: false, }, repository_routing: None, + running_state_refresh_interval_ms: 30_000, } } From b6ec77e0cb3a7442df3147acfe42361599f5e443 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Sun, 8 Nov 2026 15:25:23 -0500 Subject: [PATCH 2/6] test(cli): resolve bash without sourcing the login profile doctor and install tests located the real bash with `bash -lc`, which sources the developer's profile and took about 5s per lookup on macOS. A non-login shell resolves the same binary. --- crates/opensymphony-cli/tests/doctor.rs | 4 +++- crates/opensymphony-cli/tests/install.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/opensymphony-cli/tests/doctor.rs b/crates/opensymphony-cli/tests/doctor.rs index 5a13f78c9..044886bc0 100644 --- a/crates/opensymphony-cli/tests/doctor.rs +++ b/crates/opensymphony-cli/tests/doctor.rs @@ -660,8 +660,10 @@ fn write_bash_wrapper(path: PathBuf) { } fn real_bash_path() -> PathBuf { + // A non-login shell resolves the same `bash` this command launched without + // sourcing the developer's profile, which can take seconds. let output = Command::new("bash") - .args(["-lc", "command -v bash"]) + .args(["-c", "command -v bash"]) .output() .expect("bash lookup should run"); assert!( diff --git a/crates/opensymphony-cli/tests/install.rs b/crates/opensymphony-cli/tests/install.rs index 9918872c4..f3a44d22e 100644 --- a/crates/opensymphony-cli/tests/install.rs +++ b/crates/opensymphony-cli/tests/install.rs @@ -302,8 +302,10 @@ fn write_bash_wrapper(path: PathBuf) { } fn real_bash_path() -> PathBuf { + // A non-login shell resolves the same `bash` this command launched without + // sourcing the developer's profile, which can take seconds. let output = Command::new("bash") - .args(["-lc", "command -v bash"]) + .args(["-c", "command -v bash"]) .output() .expect("bash lookup should run"); assert!( From 102d6a4dc7c8352b9bc82529cfc3c6c55364d398 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Sun, 8 Nov 2026 15:27:35 -0500 Subject: [PATCH 3/6] docs: document the running-state refresh override and test timing guidance --- docs/configuration.md | 6 ++++++ docs/operations.md | 6 +++++- docs/testing-and-operations.md | 7 +++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 31578de62..12d5486cb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -566,6 +566,12 @@ For local Codex stdio execution, `OPENSYMPHONY_CODEX_BIN` may point to an alternate Codex binary in trusted operator environments only; it is not a hosted-mode tenant input. +`OPENSYMPHONY_RUNNING_STATE_REFRESH_MS` sets how often `opensymphony run` +re-reads tracker states for issues it is already running. The default is +30000 ms. Shorter values make tracker transitions such as Merging visible +sooner at the cost of more Linear reads, so reserve them for local diagnostics +and tests against a local tracker stub. + OpenAI ChatGPT/Codex subscription credentials are available only when OpenSymphony is built with the `openhands-subscription-credentials` Cargo feature. The workflow stores environment-variable names and auth-directory diff --git a/docs/operations.md b/docs/operations.md index f99ab471d..2f66c6def 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -482,7 +482,11 @@ Operational implications: - `opensymphony run` keeps its local worker/snapshot tick every 5s, while Linear reads use cheaper internal cadences: running state every 30s, dispatch discovery every 60s, terminal cleanup every 5 minutes, and full - issue details hourly after startup/dispatch. When all required children of a + issue details hourly after startup/dispatch. + `OPENSYMPHONY_RUNNING_STATE_REFRESH_MS` overrides the running-state cadence + with a positive millisecond value for local diagnostics and process-level + tests; invalid values are ignored with a warning. Leave it unset against + hosted Linear so running-state reads stay bounded. When all required children of a waiting parent have terminal orchestrator outcomes, full details refresh on the 5-minute terminal cadence so parent eligibility can use a complete hierarchy observation diff --git a/docs/testing-and-operations.md b/docs/testing-and-operations.md index 9c98fd2ad..926c123df 100644 --- a/docs/testing-and-operations.md +++ b/docs/testing-and-operations.md @@ -664,6 +664,13 @@ than 30 s without terminating them. To reproduce the CI test run locally, use with `DUCKDB_DOWNLOAD_LIB=1`, or the system-DuckDB environment from the `cargo *-system-duckdb` aliases. +Keep long tests event-driven rather than clock-driven. A process-level test +that waits for the scheduler to notice a tracker transition on a running issue +sets `OPENSYMPHONY_RUNNING_STATE_REFRESH_MS` on the `opensymphony run` child +instead of waiting out the 30 s production cadence. Test helpers that need a +real shell resolve it with a non-login `bash -c`; a login shell sources the +developer's profile and can add seconds per call. + ## 6. Operations Operational guidance now lives in [operations.md](operations.md). From 36bff3edf5e11e25b18ba9390323d33edc566ca4 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Sun, 8 Nov 2026 15:27:35 -0500 Subject: [PATCH 4/6] test(gateway): run large atlas fixtures in their own test The contract-endpoint test persisted two 505-document fixtures to cross the 500-record atlas cap. Those scenarios use their own repository ids, so they move to a separate test with a fresh store and run in parallel. --- crates/opensymphony-gateway/tests/gateway.rs | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/crates/opensymphony-gateway/tests/gateway.rs b/crates/opensymphony-gateway/tests/gateway.rs index f38e22eb7..36b211630 100644 --- a/crates/opensymphony-gateway/tests/gateway.rs +++ b/crates/opensymphony-gateway/tests/gateway.rs @@ -3436,6 +3436,33 @@ async fn gateway_serves_code_graph_contract_endpoints() { .contains(&"src/empty.rs".to_string()) ); + server_task.abort(); +} + +/// The atlas caps file nodes at the 500-record gateway contract limit. These +/// fixtures need more than 500 documents each, so they use their own store and +/// run alongside the contract-endpoint checks above. +#[tokio::test] +async fn gateway_code_graph_atlas_caps_and_dedupes_large_repositories() { + let repo = tempfile::tempdir().expect("memory repo"); + let config_for_revision_regression = + MemoryConfig::load(repo.path(), None).expect("memory config should load"); + let store = SnapshotStore::new(fixture_snapshot(0)); + let server = + GatewayServer::new(store).with_memory_config(Some(config_for_revision_regression.clone())); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test listener"); + let address = listener.local_addr().expect("test listener address"); + let server_task = tokio::spawn(async move { + server + .serve(listener) + .await + .expect("test gateway server should serve") + }); + let client = reqwest::Client::new(); + let base = format!("http://{address}/api/v1/code"); + persist_code_intel_documents( &config_for_revision_regression, CodeIntelPersistBatch { From f39669a30a9c54e87383942b7e7abf5ecf150412 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Sun, 8 Nov 2026 15:33:00 -0500 Subject: [PATCH 5/6] ci(rust-test): cache the downloaded DuckDB library with the target dir A pull request that restored the main cache reused the duckdb build-script output, which skipped the download, while rust-cache had pruned target/duckdb-download. Linking the test binaries then failed with `unable to find library -lduckdb`. --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4570ec1bb..16ccee4a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,10 @@ jobs: - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.ref == 'refs/heads/main' }} + # rust-cache keeps the duckdb build-script output but prunes the + # library it downloaded, so a cache hit skips the download and the + # test binaries fail to link with `-lduckdb`. Cache the library too. + cache-directories: target/duckdb-download - uses: taiki-e/install-action@v2 with: tool: cargo-nextest From bd68af725cf271e88481bd7a1f591c20bb045758 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Sun, 8 Nov 2026 15:33:32 -0500 Subject: [PATCH 6/6] ci(rust-test): key the cache so restores include the DuckDB library Existing caches predate the cache-directories entry; a new key forces one cold build that saves the library alongside the target dir. --- .github/workflows/ci.yml | 2 ++ docs/testing-and-operations.md | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16ccee4a0..d481e9b32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,8 @@ jobs: # library it downloaded, so a cache hit skips the download and the # test binaries fail to link with `-lduckdb`. Cache the library too. cache-directories: target/duckdb-download + # Bumped with cache-directories so restores carry the library. + key: duckdb-download - uses: taiki-e/install-action@v2 with: tool: cargo-nextest diff --git a/docs/testing-and-operations.md b/docs/testing-and-operations.md index 926c123df..7131bb3d1 100644 --- a/docs/testing-and-operations.md +++ b/docs/testing-and-operations.md @@ -647,6 +647,10 @@ Only `main` saves the Rust caches (`Swatinem/rust-cache` with `save-if`). A pull request can read the `main` cache but not another pull request's cache, so per-PR caches only consumed the 10 GB repository quota and evicted the shared one. +The `rust-test` cache also holds `target/duckdb-download`. rust-cache keeps the +`libduckdb-sys` build-script output but prunes directories it does not know, so +without that entry a cache hit skips the download and the test binaries fail to +link against `-lduckdb`. Timing evidence is attached to every run: