diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4570ec1bb..d481e9b32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,12 @@ jobs: - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.ref == 'refs/heads/main' }} + # rust-cache keeps the duckdb build-script output but prunes the + # library it downloaded, so a cache hit skips the download and the + # test binaries fail to link with `-lduckdb`. Cache the library too. + cache-directories: target/duckdb-download + # Bumped with cache-directories so restores carry the library. + key: duckdb-download - uses: taiki-e/install-action@v2 with: tool: cargo-nextest diff --git a/crates/opensymphony-cli/src/orchestrator_run/mod.rs b/crates/opensymphony-cli/src/orchestrator_run/mod.rs index 3d8c27eb8..d3fae68ef 100644 --- a/crates/opensymphony-cli/src/orchestrator_run/mod.rs +++ b/crates/opensymphony-cli/src/orchestrator_run/mod.rs @@ -68,6 +68,25 @@ use self::{ const MEMORY_SERVER_BIND_STATE: &str = ".opensymphony-memory-bind.json"; +/// Overrides how often `opensymphony run` refreshes tracker states for running +/// issues. Intended for local diagnostics and process-level tests; production +/// runs keep the scheduler's 30-second default to bound Linear reads. +const RUNNING_STATE_REFRESH_INTERVAL_ENV: &str = "OPENSYMPHONY_RUNNING_STATE_REFRESH_MS"; + +fn running_state_refresh_interval_override(value: Option<&str>) -> Option { + let value = value?.trim(); + match value.parse::() { + Ok(interval_ms) if interval_ms > 0 => Some(interval_ms), + _ => { + tracing::warn!( + value, + "ignoring invalid {RUNNING_STATE_REFRESH_INTERVAL_ENV}; expected a positive integer of milliseconds" + ); + None + } + } +} + enum RunWake { Shutdown, Server(Result, tokio::task::JoinError>), @@ -1236,6 +1255,17 @@ async fn run_orchestrator(args: RunArgs) -> Result<(), RunCommandError> { let mut scheduler_config = SchedulerConfig::from_workflow(&runtime.workflow)?; scheduler_config.max_retry_attempts = runtime.retry_max_attempts; scheduler_config.repository_routing = runtime.repository_routing.clone(); + if let Some(interval_ms) = running_state_refresh_interval_override( + std::env::var(RUNNING_STATE_REFRESH_INTERVAL_ENV) + .ok() + .as_deref(), + ) { + tracing::info!( + interval_ms, + "using {RUNNING_STATE_REFRESH_INTERVAL_ENV} running-state refresh cadence" + ); + scheduler_config.running_state_refresh_interval_ms = interval_ms; + } let mut scheduler = Scheduler::new(tracker, workspace, worker, scheduler_config); let mut recent_events = VecDeque::new(); @@ -2240,6 +2270,22 @@ mod tests { use super::*; use crate::opensymphony_memory::MemoryError; + #[test] + fn running_state_refresh_override_accepts_only_positive_milliseconds() { + assert_eq!(running_state_refresh_interval_override(None), None); + assert_eq!( + running_state_refresh_interval_override(Some("250")), + Some(250) + ); + assert_eq!( + running_state_refresh_interval_override(Some(" 50 ")), + Some(50) + ); + assert_eq!(running_state_refresh_interval_override(Some("0")), None); + assert_eq!(running_state_refresh_interval_override(Some("-5")), None); + assert_eq!(running_state_refresh_interval_override(Some("soon")), None); + } + #[tokio::test] async fn pending_operator_flush_allows_worker_wake_and_shutdown() { use crate::opensymphony_gateway_schema::approval::{ diff --git a/crates/opensymphony-cli/tests/doctor.rs b/crates/opensymphony-cli/tests/doctor.rs index 5a13f78c9..044886bc0 100644 --- a/crates/opensymphony-cli/tests/doctor.rs +++ b/crates/opensymphony-cli/tests/doctor.rs @@ -660,8 +660,10 @@ fn write_bash_wrapper(path: PathBuf) { } fn real_bash_path() -> PathBuf { + // A non-login shell resolves the same `bash` this command launched without + // sourcing the developer's profile, which can take seconds. let output = Command::new("bash") - .args(["-lc", "command -v bash"]) + .args(["-c", "command -v bash"]) .output() .expect("bash lookup should run"); assert!( diff --git a/crates/opensymphony-cli/tests/install.rs b/crates/opensymphony-cli/tests/install.rs index 9918872c4..f3a44d22e 100644 --- a/crates/opensymphony-cli/tests/install.rs +++ b/crates/opensymphony-cli/tests/install.rs @@ -302,8 +302,10 @@ fn write_bash_wrapper(path: PathBuf) { } fn real_bash_path() -> PathBuf { + // A non-login shell resolves the same `bash` this command launched without + // sourcing the developer's profile, which can take seconds. let output = Command::new("bash") - .args(["-lc", "command -v bash"]) + .args(["-c", "command -v bash"]) .output() .expect("bash lookup should run"); assert!( diff --git a/crates/opensymphony-cli/tests/run.rs b/crates/opensymphony-cli/tests/run.rs index 10f168b28..1b3185bfa 100644 --- a/crates/opensymphony-cli/tests/run.rs +++ b/crates/opensymphony-cli/tests/run.rs @@ -632,7 +632,7 @@ async fn run_recovers_human_review_worker_and_interrupts_when_tracker_reports_me .await; seed_fake_openhands_conversation(&openhands, conversation_id, project.path()).await; - let mut child = spawn_run_child(project.path(), &[]); + let mut child = spawn_run_child_with_fast_running_refresh(project.path(), &[], None); let snapshot = wait_for_merging_supersede_event(&format!("http://{bind_addr}/api/v1/snapshot")) .await @@ -661,12 +661,14 @@ async fn run_interrupts_active_codex_stdio_worker_when_tracker_reports_merging() write_codex_merging_supersede_project_files(project.path(), linear.base_url(), bind_addr); write_memory_config(project.path()); - let mut child = spawn_run_child_with_codex_bin( + let mut child = spawn_run_child_with_fast_running_refresh( project.path(), &[], - fake_codex - .to_str() - .expect("fake codex path should be utf-8"), + Some( + fake_codex + .to_str() + .expect("fake codex path should be utf-8"), + ), ); wait_for_codex_merging_interrupt_ack(&format!("http://{bind_addr}/api/v1/snapshot")) @@ -814,23 +816,34 @@ Run the scheduler. } fn spawn_run_child(project_root: &std::path::Path, extra_args: &[&str]) -> Child { - spawn_run_child_configured(project_root, extra_args, None, None) + spawn_run_child_configured(project_root, extra_args, None, None, None) } -fn spawn_run_child_with_worker_stack( +/// Running-state refresh cadence for tests that wait for the scheduler to +/// observe a tracker transition on an already-running issue. The production +/// default is 30 seconds; the mock tracker is local, so poll it every tick. +const FAST_RUNNING_STATE_REFRESH_MS: u64 = 50; + +fn spawn_run_child_with_fast_running_refresh( project_root: &std::path::Path, extra_args: &[&str], - stack_bytes: usize, + codex_bin: Option<&str>, ) -> Child { - spawn_run_child_configured(project_root, extra_args, None, Some(stack_bytes)) + spawn_run_child_configured( + project_root, + extra_args, + codex_bin, + None, + Some(FAST_RUNNING_STATE_REFRESH_MS), + ) } -fn spawn_run_child_with_codex_bin( +fn spawn_run_child_with_worker_stack( project_root: &std::path::Path, extra_args: &[&str], - codex_bin: &str, + stack_bytes: usize, ) -> Child { - spawn_run_child_configured(project_root, extra_args, Some(codex_bin), None) + spawn_run_child_configured(project_root, extra_args, None, Some(stack_bytes), None) } fn spawn_run_child_configured( @@ -838,6 +851,7 @@ fn spawn_run_child_configured( extra_args: &[&str], codex_bin: Option<&str>, worker_stack_bytes: Option, + running_state_refresh_ms: Option, ) -> Child { let mut command = Command::new(env!("CARGO_BIN_EXE_opensymphony")); command @@ -852,12 +866,19 @@ fn spawn_run_child_configured( .env_remove("LLM_MODEL") .env_remove("LLM_API_KEY") .env_remove("LLM_BASE_URL") + .env_remove("OPENSYMPHONY_RUNNING_STATE_REFRESH_MS") .stdout(Stdio::null()) .stderr(Stdio::piped()) .kill_on_drop(true); if let Some(codex_bin) = codex_bin { command.env("OPENSYMPHONY_CODEX_BIN", codex_bin); } + if let Some(interval_ms) = running_state_refresh_ms { + command.env( + "OPENSYMPHONY_RUNNING_STATE_REFRESH_MS", + interval_ms.to_string(), + ); + } if cfg!(unix) && let Some(stack_bytes) = worker_stack_bytes { @@ -1190,7 +1211,7 @@ async fn wait_for_merging_supersede_event(url: &str) -> Result { } last_snapshot = Some(snapshot); } - sleep(Duration::from_millis(250)).await; + sleep(Duration::from_millis(50)).await; } Err(format!( "timed out waiting for Merging supersede event at {url}; last snapshot: {}", @@ -1212,7 +1233,7 @@ async fn wait_for_codex_merging_interrupt_ack(url: &str) -> Result, pub routing: RoutingConfig, pub repository_routing: Option, + /// How often running issues have their tracker state refreshed between + /// full-detail snapshots. `from_workflow` uses the 30-second default. + pub running_state_refresh_interval_ms: u64, } impl SchedulerConfig { @@ -272,6 +278,7 @@ impl SchedulerConfig { tracker_project_slugs: workflow.config.tracker.project_slugs.clone(), routing: workflow.config.routing.clone(), repository_routing: None, + running_state_refresh_interval_ms: RUNNING_STATE_REFRESH_INTERVAL_MS, }) } @@ -2573,7 +2580,7 @@ where && pre_update_full_snapshot.is_none() && due( self.last_running_state_refresh_at, - RUNNING_STATE_REFRESH_INTERVAL_MS, + self.config.running_state_refresh_interval_ms, observed_at, ) && self.has_reconcilable_executions() @@ -10774,6 +10781,7 @@ mod tests { dry_run: false, }, repository_routing: None, + running_state_refresh_interval_ms: 30_000, }; assert!(!project_belongs_to_configured_project( diff --git a/crates/opensymphony-orchestrator/tests/scheduler.rs b/crates/opensymphony-orchestrator/tests/scheduler.rs index 7e7a2ed8d..e0d71a0f6 100644 --- a/crates/opensymphony-orchestrator/tests/scheduler.rs +++ b/crates/opensymphony-orchestrator/tests/scheduler.rs @@ -80,6 +80,7 @@ fn scheduler_config() -> SchedulerConfig { dry_run: false, }, repository_routing: None, + running_state_refresh_interval_ms: 30_000, } } diff --git a/docs/configuration.md b/docs/configuration.md index 31578de62..12d5486cb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -566,6 +566,12 @@ For local Codex stdio execution, `OPENSYMPHONY_CODEX_BIN` may point to an alternate Codex binary in trusted operator environments only; it is not a hosted-mode tenant input. +`OPENSYMPHONY_RUNNING_STATE_REFRESH_MS` sets how often `opensymphony run` +re-reads tracker states for issues it is already running. The default is +30000 ms. Shorter values make tracker transitions such as Merging visible +sooner at the cost of more Linear reads, so reserve them for local diagnostics +and tests against a local tracker stub. + OpenAI ChatGPT/Codex subscription credentials are available only when OpenSymphony is built with the `openhands-subscription-credentials` Cargo feature. The workflow stores environment-variable names and auth-directory diff --git a/docs/operations.md b/docs/operations.md index f99ab471d..2f66c6def 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -482,7 +482,11 @@ Operational implications: - `opensymphony run` keeps its local worker/snapshot tick every 5s, while Linear reads use cheaper internal cadences: running state every 30s, dispatch discovery every 60s, terminal cleanup every 5 minutes, and full - issue details hourly after startup/dispatch. When all required children of a + issue details hourly after startup/dispatch. + `OPENSYMPHONY_RUNNING_STATE_REFRESH_MS` overrides the running-state cadence + with a positive millisecond value for local diagnostics and process-level + tests; invalid values are ignored with a warning. Leave it unset against + hosted Linear so running-state reads stay bounded. When all required children of a waiting parent have terminal orchestrator outcomes, full details refresh on the 5-minute terminal cadence so parent eligibility can use a complete hierarchy observation diff --git a/docs/testing-and-operations.md b/docs/testing-and-operations.md index 9c98fd2ad..7131bb3d1 100644 --- a/docs/testing-and-operations.md +++ b/docs/testing-and-operations.md @@ -647,6 +647,10 @@ Only `main` saves the Rust caches (`Swatinem/rust-cache` with `save-if`). A pull request can read the `main` cache but not another pull request's cache, so per-PR caches only consumed the 10 GB repository quota and evicted the shared one. +The `rust-test` cache also holds `target/duckdb-download`. rust-cache keeps the +`libduckdb-sys` build-script output but prunes directories it does not know, so +without that entry a cache hit skips the download and the test binaries fail to +link against `-lduckdb`. Timing evidence is attached to every run: @@ -664,6 +668,13 @@ than 30 s without terminating them. To reproduce the CI test run locally, use with `DUCKDB_DOWNLOAD_LIB=1`, or the system-DuckDB environment from the `cargo *-system-duckdb` aliases. +Keep long tests event-driven rather than clock-driven. A process-level test +that waits for the scheduler to notice a tracker transition on a running issue +sets `OPENSYMPHONY_RUNNING_STATE_REFRESH_MS` on the `opensymphony run` child +instead of waiting out the 30 s production cadence. Test helpers that need a +real shell resolve it with a non-login `bash -c`; a login shell sources the +developer's profile and can add seconds per call. + ## 6. Operations Operational guidance now lives in [operations.md](operations.md).