Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,12 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.ref == 'refs/heads/main' }}
# rust-cache keeps the duckdb build-script output but prunes the
# library it downloaded, so a cache hit skips the download and the
# test binaries fail to link with `-lduckdb`. Cache the library too.
cache-directories: target/duckdb-download
# Bumped with cache-directories so restores carry the library.
key: duckdb-download
- uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
Expand Down
46 changes: 46 additions & 0 deletions crates/opensymphony-cli/src/orchestrator_run/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,25 @@ use self::{

const MEMORY_SERVER_BIND_STATE: &str = ".opensymphony-memory-bind.json";

/// Overrides how often `opensymphony run` refreshes tracker states for running
/// issues. Intended for local diagnostics and process-level tests; production
/// runs keep the scheduler's 30-second default to bound Linear reads.
const RUNNING_STATE_REFRESH_INTERVAL_ENV: &str = "OPENSYMPHONY_RUNNING_STATE_REFRESH_MS";

fn running_state_refresh_interval_override(value: Option<&str>) -> Option<u64> {
let value = value?.trim();
match value.parse::<u64>() {
Ok(interval_ms) if interval_ms > 0 => Some(interval_ms),
_ => {
tracing::warn!(
value,
"ignoring invalid {RUNNING_STATE_REFRESH_INTERVAL_ENV}; expected a positive integer of milliseconds"
);
None
}
}
}

enum RunWake {
Shutdown,
Server(Result<io::Result<()>, tokio::task::JoinError>),
Expand Down Expand Up @@ -1236,6 +1255,17 @@ async fn run_orchestrator(args: RunArgs) -> Result<(), RunCommandError> {
let mut scheduler_config = SchedulerConfig::from_workflow(&runtime.workflow)?;
scheduler_config.max_retry_attempts = runtime.retry_max_attempts;
scheduler_config.repository_routing = runtime.repository_routing.clone();
if let Some(interval_ms) = running_state_refresh_interval_override(
std::env::var(RUNNING_STATE_REFRESH_INTERVAL_ENV)
.ok()
.as_deref(),
) {
tracing::info!(
interval_ms,
"using {RUNNING_STATE_REFRESH_INTERVAL_ENV} running-state refresh cadence"
);
scheduler_config.running_state_refresh_interval_ms = interval_ms;
}
let mut scheduler = Scheduler::new(tracker, workspace, worker, scheduler_config);

let mut recent_events = VecDeque::new();
Expand Down Expand Up @@ -2240,6 +2270,22 @@ mod tests {
use super::*;
use crate::opensymphony_memory::MemoryError;

#[test]
fn running_state_refresh_override_accepts_only_positive_milliseconds() {
assert_eq!(running_state_refresh_interval_override(None), None);
assert_eq!(
running_state_refresh_interval_override(Some("250")),
Some(250)
);
assert_eq!(
running_state_refresh_interval_override(Some(" 50 ")),
Some(50)
);
assert_eq!(running_state_refresh_interval_override(Some("0")), None);
assert_eq!(running_state_refresh_interval_override(Some("-5")), None);
assert_eq!(running_state_refresh_interval_override(Some("soon")), None);
}

#[tokio::test]
async fn pending_operator_flush_allows_worker_wake_and_shutdown() {
use crate::opensymphony_gateway_schema::approval::{
Expand Down
4 changes: 3 additions & 1 deletion crates/opensymphony-cli/tests/doctor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -660,8 +660,10 @@ fn write_bash_wrapper(path: PathBuf) {
}

fn real_bash_path() -> PathBuf {
// A non-login shell resolves the same `bash` this command launched without
// sourcing the developer's profile, which can take seconds.
let output = Command::new("bash")
.args(["-lc", "command -v bash"])
.args(["-c", "command -v bash"])
.output()
.expect("bash lookup should run");
assert!(
Expand Down
4 changes: 3 additions & 1 deletion crates/opensymphony-cli/tests/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -302,8 +302,10 @@ fn write_bash_wrapper(path: PathBuf) {
}

fn real_bash_path() -> PathBuf {
// A non-login shell resolves the same `bash` this command launched without
// sourcing the developer's profile, which can take seconds.
let output = Command::new("bash")
.args(["-lc", "command -v bash"])
.args(["-c", "command -v bash"])
.output()
.expect("bash lookup should run");
assert!(
Expand Down
49 changes: 35 additions & 14 deletions crates/opensymphony-cli/tests/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -632,7 +632,7 @@ async fn run_recovers_human_review_worker_and_interrupts_when_tracker_reports_me
.await;
seed_fake_openhands_conversation(&openhands, conversation_id, project.path()).await;

let mut child = spawn_run_child(project.path(), &[]);
let mut child = spawn_run_child_with_fast_running_refresh(project.path(), &[], None);

let snapshot = wait_for_merging_supersede_event(&format!("http://{bind_addr}/api/v1/snapshot"))
.await
Expand Down Expand Up @@ -661,12 +661,14 @@ async fn run_interrupts_active_codex_stdio_worker_when_tracker_reports_merging()
write_codex_merging_supersede_project_files(project.path(), linear.base_url(), bind_addr);
write_memory_config(project.path());

let mut child = spawn_run_child_with_codex_bin(
let mut child = spawn_run_child_with_fast_running_refresh(
project.path(),
&[],
fake_codex
.to_str()
.expect("fake codex path should be utf-8"),
Some(
fake_codex
.to_str()
.expect("fake codex path should be utf-8"),
),
);

wait_for_codex_merging_interrupt_ack(&format!("http://{bind_addr}/api/v1/snapshot"))
Expand Down Expand Up @@ -814,30 +816,42 @@ Run the scheduler.
}

fn spawn_run_child(project_root: &std::path::Path, extra_args: &[&str]) -> Child {
spawn_run_child_configured(project_root, extra_args, None, None)
spawn_run_child_configured(project_root, extra_args, None, None, None)
}

fn spawn_run_child_with_worker_stack(
/// Running-state refresh cadence for tests that wait for the scheduler to
/// observe a tracker transition on an already-running issue. The production
/// default is 30 seconds; the mock tracker is local, so poll it every tick.
const FAST_RUNNING_STATE_REFRESH_MS: u64 = 50;

fn spawn_run_child_with_fast_running_refresh(
project_root: &std::path::Path,
extra_args: &[&str],
stack_bytes: usize,
codex_bin: Option<&str>,
) -> Child {
spawn_run_child_configured(project_root, extra_args, None, Some(stack_bytes))
spawn_run_child_configured(
project_root,
extra_args,
codex_bin,
None,
Some(FAST_RUNNING_STATE_REFRESH_MS),
)
}

fn spawn_run_child_with_codex_bin(
fn spawn_run_child_with_worker_stack(
project_root: &std::path::Path,
extra_args: &[&str],
codex_bin: &str,
stack_bytes: usize,
) -> Child {
spawn_run_child_configured(project_root, extra_args, Some(codex_bin), None)
spawn_run_child_configured(project_root, extra_args, None, Some(stack_bytes), None)
}

fn spawn_run_child_configured(
project_root: &std::path::Path,
extra_args: &[&str],
codex_bin: Option<&str>,
worker_stack_bytes: Option<usize>,
running_state_refresh_ms: Option<u64>,
) -> Child {
let mut command = Command::new(env!("CARGO_BIN_EXE_opensymphony"));
command
Expand All @@ -852,12 +866,19 @@ fn spawn_run_child_configured(
.env_remove("LLM_MODEL")
.env_remove("LLM_API_KEY")
.env_remove("LLM_BASE_URL")
.env_remove("OPENSYMPHONY_RUNNING_STATE_REFRESH_MS")
.stdout(Stdio::null())
.stderr(Stdio::piped())
.kill_on_drop(true);
if let Some(codex_bin) = codex_bin {
command.env("OPENSYMPHONY_CODEX_BIN", codex_bin);
}
if let Some(interval_ms) = running_state_refresh_ms {
command.env(
"OPENSYMPHONY_RUNNING_STATE_REFRESH_MS",
interval_ms.to_string(),
);
}
if cfg!(unix)
&& let Some(stack_bytes) = worker_stack_bytes
{
Expand Down Expand Up @@ -1190,7 +1211,7 @@ async fn wait_for_merging_supersede_event(url: &str) -> Result<Value, String> {
}
last_snapshot = Some(snapshot);
}
sleep(Duration::from_millis(250)).await;
sleep(Duration::from_millis(50)).await;
}
Err(format!(
"timed out waiting for Merging supersede event at {url}; last snapshot: {}",
Expand All @@ -1212,7 +1233,7 @@ async fn wait_for_codex_merging_interrupt_ack(url: &str) -> Result<Value, String
{
return Ok(snapshot);
}
sleep(Duration::from_millis(250)).await;
sleep(Duration::from_millis(50)).await;
}
Err(format!(
"timed out waiting for Codex Merging interrupt acknowledgement at {url}"
Expand Down
27 changes: 27 additions & 0 deletions crates/opensymphony-gateway/tests/gateway.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3436,6 +3436,33 @@ async fn gateway_serves_code_graph_contract_endpoints() {
.contains(&"src/empty.rs".to_string())
);

server_task.abort();
}

/// The atlas caps file nodes at the 500-record gateway contract limit. These
/// fixtures need more than 500 documents each, so they use their own store and
/// run alongside the contract-endpoint checks above.
#[tokio::test]
async fn gateway_code_graph_atlas_caps_and_dedupes_large_repositories() {
let repo = tempfile::tempdir().expect("memory repo");
let config_for_revision_regression =
MemoryConfig::load(repo.path(), None).expect("memory config should load");
let store = SnapshotStore::new(fixture_snapshot(0));
let server =
GatewayServer::new(store).with_memory_config(Some(config_for_revision_regression.clone()));
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind test listener");
let address = listener.local_addr().expect("test listener address");
let server_task = tokio::spawn(async move {
server
.serve(listener)
.await
.expect("test gateway server should serve")
});
let client = reqwest::Client::new();
let base = format!("http://{address}/api/v1/code");

persist_code_intel_documents(
&config_for_revision_regression,
CodeIntelPersistBatch {
Expand Down
10 changes: 9 additions & 1 deletion crates/opensymphony-orchestrator/src/scheduler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ use super::{

const DISABLED_STALL_TIMEOUT_MS: u64 = u64::MAX / 4;
const ROUTING_TASK_ISSUE_EXECUTION: &str = "issue_execution";
/// Default cadence for refreshing tracker states of issues the scheduler is
/// already running. Operators keep this default; `SchedulerConfig` allows a
/// shorter cadence for local diagnostics and process-level tests.
const RUNNING_STATE_REFRESH_INTERVAL_MS: u64 = 30_000;
const DISPATCH_DISCOVERY_INTERVAL_MS: u64 = 60_000;
const TERMINAL_REFRESH_INTERVAL_MS: u64 = 300_000;
Expand Down Expand Up @@ -213,6 +216,9 @@ pub struct SchedulerConfig {
pub tracker_project_slugs: Vec<String>,
pub routing: RoutingConfig,
pub repository_routing: Option<RepositoryRouting>,
/// How often running issues have their tracker state refreshed between
/// full-detail snapshots. `from_workflow` uses the 30-second default.
pub running_state_refresh_interval_ms: u64,
}

impl SchedulerConfig {
Expand Down Expand Up @@ -272,6 +278,7 @@ impl SchedulerConfig {
tracker_project_slugs: workflow.config.tracker.project_slugs.clone(),
routing: workflow.config.routing.clone(),
repository_routing: None,
running_state_refresh_interval_ms: RUNNING_STATE_REFRESH_INTERVAL_MS,
})
}

Expand Down Expand Up @@ -2573,7 +2580,7 @@ where
&& pre_update_full_snapshot.is_none()
&& due(
self.last_running_state_refresh_at,
RUNNING_STATE_REFRESH_INTERVAL_MS,
self.config.running_state_refresh_interval_ms,
observed_at,
)
&& self.has_reconcilable_executions()
Expand Down Expand Up @@ -10774,6 +10781,7 @@ mod tests {
dry_run: false,
},
repository_routing: None,
running_state_refresh_interval_ms: 30_000,
};

assert!(!project_belongs_to_configured_project(
Expand Down
1 change: 1 addition & 0 deletions crates/opensymphony-orchestrator/tests/scheduler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ fn scheduler_config() -> SchedulerConfig {
dry_run: false,
},
repository_routing: None,
running_state_refresh_interval_ms: 30_000,
}
}

Expand Down
6 changes: 6 additions & 0 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,12 @@ For local Codex stdio execution, `OPENSYMPHONY_CODEX_BIN` may point to an
alternate Codex binary in trusted operator environments only; it is not a
hosted-mode tenant input.

`OPENSYMPHONY_RUNNING_STATE_REFRESH_MS` sets how often `opensymphony run`
re-reads tracker states for issues it is already running. The default is
30000 ms. Shorter values make tracker transitions such as Merging visible
sooner at the cost of more Linear reads, so reserve them for local diagnostics
and tests against a local tracker stub.

OpenAI ChatGPT/Codex subscription credentials are available only when
OpenSymphony is built with the `openhands-subscription-credentials` Cargo
feature. The workflow stores environment-variable names and auth-directory
Expand Down
6 changes: 5 additions & 1 deletion docs/operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,11 @@ Operational implications:
- `opensymphony run` keeps its local worker/snapshot tick every 5s, while
Linear reads use cheaper internal cadences: running state every 30s,
dispatch discovery every 60s, terminal cleanup every 5 minutes, and full
issue details hourly after startup/dispatch. When all required children of a
issue details hourly after startup/dispatch.
`OPENSYMPHONY_RUNNING_STATE_REFRESH_MS` overrides the running-state cadence
with a positive millisecond value for local diagnostics and process-level
tests; invalid values are ignored with a warning. Leave it unset against
hosted Linear so running-state reads stay bounded. When all required children of a
waiting parent have terminal orchestrator outcomes, full details refresh on
the 5-minute terminal cadence so parent eligibility can use a complete
hierarchy observation
Expand Down
11 changes: 11 additions & 0 deletions docs/testing-and-operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,10 @@ Only `main` saves the Rust caches (`Swatinem/rust-cache` with `save-if`). A pull
request can read the `main` cache but not another pull request's cache, so
per-PR caches only consumed the 10 GB repository quota and evicted the shared
one.
The `rust-test` cache also holds `target/duckdb-download`. rust-cache keeps the
`libduckdb-sys` build-script output but prunes directories it does not know, so
without that entry a cache hit skips the download and the test binaries fail to
link against `-lduckdb`.

Timing evidence is attached to every run:

Expand All @@ -664,6 +668,13 @@ than 30 s without terminating them. To reproduce the CI test run locally, use
with `DUCKDB_DOWNLOAD_LIB=1`, or the system-DuckDB environment from the
`cargo *-system-duckdb` aliases.

Keep long tests event-driven rather than clock-driven. A process-level test
that waits for the scheduler to notice a tracker transition on a running issue
sets `OPENSYMPHONY_RUNNING_STATE_REFRESH_MS` on the `opensymphony run` child
instead of waiting out the 30 s production cadence. Test helpers that need a
real shell resolve it with a non-login `bash -c`; a login shell sources the
developer's profile and can add seconds per call.

## 6. Operations

Operational guidance now lives in [operations.md](operations.md).
Expand Down
Loading