From 8fe9ea333c0d0b16cfb24b6d2b5476eca574563d Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Fri, 18 Sep 2026 11:20:01 +0800 Subject: [PATCH 1/6] feat(agents-api): add bounded workspace output exporter --- CONTRIBUTING.md | 10 ++ packages/codex-executor/Cargo.lock | 11 ++ packages/codex-executor/Cargo.toml | 5 + packages/codex-executor/README.md | 23 +++ packages/codex-executor/src/bin/export.rs | 20 +++ packages/codex-executor/src/export.rs | 142 +++++++++++++++++ packages/codex-executor/src/export_tests.rs | 160 ++++++++++++++++++++ scripts/build-agents-executor.sh | 2 +- 8 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 packages/codex-executor/src/bin/export.rs create mode 100644 packages/codex-executor/src/export.rs create mode 100644 packages/codex-executor/src/export_tests.rs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38abb570..12ec9eb5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -424,6 +424,16 @@ large objects; live deletion does not erase WAL or historical backups. Schema rollback must not orphan existing source objects. Do not reuse product capability tables or introduce a second destination writer for file_id. +Session Artifacts are immutable published output copies, separate from live +workspace files and general source Files. A private output exporter must reuse +the authorized workspace path boundary and stream bounded bytes. Require complete +capture and confirmed helper/transport success before publication; valid archive +syntax alone is insufficient. Never extract an output archive into Core's +filesystem or hold the global execution lease through a large transfer. Keep +publication ordered with Turn completion, and authorize stored reads independently +of Environment availability so published outputs can survive its expiration. +Exporter component checks do not establish public Artifact compatibility. + Local inline file delivery uses the same authenticated daemon connection and exact Environment/Session binding. The optional startup-owned `PARSAR_RUNTIME_WRITE_HELPER` and `PARSAR_RUNTIME_STAGING` enable only the bounded installer primitive; they do diff --git a/packages/codex-executor/Cargo.lock b/packages/codex-executor/Cargo.lock index dc55705d..d63b1387 100644 --- a/packages/codex-executor/Cargo.lock +++ b/packages/codex-executor/Cargo.lock @@ -66,6 +66,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "tar", "tempfile", "tokio", "url", @@ -6880,6 +6881,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", +] + [[package]] name = "tempfile" version = "3.27.0" diff --git a/packages/codex-executor/Cargo.toml b/packages/codex-executor/Cargo.toml index 3e28c698..7fb9ebc4 100644 --- a/packages/codex-executor/Cargo.toml +++ b/packages/codex-executor/Cargo.toml @@ -13,6 +13,10 @@ path = "src/bin/directory.rs" name = "agents-api-codex-write" path = "src/bin/write.rs" +[[bin]] +name = "agents-api-workspace-export" +path = "src/bin/export.rs" + [dependencies] codex-api = { git = "https://github.com/openai/codex", rev = "3d2ee51ca2d5db578f328aa75e20aa22c0197c9a" } codex-exec-server = { git = "https://github.com/openai/codex", rev = "3d2ee51ca2d5db578f328aa75e20aa22c0197c9a" } @@ -23,6 +27,7 @@ rustix = { version = "=1.1.4", features = ["fs"] } serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "=0.10.9" +tar = { version = "=0.4.46", default-features = false } tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] } url = "2" uuid = { version = "1", features = ["v4"] } diff --git a/packages/codex-executor/README.md b/packages/codex-executor/README.md index 8682e2d6..8f5c3e43 100644 --- a/packages/codex-executor/README.md +++ b/packages/codex-executor/README.md @@ -66,6 +66,29 @@ not change stock native filesystem methods, create a daemon connection, or enabl public Files. The [native fixture](../../services/agents-api/tests/native/directory/README.md) qualifies the standalone helper independently of later adapter/public wiring. +## Scoped output exporter + +`agents-api-workspace-export` takes one authorized absolute workspace root and +streams regular files beneath its `outputs` directory as a standard tar archive +on stdout. It reuses the directory helper's descriptor-relative path protection; +it does not invoke a shell, model or provider command. The caller must supply the +exact Environment's frozen root and independently authorize the operation. + +Require both a complete validated archive and successful process exit before +publishing anything. On failure the stdout prefix may still look like a valid +archive. Never extract it into Core's filesystem. Consumers must bound and stream +individual entries into private storage, then publish only after the entire +capture succeeds. A missing `outputs` directory produces an empty archive. + +The exporter rejects symlinks, multiply linked files, special files, device +changes and detected concurrent modifications. It limits each file to 200 MiB +and aggregate bytes to 500 MiB, following the current official Files guide. +Traversal is additionally bounded at 4096 entries, 64 directory levels and +4096-byte relative paths; those are implementation limits, not upstream promises. +This is not a filesystem-wide point-in-time snapshot. Immutable publication, +tenant isolation, Turn ordering and storage cleanup remain Core/Runtime duties; +the helper alone does not enable public Artifacts. + ## Scoped file installer The optional `agents-api-codex-write` helper addresses two pinned native write diff --git a/packages/codex-executor/src/bin/export.rs b/packages/codex-executor/src/bin/export.rs new file mode 100644 index 00000000..d1ca3672 --- /dev/null +++ b/packages/codex-executor/src/bin/export.rs @@ -0,0 +1,20 @@ +#[path = "../directory.rs"] +mod directory; +#[path = "../export.rs"] +mod export; +#[path = "../workspace_path.rs"] +mod workspace_path; + +use std::{io, path::Path, process::ExitCode}; + +fn main() -> ExitCode { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + if args.len() != 1 { + return ExitCode::FAILURE; + } + // A valid archive prefix alone is not success: callers must also observe exit 0. + match export::outputs(Path::new(&args[0]), io::stdout().lock()) { + Ok(()) => ExitCode::SUCCESS, + Err(_) => ExitCode::FAILURE, + } +} diff --git a/packages/codex-executor/src/export.rs b/packages/codex-executor/src/export.rs new file mode 100644 index 00000000..5ba7df83 --- /dev/null +++ b/packages/codex-executor/src/export.rs @@ -0,0 +1,142 @@ +use crate::{directory::observe, workspace_path}; +use rustix::fs::{FileType, Mode, OFlags, Stat, fstat, openat}; +use std::fs::File; +use std::io::{self, Read, Write}; +use std::os::fd::OwnedFd; +use std::path::Path; + +const MAX_FILE_BYTES: u64 = 200 * 1024 * 1024; +const MAX_TOTAL_BYTES: u64 = 500 * 1024 * 1024; +// Defensive traversal bounds, not upstream protocol limits. +const MAX_ENTRIES: usize = 4096; +const MAX_DEPTH: usize = 64; + +#[derive(Default)] +struct Budget { + entries: usize, + bytes: u64, +} + +fn invalid() -> io::Error { + io::ErrorKind::InvalidData.into() +} + +pub(crate) fn outputs(root: &Path, output: impl Write) -> io::Result<()> { + let root = workspace_path::anchor(root)?; + let mut archive = tar::Builder::new(output); + let selected = match workspace_path::directory(&root, "outputs") { + Ok(fd) => fd, + Err(error) if error.kind() == io::ErrorKind::NotFound => return archive.finish(), + Err(error) => return Err(error), + }; + let device = fstat(&root)?.st_dev; + walk( + &selected, + "outputs", + device, + 0, + &mut Budget::default(), + &mut archive, + )?; + archive.finish() +} + +fn walk( + parent: &OwnedFd, + path: &str, + device: u64, + depth: usize, + budget: &mut Budget, + archive: &mut tar::Builder, +) -> io::Result<()> { + if depth > MAX_DEPTH || path.len() > 4096 { + return Err(invalid()); + } + let before = fstat(parent)?; + if before.st_dev != device { + return Err(invalid()); + } + let mut listing = observe(parent, MAX_ENTRIES)?; + budget.entries += listing.visited; + if listing.truncated || budget.entries > MAX_ENTRIES { + return Err(invalid()); + } + listing.entries.sort_by(|a, b| a.name.cmp(&b.name)); + for entry in &listing.entries { + let path = format!("{path}/{}", entry.name); + if path.len() > 4096 { + return Err(invalid()); + } + match entry.kind { + FileType::Directory => { + let child = workspace_path::directory(parent, &entry.name)?; + walk(&child, &path, device, depth + 1, budget, archive)?; + } + FileType::RegularFile => { + let file = openat( + parent, + entry.name.as_str(), + OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC, + Mode::empty(), + )?; + append(File::from(file), &path, device, budget, archive)?; + } + _ => return Err(invalid()), + } + } + // Directory timestamps may have coarser resolution than successive mutations. + let mut after = observe(parent, MAX_ENTRIES)?; + after.entries.sort_by(|a, b| a.name.cmp(&b.name)); + if after.truncated || after.entries != listing.entries || !unchanged(&before, &fstat(parent)?) { + return Err(invalid()); + } + Ok(()) +} + +fn append( + mut file: File, + path: &str, + device: u64, + budget: &mut Budget, + archive: &mut tar::Builder, +) -> io::Result<()> { + let before = fstat(&file)?; + if FileType::from_raw_mode(before.st_mode) != FileType::RegularFile + || before.st_dev != device + || before.st_nlink != 1 + { + return Err(invalid()); + } + let size = u64::try_from(before.st_size).map_err(|_| invalid())?; + if size > MAX_FILE_BYTES || size > MAX_TOTAL_BYTES - budget.bytes { + return Err(invalid()); + } + budget.bytes += size; + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_mode(0o600); + header.set_size(size); + // Read through the held descriptor, never reopen an attacker-controlled path. + let mut body = (&mut file).take(size); + archive.append_data(&mut header, path, &mut body)?; + if body.limit() != 0 || !unchanged(&before, &fstat(&file)?) { + return Err(invalid()); + } + Ok(()) +} + +fn unchanged(a: &Stat, b: &Stat) -> bool { + a.st_dev == b.st_dev + && a.st_ino == b.st_ino + && a.st_mode == b.st_mode + && a.st_nlink == b.st_nlink + && a.st_size == b.st_size + && a.st_mtime == b.st_mtime + && a.st_mtime_nsec == b.st_mtime_nsec + && a.st_ctime == b.st_ctime + && a.st_ctime_nsec == b.st_ctime_nsec +} + +#[cfg(test)] +#[path = "export_tests.rs"] +mod tests; diff --git a/packages/codex-executor/src/export_tests.rs b/packages/codex-executor/src/export_tests.rs new file mode 100644 index 00000000..61c8091f --- /dev/null +++ b/packages/codex-executor/src/export_tests.rs @@ -0,0 +1,160 @@ +use super::*; +use std::collections::BTreeMap; +use std::fs; +use std::os::unix::{fs::symlink, net::UnixListener}; + +#[test] +fn captures_nested_binary_empty_and_long_names_without_other_workspace_files() { + let root = tempfile::tempdir().unwrap(); + fs::create_dir_all(root.path().join("outputs/nested")).unwrap(); + fs::write(root.path().join("private.txt"), "not an output").unwrap(); + fs::write(root.path().join("outputs/empty"), []).unwrap(); + let body: Vec<_> = (0..=255).cycle().take(131_079).collect(); + let name = format!("outputs/nested/{}", "x".repeat(200)); + fs::write(root.path().join(&name), &body).unwrap(); + let mut bytes = Vec::new(); + outputs(root.path(), &mut bytes).unwrap(); + let mut archive = tar::Archive::new(bytes.as_slice()); + let mut actual = BTreeMap::new(); + for entry in archive.entries().unwrap() { + let mut entry = entry.unwrap(); + assert!(entry.header().entry_type().is_file()); + let name = entry.path().unwrap().to_str().unwrap().to_owned(); + let mut data = Vec::new(); + entry.read_to_end(&mut data).unwrap(); + actual.insert(name, data); + } + assert_eq!( + actual, + BTreeMap::from([("outputs/empty".into(), vec![]), (name, body)]) + ); +} + +#[test] +fn absent_outputs_is_an_empty_archive() { + let root = tempfile::tempdir().unwrap(); + let mut bytes = Vec::new(); + outputs(root.path(), &mut bytes).unwrap(); + assert_eq!( + tar::Archive::new(bytes.as_slice()) + .entries() + .unwrap() + .count(), + 0 + ); +} + +#[test] +fn rejects_links_and_special_files_without_exposing_their_contents() { + for kind in ["root-symlink", "file-symlink", "hardlink", "socket"] { + let root = tempfile::tempdir().unwrap(); + let other = tempfile::tempdir().unwrap(); + let secret = b"private daemon credential marker"; + fs::write(other.path().join("secret"), secret).unwrap(); + if kind == "root-symlink" { + symlink(other.path(), root.path().join("outputs")).unwrap(); + } else { + fs::create_dir(root.path().join("outputs")).unwrap(); + } + let target = root.path().join("outputs/file"); + let _socket = match kind { + "file-symlink" => { + symlink(other.path().join("secret"), target).unwrap(); + None + } + "hardlink" => { + fs::hard_link(other.path().join("secret"), target).unwrap(); + None + } + "socket" => Some(UnixListener::bind(target).unwrap()), + _ => None, + }; + let mut bytes = Vec::new(); + assert!(outputs(root.path(), &mut bytes).is_err(), "{kind}"); + assert!( + !bytes.windows(secret.len()).any(|part| part == secret), + "{kind}" + ); + } +} + +#[test] +fn enforces_file_and_total_bytes_without_loading_bodies() { + for sizes in [ + vec![MAX_FILE_BYTES + 1], + vec![MAX_FILE_BYTES, MAX_FILE_BYTES, 101 << 20], + ] { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join("outputs")).unwrap(); + for (index, size) in sizes.iter().enumerate() { + File::create(root.path().join(format!("outputs/{index}"))) + .unwrap() + .set_len(*size) + .unwrap(); + } + assert!(outputs(root.path(), io::sink()).is_err()); + } +} + +#[test] +fn rejects_a_changed_file_or_directory_during_export() { + struct Mutate<'a> { + root: &'a Path, + directory: bool, + done: bool, + } + impl Write for Mutate<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if !self.done { + self.done = true; + if self.directory { + fs::write(self.root.join("outputs/new"), "late")?; + } else { + fs::write(self.root.join("outputs/file"), "changed-length")?; + } + } + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + for directory in [false, true] { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join("outputs")).unwrap(); + fs::write(root.path().join("outputs/file"), "initial").unwrap(); + assert!( + outputs( + root.path(), + Mutate { + root: root.path(), + directory, + done: false + } + ) + .is_err(), + "directory={directory}" + ); + } +} + +#[test] +fn rejects_truncated_traversal_and_broken_destination() { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join("outputs")).unwrap(); + for index in 0..=MAX_ENTRIES { + fs::write(root.path().join(format!("outputs/{index}")), []).unwrap(); + } + assert!(outputs(root.path(), io::sink()).is_err()); + struct Broken; + impl Write for Broken { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::ErrorKind::BrokenPipe.into()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let empty = tempfile::tempdir().unwrap(); + assert!(outputs(empty.path(), Broken).is_err()); +} diff --git a/scripts/build-agents-executor.sh b/scripts/build-agents-executor.sh index 79d20c7c..5595b16e 100755 --- a/scripts/build-agents-executor.sh +++ b/scripts/build-agents-executor.sh @@ -35,7 +35,7 @@ cp -R "$repo_root/packages/codex-executor/src" "$build_context/src" cargo build --locked --release ) mkdir -p "$output_dir" -for binary in agents-api-codex-executor agents-api-codex-directory agents-api-codex-write; do +for binary in agents-api-codex-executor agents-api-codex-directory agents-api-codex-write agents-api-workspace-export; do cp "$CARGO_TARGET_DIR/release/$binary" "$output_dir/$binary.tmp" mv -f "$output_dir/$binary.tmp" "$output_dir/$binary" done From d87f2a74ce67f784429305658a204345ee566192 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Fri, 18 Sep 2026 12:25:30 +0800 Subject: [PATCH 2/6] Add bounded Runtime artifact capture and scoped public reads --- CONTRIBUTING.md | 8 + apps/parsar-daemon/internal/cli/connect.go | 7 +- .../internal/dispatch/preparation.go | 2 +- .../internal/dispatch/preparation_start.go | 4 + .../parsar-daemon/internal/dispatch/prompt.go | 2 +- .../parsar-daemon/internal/dispatch/router.go | 3 + .../internal/dispatch/workspace_export.go | 134 +++++++++ .../dispatch/workspace_export_test.go | 170 +++++++++++ .../internal/dispatch/workspace_write.go | 4 + .../internal/localworkspace/binding.go | 15 +- .../internal/localworkspace/export.go | 40 +++ contracts/agents-api/openapi.yaml | 282 ++++++++++++++++++ contracts/agents-api/v1/session_artifacts.go | 23 ++ internal/agentdaemon/device/state.go | 1 + internal/agentdaemon/gateway/session.go | 23 +- .../agentdaemon/gateway/workspace_export.go | 129 ++++++++ .../gateway/workspace_export_test.go | 121 ++++++++ internal/agentdaemon/proto/inbound.go | 1 + .../agentdaemon/proto/workspace_export.go | 31 ++ scripts/build-agents-runtime.sh | 4 +- services/agents-api/cmd/server/main.go | 2 +- services/agents-api/deploy/codex/Dockerfile | 3 +- services/agents-api/deploy/codex/README.md | 10 + services/agents-api/internal/api/handler.go | 5 + .../internal/api/session_artifacts.go | 141 +++++++++ .../internal/api/session_artifacts_test.go | 108 +++++++ .../internal/api/source_files_content.go | 16 +- .../internal/db/queries/session_artifacts.sql | 44 +++ .../agents-api/internal/db/sqlc/models.go | 12 + .../internal/db/sqlc/session_artifacts.sql.go | 199 ++++++++++++ .../internal/execution/artifacts.go | 36 +++ .../execution/directory_preparation.go | 27 +- .../internal/execution/prepared_dispatch.go | 1 + .../agents-api/internal/execution/support.go | 4 +- .../internal/store/artifact_capture.go | 135 +++++++++ .../internal/store/artifact_lifecycle.go | 14 + .../store/local_artifact_export_test.go | 53 ++++ .../store/local_environment_worker_test.go | 6 + .../internal/store/session_artifacts.go | 159 ++++++++++ .../internal/store/session_artifacts_test.go | 261 ++++++++++++++++ .../internal/store/session_deletion.go | 3 + .../internal/store/session_events.go | 3 + .../migrations/000040_session_artifacts.sql | 26 ++ 43 files changed, 2242 insertions(+), 30 deletions(-) create mode 100644 apps/parsar-daemon/internal/dispatch/workspace_export.go create mode 100644 apps/parsar-daemon/internal/dispatch/workspace_export_test.go create mode 100644 apps/parsar-daemon/internal/localworkspace/export.go create mode 100644 contracts/agents-api/v1/session_artifacts.go create mode 100644 internal/agentdaemon/gateway/workspace_export.go create mode 100644 internal/agentdaemon/gateway/workspace_export_test.go create mode 100644 internal/agentdaemon/proto/workspace_export.go create mode 100644 services/agents-api/internal/api/session_artifacts.go create mode 100644 services/agents-api/internal/api/session_artifacts_test.go create mode 100644 services/agents-api/internal/db/queries/session_artifacts.sql create mode 100644 services/agents-api/internal/db/sqlc/session_artifacts.sql.go create mode 100644 services/agents-api/internal/execution/artifacts.go create mode 100644 services/agents-api/internal/store/artifact_capture.go create mode 100644 services/agents-api/internal/store/artifact_lifecycle.go create mode 100644 services/agents-api/internal/store/local_artifact_export_test.go create mode 100644 services/agents-api/internal/store/session_artifacts.go create mode 100644 services/agents-api/internal/store/session_artifacts_test.go create mode 100644 services/agents-api/migrations/000040_session_artifacts.sql diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 12ec9eb5..876388d2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -432,6 +432,14 @@ syntax alone is insufficient. Never extract an output archive into Core's filesystem or hold the global execution lease through a large transfer. Keep publication ordered with Turn completion, and authorize stored reads independently of Environment availability so published outputs can survive its expiration. +Capture bytes into private PostgreSQL large objects without a Session admission +lock; after confirmed export, lock and recheck the live Turn before staging metadata. +Publish metadata in the same transaction as Turn completion. Failed/cancelled Turns +discard private objects, and Session deletion removes both private and published +copies. Reuse the source-file snapshot reader pattern and common content response; +artifact deletion does not alter workspace files. Hosted execution requires the +Runtime's bounded output-export capability and exact read-only preparation binding; +capability advertisement alone does not qualify an operator's deployment. Exporter component checks do not establish public Artifact compatibility. Local inline file delivery uses the same authenticated daemon connection and exact diff --git a/apps/parsar-daemon/internal/cli/connect.go b/apps/parsar-daemon/internal/cli/connect.go index 0b6d1bd4..39af2b5c 100644 --- a/apps/parsar-daemon/internal/cli/connect.go +++ b/apps/parsar-daemon/internal/cli/connect.go @@ -372,12 +372,17 @@ func pumpConn(parentCtx context.Context, conn *transport.Conn, registry *agent.R }() conn.StartHeartbeats(parentCtx, boot.HeartbeatInterval(), func() proto.HeartbeatPayload { + kinds := registry.SupportedAgentKinds() + for i := range kinds { + caps := &kinds[i].Capabilities + caps.WorkspaceOutputExport = local.CanExport() && caps.LocalEnvironment && caps.WorkspaceReadPreparation + } return proto.HeartbeatPayload{ Timestamp: time.Now().Unix(), ActiveRequests: router.ActiveRuns(), DaemonVersion: Version, ClaudeAvailable: agentCLIs.ClaudeCode.Available, // legacy server compatibility - SupportedAgentKinds: registry.SupportedAgentKinds(), + SupportedAgentKinds: kinds, } }, obslog.Bg().With("component", "heartbeat")) diff --git a/apps/parsar-daemon/internal/dispatch/preparation.go b/apps/parsar-daemon/internal/dispatch/preparation.go index fba8ccd5..9398c986 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation.go +++ b/apps/parsar-daemon/internal/dispatch/preparation.go @@ -73,7 +73,7 @@ func (r *Router) handleExecutionPrepare(ctx context.Context, env proto.Envelope) r.mu.Unlock() return ErrRouterClosed } - if r.workspaceWrite != nil && !req.WorkspaceReadOnly { + if (r.workspaceWrite != nil || r.workspaceExport != nil) && !req.WorkspaceReadOnly { r.mu.Unlock() return r.rejectPreparation(env, "resource_unavailable") } diff --git a/apps/parsar-daemon/internal/dispatch/preparation_start.go b/apps/parsar-daemon/internal/dispatch/preparation_start.go index e748189e..6a359676 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation_start.go +++ b/apps/parsar-daemon/internal/dispatch/preparation_start.go @@ -22,6 +22,10 @@ func (r *Router) handleExecutionStart(_ context.Context, env proto.Envelope) err r.mu.Unlock() return ErrRouterClosed } + if r.workspaceExport != nil { + r.mu.Unlock() + return r.rejectPreparation(env, "resource_unavailable") + } p := r.preparations[input.Handle] if p == nil || p.requestID != env.ID { r.mu.Unlock() diff --git a/apps/parsar-daemon/internal/dispatch/prompt.go b/apps/parsar-daemon/internal/dispatch/prompt.go index 7a74ebff..58e2d4dc 100644 --- a/apps/parsar-daemon/internal/dispatch/prompt.go +++ b/apps/parsar-daemon/internal/dispatch/prompt.go @@ -74,7 +74,7 @@ func (r *Router) handlePromptRequest(callerCtx context.Context, env proto.Envelo r.log.ErrorContext(callerCtx, "handlePromptRequest: router closed", "run_id", runID) return ErrRouterClosed } - if r.workspaceWrite != nil { + if r.workspaceWrite != nil || r.workspaceExport != nil { r.mu.Unlock() err := errors.New("local workspace has an unsettled write") r.emitTerminalError(callerCtx, runID, err.Error()) diff --git a/apps/parsar-daemon/internal/dispatch/router.go b/apps/parsar-daemon/internal/dispatch/router.go index d840b18c..62a2999e 100644 --- a/apps/parsar-daemon/internal/dispatch/router.go +++ b/apps/parsar-daemon/internal/dispatch/router.go @@ -53,6 +53,7 @@ type Router struct { preparationRequests map[string]*preparationState preparationTimeout time.Duration workspaceWrite *workspaceUpload + workspaceExport *workspaceExport workspaceReads map[string]struct{} localWorkspace *localworkspace.Binding } @@ -160,6 +161,8 @@ func (r *Router) Handle(ctx context.Context, env proto.Envelope) error { ctx = adoptEnvelopeTrace(ctx, env) switch env.Type { + case proto.TypeWorkspaceExport: + return r.handleWorkspaceExport(ctx, env) case proto.TypeWorkspaceWrite: return r.handleWorkspaceWrite(ctx, env) case proto.TypeWorkspaceRead: diff --git a/apps/parsar-daemon/internal/dispatch/workspace_export.go b/apps/parsar-daemon/internal/dispatch/workspace_export.go new file mode 100644 index 00000000..112be4db --- /dev/null +++ b/apps/parsar-daemon/internal/dispatch/workspace_export.go @@ -0,0 +1,134 @@ +package dispatch + +import ( + "context" + "errors" + "io" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +type workspaceExport struct { + id string + requests chan proto.WorkspaceExportPayload + cancel context.CancelFunc +} + +func (r *Router) handleWorkspaceExport(ctx context.Context, env proto.Envelope) error { + var request proto.WorkspaceExportPayload + if len(env.ID) == 0 || len(env.ID) > proto.WorkspaceReadMaxIDBytes || len(env.Payload) > proto.WorkspaceReadMaxRequestBytes || env.DecodePayload(&request) != nil || !proto.ValidWorkspaceExportRequest(request) { + return errors.New("dispatch: invalid workspace export request") + } + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return ErrRouterClosed + } + u := r.workspaceExport + if request.Step != "begin" { + if u == nil || u.id != env.ID { + r.mu.Unlock() + return r.sendWorkspaceExport(ctx, env.ID, proto.WorkspaceExportResultPayload{Outcome: "rejected", ErrorCode: "resource_unavailable"}) + } + if request.Step == "cancel" { + u.cancel() + r.mu.Unlock() + return nil + } + select { + case u.requests <- request: + r.mu.Unlock() + return nil + default: + u.cancel() + r.mu.Unlock() + return errors.New("dispatch: workspace export request already pending") + } + } + _, code := r.workspaceResourceLocked(proto.WorkspaceReadPayload{Handle: request.Handle, EnvironmentID: request.EnvironmentID}) + p := r.preparations[request.Handle] + if u != nil || r.workspaceWrite != nil || !r.localWorkspace.CanExport() || code != "" || p == nil || !p.workspaceReadOnly { + r.mu.Unlock() + return r.sendWorkspaceExport(ctx, env.ID, proto.WorkspaceExportResultPayload{Outcome: "rejected", ErrorCode: "resource_unavailable"}) + } + owner, stop := r.shutdownContext(p.ctx) + owner, cancel := context.WithTimeout(owner, 180*time.Second) + u = &workspaceExport{id: env.ID, requests: make(chan proto.WorkspaceExportPayload, 1), cancel: func() { cancel(); stop() }} + u.requests <- request + r.workspaceExport = u + r.shutdownWG.Add(1) + r.mu.Unlock() + go r.runWorkspaceExport(owner, u) + return nil +} + +func (r *Router) runWorkspaceExport(ctx context.Context, u *workspaceExport) { + defer r.shutdownWG.Done() + reader, writer := io.Pipe() + done := make(chan struct{}) + go func() { + defer close(done) + err := r.localWorkspace.ExportOutputs(ctx, writer) + _ = writer.CloseWithError(err) + }() + defer func() { + u.cancel() + _ = reader.Close() + <-done + r.mu.Lock() + if r.workspaceExport == u { + r.workspaceExport = nil + } + r.mu.Unlock() + }() + // Closing the reader unblocks a pending pipe read on cancellation or shutdown. + stopRead := context.AfterFunc(ctx, func() { _ = reader.CloseWithError(ctx.Err()) }) + defer stopRead() + var offset int64 + buffer := make([]byte, proto.WorkspaceExportChunkBytes) + for { + select { + case request := <-u.requests: + if request.Offset != offset { + _ = r.sendWorkspaceExport(ctx, u.id, proto.WorkspaceExportResultPayload{Outcome: "failed", Offset: offset, ErrorCode: "invalid_request"}) + return + } + case <-ctx.Done(): + return + } + n, err := reader.Read(buffer) + result := proto.WorkspaceExportResultPayload{Offset: offset} + switch { + case err == io.EOF: + result.Outcome = "completed" + case err != nil || n == 0 || int64(n) > proto.WorkspaceExportMaxBytes-offset: + result.Outcome, result.ErrorCode = "failed", "export_failed" + default: + result.Outcome, result.Data = "chunk", buffer[:n] + offset += int64(n) + } + if result.Outcome == "completed" { + // The next owner may start immediately after receiving completion. + <-done + r.mu.Lock() + if r.workspaceExport == u { + r.workspaceExport = nil + } + r.mu.Unlock() + } + if r.sendWorkspaceExport(ctx, u.id, result) != nil || result.Outcome != "chunk" { + return + } + } +} + +func (r *Router) sendWorkspaceExport(ctx context.Context, id string, result proto.WorkspaceExportResultPayload) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + env, err := proto.NewEnvelope(proto.TypeWorkspaceExportResult, id, result) + if err != nil { + return err + } + return r.sender.Send(ctx, env) +} diff --git a/apps/parsar-daemon/internal/dispatch/workspace_export_test.go b/apps/parsar-daemon/internal/dispatch/workspace_export_test.go new file mode 100644 index 00000000..490130f5 --- /dev/null +++ b/apps/parsar-daemon/internal/dispatch/workspace_export_test.go @@ -0,0 +1,170 @@ +package dispatch + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/localworkspace" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/google/uuid" +) + +type exportSender struct{ replies chan proto.Envelope } + +func (s exportSender) Send(ctx context.Context, env proto.Envelope) error { + select { + case s.replies <- env: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func exporterRouter(t *testing.T, program string) (*Router, exportSender, proto.WorkspaceExportPayload) { + t.Helper() + workspace, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + helper := filepath.Join(dir, "export") + if err := os.WriteFile(helper, []byte("#!/bin/sh\n"+program+"\n"), 0700); err != nil { + t.Fatal(err) + } + environment, session := uuid.NewString(), uuid.NewString() + for key, value := range map[string]string{"PARSAR_RUNTIME_ENVIRONMENT_ID": environment, "PARSAR_RUNTIME_SESSION_ID": session, "PARSAR_RUNTIME_WORKSPACE": workspace, "PARSAR_RUNTIME_DIRECTORY_HELPER": helper, "PARSAR_RUNTIME_EXPORT_HELPER": helper, "PARSAR_RUNTIME_WRITE_HELPER": "", "PARSAR_RUNTIME_STAGING": "", "PARSAR_RUNTIME_NETWORK_ACCESS": ""} { + t.Setenv(key, value) + } + binding, err := localworkspace.Load() + if err != nil { + t.Fatal(err) + } + sender := exportSender{make(chan proto.Envelope, 8)} + r, err := New(Config{Registry: agent.NewRegistry(), Sender: sender, LocalWorkspace: binding}) + if err != nil { + t.Fatal(err) + } + handle := uuid.NewString() + r.preparations[handle] = &preparationState{workspaceReadOnly: true, environmentID: environment, owns: true, ctx: context.Background(), deadline: time.Now().Add(time.Hour), status: proto.PreparationStatusPayload{State: "ready"}} + t.Cleanup(func() { + r.mu.Lock() + delete(r.preparations, handle) + r.mu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := r.Shutdown(ctx); err != nil { + t.Error(err) + } + }) + return r, sender, proto.WorkspaceExportPayload{Step: "begin", Handle: handle, EnvironmentID: environment} +} + +func sendExport(t *testing.T, r *Router, id string, p proto.WorkspaceExportPayload) { + t.Helper() + env, err := proto.NewEnvelope(proto.TypeWorkspaceExport, id, p) + if err != nil { + t.Fatal(err) + } + if err := r.Handle(t.Context(), env); err != nil { + t.Fatal(err) + } +} +func readExport(t *testing.T, s exportSender) proto.WorkspaceExportResultPayload { + t.Helper() + select { + case env := <-s.replies: + var p proto.WorkspaceExportResultPayload + if env.DecodePayload(&p) != nil { + t.Fatal("invalid reply") + } + return p + case <-time.After(3 * time.Second): + t.Fatal("missing export reply") + } + return proto.WorkspaceExportResultPayload{} +} + +func TestWorkspaceExportUsesExactReadPreparationAndPullsBoundedBytes(t *testing.T) { + r, s, request := exporterRouter(t, "head -c 131089 /dev/zero") + for _, field := range []string{"environment", "handle"} { + bad := request + if field == "environment" { + bad.EnvironmentID = uuid.NewString() + } else { + bad.Handle = uuid.NewString() + } + sendExport(t, r, uuid.NewString(), bad) + if result := readExport(t, s); result.Outcome != "rejected" { + t.Fatal("foreign authority accepted") + } + } + id := uuid.NewString() + sendExport(t, r, id, request) + var offset int64 + for { + p := readExport(t, s) + if p.Offset != offset { + t.Fatal("wrong offset") + } + if p.Outcome == "completed" { + break + } + if p.Outcome != "chunk" || len(p.Data) == 0 || len(p.Data) > proto.WorkspaceExportChunkBytes { + t.Fatal("invalid chunk") + } + for _, b := range p.Data { + if b != 0 { + t.Fatal("wrong byte") + } + } + offset += int64(len(p.Data)) + select { + case <-s.replies: + t.Fatal("export pushed unrequested data") + default: + } + sendExport(t, r, id, proto.WorkspaceExportPayload{Step: "next", Offset: offset}) + } + if offset != 131089 { + t.Fatal("truncated export", offset) + } +} + +func TestWorkspaceExportFailureAfterBytesCannotComplete(t *testing.T) { + r, s, request := exporterRouter(t, "printf abc; exit 1") + id := uuid.NewString() + sendExport(t, r, id, request) + p := readExport(t, s) + if p.Outcome != "chunk" || string(p.Data) != "abc" { + t.Fatal("missing prefix", p) + } + sendExport(t, r, id, proto.WorkspaceExportPayload{Step: "next", Offset: 3}) + if p := readExport(t, s); p.Outcome != "failed" { + t.Fatal("failed process appeared complete", p) + } +} + +func TestWorkspaceExportCancelUnblocksProcessAndReleasesCapacity(t *testing.T) { + r, _, request := exporterRouter(t, "exec sleep 30") + id := uuid.NewString() + sendExport(t, r, id, request) + sendExport(t, r, id, proto.WorkspaceExportPayload{Step: "cancel"}) + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + r.mu.Lock() + active := r.workspaceExport != nil + r.mu.Unlock() + if !active { + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("export cancellation did not release capacity") +} diff --git a/apps/parsar-daemon/internal/dispatch/workspace_write.go b/apps/parsar-daemon/internal/dispatch/workspace_write.go index 893d79cf..338e3bdd 100644 --- a/apps/parsar-daemon/internal/dispatch/workspace_write.go +++ b/apps/parsar-daemon/internal/dispatch/workspace_write.go @@ -46,6 +46,10 @@ func (r *Router) handleWorkspaceWrite(ctx context.Context, env proto.Envelope) e return ErrRouterClosed } if request.Step == "begin" { + if r.workspaceExport != nil { + r.mu.Unlock() + return r.sendWorkspaceWrite(ctx, env.ID, rejectedWorkspaceWrite("resource_unavailable")) + } if r.workspaceWrite != nil { duplicate := r.workspaceWrite.envelope.ID == env.ID r.mu.Unlock() diff --git a/apps/parsar-daemon/internal/localworkspace/binding.go b/apps/parsar-daemon/internal/localworkspace/binding.go index 43c5338f..938c0c34 100644 --- a/apps/parsar-daemon/internal/localworkspace/binding.go +++ b/apps/parsar-daemon/internal/localworkspace/binding.go @@ -17,6 +17,7 @@ type Binding struct { stateKey string workspace string helper string + exportHelper string writer *fileWriter } @@ -50,7 +51,8 @@ func Load() (*Binding, error) { return nil, errors.New("unsupported local Runtime network policy") } writeHelper, staging := os.Getenv("PARSAR_RUNTIME_WRITE_HELPER"), os.Getenv("PARSAR_RUNTIME_STAGING") - if strings.Join(values, "") == "" && writeHelper == "" && staging == "" && network == "" { + exportHelper := os.Getenv("PARSAR_RUNTIME_EXPORT_HELPER") + if strings.Join(values, "") == "" && writeHelper == "" && staging == "" && network == "" && exportHelper == "" { return nil, nil } b, err := New(values[0], values[1], values[2], values[3]) @@ -58,6 +60,17 @@ func Load() (*Binding, error) { return nil, err } b.networkAccess = network + if exportHelper != "" { + // Reuse the startup executable/root checks; this grants no caller authority. + if _, err := New(values[0], values[1], values[2], exportHelper); err != nil { + return nil, err + } + resolved, err := filepath.EvalSymlinks(exportHelper) + if err != nil || resolved != exportHelper { + return nil, errors.New("workspace exporter must be a canonical executable") + } + b.exportHelper = exportHelper + } if writeHelper != "" || staging != "" { if err := b.bindWriter(writeHelper, staging); err != nil { return nil, err diff --git a/apps/parsar-daemon/internal/localworkspace/export.go b/apps/parsar-daemon/internal/localworkspace/export.go new file mode 100644 index 00000000..b94bcbd3 --- /dev/null +++ b/apps/parsar-daemon/internal/localworkspace/export.go @@ -0,0 +1,40 @@ +package localworkspace + +import ( + "context" + "errors" + "io" + "os/exec" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +func (b *Binding) CanExport() bool { return b != nil && b.exportHelper != "" } + +// ExportOutputs streams only from the deployment-owned root; successful exit is mandatory. +func (b *Binding) ExportOutputs(ctx context.Context, output io.Writer) error { + if !b.CanExport() || output == nil { + return errors.New("workspace export unavailable") + } + cmd := exec.CommandContext(ctx, b.exportHelper, b.workspace) + cmd.Dir = "/" + cmd.Env = []string{"PATH=/usr/bin:/bin", "LANG=C.UTF-8"} + cmd.Stdout = &exportWriter{output: output} + cmd.WaitDelay = time.Second + return cmd.Run() +} + +type exportWriter struct { + output io.Writer + size int64 +} + +func (w *exportWriter) Write(data []byte) (int, error) { + if int64(len(data)) > proto.WorkspaceExportMaxBytes-w.size { + return 0, errors.New("workspace export exceeds bound") + } + n, err := w.output.Write(data) + w.size += int64(n) + return n, err +} diff --git a/contracts/agents-api/openapi.yaml b/contracts/agents-api/openapi.yaml index 4050f76e..39714f7a 100644 --- a/contracts/agents-api/openapi.yaml +++ b/contracts/agents-api/openapi.yaml @@ -778,6 +778,63 @@ definitions: - status - vault_ids type: object + v1.SessionArtifact: + properties: + created_at: + type: integer + environment_id: + type: string + id: + type: string + object: + enum: + - agent.session.artifact + type: string + path: + type: string + session_id: + type: string + size_bytes: + type: integer + turn_id: + type: string + required: + - created_at + - environment_id + - id + - object + - path + - session_id + - size_bytes + - turn_id + type: object + v1.SessionArtifactDeleted: + properties: + deleted: + type: boolean + id: + type: string + object: + enum: + - agent.session.artifact.deleted + type: string + required: + - deleted + - id + - object + type: object + v1.SessionArtifactList: + properties: + data: + items: + $ref: '#/definitions/v1.SessionArtifact' + type: array + has_more: + type: boolean + required: + - data + - has_more + type: object v1.SessionDeleted: properties: deleted: @@ -2066,6 +2123,231 @@ paths: summary: Update execution Session metadata tags: - Sessions + /agents/sessions/{session_id}/artifacts: + get: + description: Lists published outputs independently of Environment availability. + Sorting uses publication time and ID. The local default page size is 20; exact + upstream defaults and error parity remain unverified. + parameters: + - description: agents=v1 + in: header + name: OpenAI-Beta + required: true + type: string + - description: Session ID + in: path + name: session_id + required: true + type: string + - description: Producing Environment ID + in: query + name: environment_id + type: string + - description: Last immutable artifact ID + in: query + name: after + type: string + - default: 20 + description: Page size + in: query + maximum: 100 + minimum: 1 + name: limit + type: integer + - default: desc + description: Publication order + enum: + - asc + - desc + in: query + name: order + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/v1.SessionArtifactList' + "400": + description: Bad Request + schema: + $ref: '#/definitions/v1.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/v1.ErrorResponse' + "404": + description: Not Found + schema: + $ref: '#/definitions/v1.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/v1.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/v1.ErrorResponse' + security: + - BearerAuth: [] + summary: List immutable Session artifacts + tags: + - Artifacts + /agents/sessions/{session_id}/artifacts/{artifact_id}: + delete: + description: Deletes the published copy without modifying its original workspace + file. Already admitted content reads may finish; later reads reject. + parameters: + - description: agents=v1 + in: header + name: OpenAI-Beta + required: true + type: string + - description: Session ID + in: path + name: session_id + required: true + type: string + - description: Artifact ID + in: path + name: artifact_id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/v1.SessionArtifactDeleted' + "400": + description: Bad Request + schema: + $ref: '#/definitions/v1.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/v1.ErrorResponse' + "404": + description: Not Found + schema: + $ref: '#/definitions/v1.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/v1.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/v1.ErrorResponse' + security: + - BearerAuth: [] + summary: Delete a published artifact + tags: + - Artifacts + get: + parameters: + - description: agents=v1 + in: header + name: OpenAI-Beta + required: true + type: string + - description: Session ID + in: path + name: session_id + required: true + type: string + - description: Artifact ID + in: path + name: artifact_id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/v1.SessionArtifact' + "400": + description: Bad Request + schema: + $ref: '#/definitions/v1.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/v1.ErrorResponse' + "404": + description: Not Found + schema: + $ref: '#/definitions/v1.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/v1.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/v1.ErrorResponse' + security: + - BearerAuth: [] + summary: Retrieve immutable artifact metadata + tags: + - Artifacts + /agents/sessions/{session_id}/artifacts/{artifact_id}/content: + get: + description: Streams stored bytes after tenant and Session authorization, including + after Environment expiration. Exact upstream headers and Range behavior remain + unverified. + parameters: + - description: agents=v1 + in: header + name: OpenAI-Beta + required: true + type: string + - description: Session ID + in: path + name: session_id + required: true + type: string + - description: Artifact ID + in: path + name: artifact_id + required: true + type: string + produces: + - application/octet-stream + responses: + "200": + description: OK + schema: + type: file + "400": + description: Bad Request + schema: + $ref: '#/definitions/v1.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/v1.ErrorResponse' + "404": + description: Not Found + schema: + $ref: '#/definitions/v1.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/v1.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/v1.ErrorResponse' + security: + - BearerAuth: [] + summary: Download immutable artifact bytes + tags: + - Artifacts /agents/sessions/{session_id}/events: get: description: Live-only events, including command output fragments from capable diff --git a/contracts/agents-api/v1/session_artifacts.go b/contracts/agents-api/v1/session_artifacts.go new file mode 100644 index 00000000..78158a95 --- /dev/null +++ b/contracts/agents-api/v1/session_artifacts.go @@ -0,0 +1,23 @@ +package v1 + +type SessionArtifact struct { + ID string `json:"id" binding:"required"` + CreatedAt int64 `json:"created_at" binding:"required"` + EnvironmentID string `json:"environment_id" binding:"required"` + Object string `json:"object" enums:"agent.session.artifact" binding:"required"` + Path string `json:"path" binding:"required"` + SessionID string `json:"session_id" binding:"required"` + SizeBytes int64 `json:"size_bytes" binding:"required"` + TurnID string `json:"turn_id" binding:"required"` +} + +type SessionArtifactList struct { + Data []SessionArtifact `json:"data" binding:"required"` + HasMore bool `json:"has_more" binding:"required"` +} + +type SessionArtifactDeleted struct { + ID string `json:"id" binding:"required"` + Object string `json:"object" enums:"agent.session.artifact.deleted" binding:"required"` + Deleted bool `json:"deleted" binding:"required"` +} diff --git a/internal/agentdaemon/device/state.go b/internal/agentdaemon/device/state.go index f5774d55..1d8ea629 100644 --- a/internal/agentdaemon/device/state.go +++ b/internal/agentdaemon/device/state.go @@ -74,6 +74,7 @@ type KindCapabilities struct { LocalEnvironmentNetworkPolicy bool `json:"local_environment_network_policy,omitempty"` Preparation bool `json:"preparation,omitempty"` WorkspaceReadPreparation bool `json:"workspace_read_preparation,omitempty"` + WorkspaceOutputExport bool `json:"workspace_output_export,omitempty"` WebSearchControl bool `json:"web_search_control,omitempty"` // ExecutionControls supports typed search and verbosity controls. ExecutionControls bool `json:"execution_controls,omitempty"` diff --git a/internal/agentdaemon/gateway/session.go b/internal/agentdaemon/gateway/session.go index a70cabd4..b7dabd9c 100644 --- a/internal/agentdaemon/gateway/session.go +++ b/internal/agentdaemon/gateway/session.go @@ -100,14 +100,16 @@ type Session struct { // Subscribers keyed by runID. The read loop only sends on these // channels; Unsubscribe is the only place that closes them. - subsMu sync.Mutex - subs map[string]*Subscription - preparationMu sync.Mutex - preparations map[string]*preparationSubscription - workspaceWriteMu sync.Mutex - workspaceWrites map[string]chan proto.Envelope - workspaceReadMu sync.Mutex - workspaceReads map[string]chan proto.Envelope + subsMu sync.Mutex + subs map[string]*Subscription + preparationMu sync.Mutex + preparations map[string]*preparationSubscription + workspaceWriteMu sync.Mutex + workspaceWrites map[string]chan proto.Envelope + workspaceReadMu sync.Mutex + workspaceReads map[string]chan proto.Envelope + workspaceExportMu sync.Mutex + workspaceExports map[string]chan proto.Envelope ackMu sync.Mutex ackWaiters map[string]chan proto.InteractionDecisionAckPayload @@ -259,6 +261,7 @@ func (s *Session) Close(reason string) { s.closePreparations() s.closeWorkspaceReads() s.closeWorkspaceWrites() + s.closeWorkspaceExports() s.markOfflineOnClose() s.releaseOwnerLease() }) @@ -551,6 +554,7 @@ func deviceKindsFromHeartbeat(p proto.HeartbeatPayload) []device.SupportedAgentK LocalEnvironmentNetworkPolicy: info.Capabilities.LocalEnvironmentNetworkPolicy, Preparation: info.Capabilities.Preparation, WorkspaceReadPreparation: info.Capabilities.WorkspaceReadPreparation, + WorkspaceOutputExport: info.Capabilities.WorkspaceOutputExport, WebSearchControl: info.Capabilities.WebSearchControl, TextVerbosity: info.Capabilities.TextVerbosity, ExecutionControls: info.Capabilities.ExecutionControls, @@ -570,6 +574,9 @@ func deviceKindsFromHeartbeat(p proto.HeartbeatPayload) []device.SupportedAgentK func (s *Session) dispatch(env proto.Envelope) { switch env.Type { + case proto.TypeWorkspaceExportResult: + s.dispatchWorkspaceExport(env) + return case proto.TypeWorkspaceWriteResult: s.dispatchWorkspaceWrite(env) case proto.TypeWorkspaceReadResult: diff --git a/internal/agentdaemon/gateway/workspace_export.go b/internal/agentdaemon/gateway/workspace_export.go new file mode 100644 index 00000000..8f6e5d63 --- /dev/null +++ b/internal/agentdaemon/gateway/workspace_export.go @@ -0,0 +1,129 @@ +package gateway + +import ( + "context" + "errors" + "io" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/google/uuid" +) + +// ExportWorkspaceOutputs consumes bounded chunks and requires the exporter completion receipt. +func (s *Session) ExportWorkspaceOutputs(ctx context.Context, request proto.WorkspaceExportPayload, consume func(io.Reader) error) error { + request.Step = "begin" + if !proto.ValidWorkspaceExportRequest(request) || consume == nil { + return errors.New("agentdaemon gateway: invalid workspace export") + } + id := uuid.NewString() + s.workspaceExportMu.Lock() + if s.IsClosed() { + s.workspaceExportMu.Unlock() + return ErrSessionClosed + } + if len(s.workspaceExports) != 0 { + s.workspaceExportMu.Unlock() + return errors.New("agentdaemon gateway: workspace export capacity") + } + replies := make(chan proto.Envelope, 1) + s.workspaceExports = map[string]chan proto.Envelope{id: replies} + s.workspaceExportMu.Unlock() + defer func() { s.workspaceExportMu.Lock(); delete(s.workspaceExports, id); s.workspaceExportMu.Unlock() }() + ctx, cancel := context.WithTimeout(ctx, 180*time.Second) + defer cancel() + r := &workspaceExportReader{ctx: ctx, peer: s, id: id, replies: replies, request: request} + defer func() { + if !r.completed { + stop, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + env, _ := proto.NewEnvelope(proto.TypeWorkspaceExport, id, proto.WorkspaceExportPayload{Step: "cancel"}) + _ = s.Send(stop, env) + } + }() + if err := consume(r); err != nil { + return err + } + // Archive decoders may stop at their trailer before the native exit receipt. + _, err := io.Copy(io.Discard, r) + return err +} + +type workspaceExportReader struct { + ctx context.Context + peer *Session + id string + replies <-chan proto.Envelope + request proto.WorkspaceExportPayload + data []byte + offset int64 + completed bool +} + +func (r *workspaceExportReader) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if len(r.data) > 0 { + n := copy(p, r.data) + r.data = r.data[n:] + return n, nil + } + if r.completed { + return 0, io.EOF + } + env, err := proto.NewEnvelope(proto.TypeWorkspaceExport, r.id, r.request) + if err != nil { + return 0, err + } + if err = r.peer.Send(r.ctx, env); err != nil { + return 0, err + } + select { + case env, ok := <-r.replies: + if !ok { + return 0, ErrSessionClosed + } + var result proto.WorkspaceExportResultPayload + if len(env.Payload) > proto.WorkspaceExportMaxFrameBytes || env.DecodePayload(&result) != nil || result.Offset != r.offset { + return 0, errors.New("agentdaemon gateway: invalid export receipt") + } + if result.Outcome == "completed" && len(result.Data) == 0 && result.ErrorCode == "" { + r.completed = true + return 0, io.EOF + } + if result.Outcome != "chunk" || result.ErrorCode != "" || len(result.Data) == 0 || len(result.Data) > proto.WorkspaceExportChunkBytes || int64(len(result.Data)) > proto.WorkspaceExportMaxBytes-r.offset { + return 0, errors.New("agentdaemon gateway: workspace export incomplete") + } + r.data = result.Data + r.offset += int64(len(result.Data)) + r.request = proto.WorkspaceExportPayload{Step: "next", Offset: r.offset} + return r.Read(p) + case <-r.ctx.Done(): + return 0, r.ctx.Err() + case <-r.peer.closed: + return 0, ErrSessionClosed + } +} + +func (s *Session) dispatchWorkspaceExport(env proto.Envelope) { + s.workspaceExportMu.Lock() + defer s.workspaceExportMu.Unlock() + if replies := s.workspaceExports[env.ID]; replies != nil { + select { + case replies <- env: + default: + close(replies) + delete(s.workspaceExports, env.ID) + } + } +} + +func (s *Session) closeWorkspaceExports() { + s.workspaceExportMu.Lock() + defer s.workspaceExportMu.Unlock() + for id, replies := range s.workspaceExports { + close(replies) + delete(s.workspaceExports, id) + } +} diff --git a/internal/agentdaemon/gateway/workspace_export_test.go b/internal/agentdaemon/gateway/workspace_export_test.go new file mode 100644 index 00000000..73a28957 --- /dev/null +++ b/internal/agentdaemon/gateway/workspace_export_test.go @@ -0,0 +1,121 @@ +package gateway + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +func exportReply(t *testing.T, s *Session, id string, result proto.WorkspaceExportResultPayload) { + t.Helper() + env, err := proto.NewEnvelope(proto.TypeWorkspaceExportResult, id, result) + if err != nil { + t.Fatal(err) + } + s.dispatch(env) +} + +func TestWorkspaceExportPullsOnlyAfterConsumedAndRequiresCompletion(t *testing.T) { + s := NewSession(newFakeConn(), "device", "tenant", "test", nil, nil) + defer s.Close("test") + consumed, resume := make(chan struct{}), make(chan struct{}) + done := make(chan error, 1) + body := bytes.Repeat([]byte{0, 255, 7}, 100) + go func() { + done <- s.ExportWorkspaceOutputs(t.Context(), proto.WorkspaceExportPayload{Handle: "prepared", EnvironmentID: "env"}, func(r io.Reader) error { + got := make([]byte, len(body)) + if _, err := io.ReadFull(r, got); err != nil { + return err + } + if !bytes.Equal(got, body) { + return errors.New("bytes differ") + } + close(consumed) + <-resume + return nil + }) + }() + first := <-s.sendCh + exportReply(t, s, "foreign", proto.WorkspaceExportResultPayload{Outcome: "completed"}) + exportReply(t, s, first.ID, proto.WorkspaceExportResultPayload{Outcome: "chunk", Data: body}) + <-consumed + select { + case env := <-s.sendCh: + t.Fatalf("premature next chunk: %s", env.Type) + default: + } + close(resume) + next := <-s.sendCh + var request proto.WorkspaceExportPayload + if next.DecodePayload(&request) != nil || request.Step != "next" || request.Offset != int64(len(body)) { + t.Fatal("bad continuation", request) + } + select { + case err := <-done: + t.Fatal("succeeded before native completion", err) + default: + } + exportReply(t, s, next.ID, proto.WorkspaceExportResultPayload{Outcome: "completed", Offset: int64(len(body))}) + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestWorkspaceExportRejectsInvalidChunksAndNativeFailure(t *testing.T) { + for _, result := range []proto.WorkspaceExportResultPayload{ + {Outcome: "chunk", Offset: 1, Data: []byte("x")}, + {Outcome: "chunk", Data: make([]byte, proto.WorkspaceExportChunkBytes+1)}, + {Outcome: "chunk"}, + {Outcome: "completed", Data: []byte("x")}, + {Outcome: "failed", ErrorCode: "export_failed"}, + } { + s := NewSession(newFakeConn(), "device", "tenant", "test", nil, nil) + done := make(chan error, 1) + go func() { + done <- s.ExportWorkspaceOutputs(t.Context(), proto.WorkspaceExportPayload{Handle: "prepared", EnvironmentID: "env"}, func(r io.Reader) error { _, err := io.Copy(io.Discard, r); return err }) + }() + first := <-s.sendCh + exportReply(t, s, first.ID, result) + if err := <-done; err == nil { + t.Fatal("invalid export accepted", result.Outcome) + } + cancel := <-s.sendCh + var request proto.WorkspaceExportPayload + if cancel.DecodePayload(&request) != nil || request.Step != "cancel" { + t.Fatal("missing cleanup request") + } + s.Close("test") + } +} + +func TestWorkspaceExportDisconnectAndCancellationSettleRead(t *testing.T) { + for _, disconnect := range []bool{false, true} { + s := NewSession(newFakeConn(), "device", "tenant", "test", nil, nil) + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { + done <- s.ExportWorkspaceOutputs(ctx, proto.WorkspaceExportPayload{Handle: "prepared", EnvironmentID: "env"}, func(r io.Reader) error { _, err := io.Copy(io.Discard, r); return err }) + }() + <-s.sendCh + if disconnect { + s.Close("lost") + } else { + cancel() + } + select { + case err := <-done: + if err == nil { + t.Fatal("interrupted export succeeded") + } + case <-time.After(time.Second): + t.Fatal("read did not settle") + } + cancel() + s.Close("test") + } +} diff --git a/internal/agentdaemon/proto/inbound.go b/internal/agentdaemon/proto/inbound.go index 7f162b90..f4aa965b 100644 --- a/internal/agentdaemon/proto/inbound.go +++ b/internal/agentdaemon/proto/inbound.go @@ -264,6 +264,7 @@ type AgentKindCapabilities struct { LocalEnvironmentNetworkPolicy bool `json:"local_environment_network_policy,omitempty"` Preparation bool `json:"preparation,omitempty"` WorkspaceReadPreparation bool `json:"workspace_read_preparation,omitempty"` + WorkspaceOutputExport bool `json:"workspace_output_export,omitempty"` WebSearchControl bool `json:"web_search_control,omitempty"` // ExecutionControls supports typed search and verbosity controls. ExecutionControls bool `json:"execution_controls,omitempty"` diff --git a/internal/agentdaemon/proto/workspace_export.go b/internal/agentdaemon/proto/workspace_export.go new file mode 100644 index 00000000..7b8d1f7a --- /dev/null +++ b/internal/agentdaemon/proto/workspace_export.go @@ -0,0 +1,31 @@ +package proto + +const ( + TypeWorkspaceExport = "workspace_export" + TypeWorkspaceExportResult = "workspace_export_result" + WorkspaceExportChunkBytes = 64 << 10 + WorkspaceExportMaxFrameBytes = 96 << 10 + // The archive allowance includes bounded headers and padding above 500 MiB of files. + WorkspaceExportMaxBytes int64 = 528 << 20 +) + +type WorkspaceExportPayload struct { + Step string `json:"step"` + Handle string `json:"handle,omitempty"` + EnvironmentID string `json:"environment_id,omitempty"` + Offset int64 `json:"offset,omitempty"` +} + +type WorkspaceExportResultPayload struct { + Outcome string `json:"outcome"` + Offset int64 `json:"offset"` + Data []byte `json:"data,omitempty"` + ErrorCode string `json:"error_code,omitempty"` +} + +func ValidWorkspaceExportRequest(p WorkspaceExportPayload) bool { + if p.Step == "begin" { + return p.Offset == 0 && len(p.Handle) > 0 && len(p.Handle) <= 128 && len(p.EnvironmentID) > 0 && len(p.EnvironmentID) <= 128 + } + return p.Handle == "" && p.EnvironmentID == "" && ((p.Step == "next" && p.Offset >= 0 && p.Offset <= WorkspaceExportMaxBytes) || (p.Step == "cancel" && p.Offset == 0)) +} diff --git a/scripts/build-agents-runtime.sh b/scripts/build-agents-runtime.sh index 00b38647..c7223d13 100755 --- a/scripts/build-agents-runtime.sh +++ b/scripts/build-agents-runtime.sh @@ -19,7 +19,7 @@ package = json.load(open(sys.argv[1])) assert package['name'] == '@openai/codex' and package['version'] == '0.153.4-linux-x64', 'Expected pinned official Linux x64 package' PY native_dir="$package_dir/vendor/x86_64-unknown-linux-musl" -for executable in "$native_dir/bin/codex" "$helpers_dir/agents-api-codex-directory" "$helpers_dir/agents-api-codex-write"; do +for executable in "$native_dir/bin/codex" "$helpers_dir/agents-api-codex-directory" "$helpers_dir/agents-api-codex-write" "$helpers_dir/agents-api-workspace-export"; do test -x "$executable" || { printf 'Missing executable: %s\n' "$executable" >&2; exit 1; } done test -d "$native_dir/codex-resources" @@ -31,7 +31,7 @@ trap 'rm -rf "$context"' EXIT CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -mod=readonly -trimpath \ -o "$context/parsar-daemon" ./apps/parsar-daemon/cmd/parsar-daemon ) -cp "$helpers_dir/agents-api-codex-directory" "$helpers_dir/agents-api-codex-write" "$context/" +cp "$helpers_dir/agents-api-codex-directory" "$helpers_dir/agents-api-codex-write" "$helpers_dir/agents-api-workspace-export" "$context/" cp "$native_dir/bin/codex" "$context/codex" cp -R "$native_dir/codex-resources" "$context/codex-resources" cp "$repo_root/services/agents-api/deploy/codex/requirements.toml" "$context/requirements.toml" diff --git a/services/agents-api/cmd/server/main.go b/services/agents-api/cmd/server/main.go index 5f156ea0..80c0f100 100644 --- a/services/agents-api/cmd/server/main.go +++ b/services/agents-api/cmd/server/main.go @@ -90,7 +90,7 @@ func run() error { } var workerDone chan error var worker *execution.Worker - options := []api.Option{api.WithSourceFiles(executionStore)} + options := []api.Option{api.WithSourceFiles(executionStore), api.WithSessionArtifacts(executionStore)} var daemonHandler http.Handler var registry *gateway.Registry var checkOwnership func(context.Context) error diff --git a/services/agents-api/deploy/codex/Dockerfile b/services/agents-api/deploy/codex/Dockerfile index b4b63902..3c0d2242 100644 --- a/services/agents-api/deploy/codex/Dockerfile +++ b/services/agents-api/deploy/codex/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && mkdir -p /environment/workspace /workspace /home/runtime \ && groupadd --gid 1000 runtime \ && useradd --uid 1000 --gid 1000 --home-dir /home/runtime --no-create-home runtime -COPY --chmod=0555 parsar-daemon agents-api-codex-directory agents-api-codex-write codex /usr/local/bin/ +COPY --chmod=0555 parsar-daemon agents-api-codex-directory agents-api-codex-write agents-api-workspace-export codex /usr/local/bin/ COPY codex-resources /usr/local/codex-resources COPY requirements.toml /etc/codex/requirements.toml RUN chmod 0555 /etc/codex && chmod 0444 /etc/codex/requirements.toml @@ -16,6 +16,7 @@ ENV HOME=/home/runtime PARSAR_HOME=/home/runtime/.parsar \ PARSAR_RUNTIME_WORKSPACE=/environment/workspace \ PARSAR_RUNTIME_DIRECTORY_HELPER=/usr/local/bin/agents-api-codex-directory \ PARSAR_RUNTIME_WRITE_HELPER=/usr/local/bin/agents-api-codex-write \ + PARSAR_RUNTIME_EXPORT_HELPER=/usr/local/bin/agents-api-workspace-export \ PARSAR_RUNTIME_STAGING=/environment/staging USER 1000:1000 RUN test -r /etc/codex/requirements.toml && test "$(codex --version)" = "codex-cli 0.153.4" diff --git a/services/agents-api/deploy/codex/README.md b/services/agents-api/deploy/codex/README.md index 4c4c8f58..935e3d03 100644 --- a/services/agents-api/deploy/codex/README.md +++ b/services/agents-api/deploy/codex/README.md @@ -81,6 +81,16 @@ Workspace and staging must share the same mount for atomic rename. Do not mount them separately or put daemon/model credentials, native history or other tenants inside `/environment`. The read-only Runtime can omit both writer settings. +Hosted Turn output publication additionally requires the immutable +`agents-api-workspace-export` executable selected by +`PARSAR_RUNTIME_EXPORT_HELPER`. The Runtime bundle includes it outside the workspace. +It exports regular files below `outputs` through the authenticated daemon connection; +Core stores immutable copies and publishes them with successful Turn completion. +Use a matched Core/Runtime release: older Runtime images without bounded output +export are ineligible for hosted execution. Listing and downloading already +published artifacts uses the execution database and requires no running Environment. +The export capability does not replace the deployment isolation checks above. + The installer runs as a trusted bounded daemon child with a minimal environment. Native tools retain their narrower filesystem policy. Qualify direct reads, symlink and process-root aliases, attempted staging modification, real uploaded diff --git a/services/agents-api/internal/api/handler.go b/services/agents-api/internal/api/handler.go index 13b13157..20d72849 100644 --- a/services/agents-api/internal/api/handler.go +++ b/services/agents-api/internal/api/handler.go @@ -42,6 +42,7 @@ type Handler struct { directoryReader EnvironmentDirectoryReader fileWriter EnvironmentFileWriter sourceFiles SourceFileStore + artifacts SessionArtifactStore } func NewHandler(s ResourceStore, auth *Authenticator, engine string, options ...Option) (http.Handler, error) { @@ -93,6 +94,10 @@ func NewHandler(s ResourceStore, auth *Authenticator, engine string, options ... r.Get("/agents/sessions/{session_id}/items", h.listItems) r.Get("/agents/sessions/{session_id}/turns", h.listTurns) r.Get("/agents/sessions/{session_id}/turns/{turn_id}", h.getTurn) + r.Get("/agents/sessions/{session_id}/artifacts", h.listSessionArtifacts) + r.Get("/agents/sessions/{session_id}/artifacts/{artifact_id}", h.getSessionArtifact) + r.Get("/agents/sessions/{session_id}/artifacts/{artifact_id}/content", h.sessionArtifactContent) + r.Delete("/agents/sessions/{session_id}/artifacts/{artifact_id}", h.deleteSessionArtifact) r.NotFound(func(w http.ResponseWriter, _ *http.Request) { writeError(w, http.StatusNotFound, "unsupported_operation", "This API operation is not supported.") }) diff --git a/services/agents-api/internal/api/session_artifacts.go b/services/agents-api/internal/api/session_artifacts.go new file mode 100644 index 00000000..32aad833 --- /dev/null +++ b/services/agents-api/internal/api/session_artifacts.go @@ -0,0 +1,141 @@ +package api + +import ( + "context" + "io" + "net/http" + "path" + + v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/go-chi/chi/v5" +) + +type SessionArtifactStore interface { + GetSessionArtifact(context.Context, string, string, string) (store.SessionArtifact, error) + ListSessionArtifacts(context.Context, string, string, string, string, int, bool) (store.ArtifactPage, error) + ReadSessionArtifact(context.Context, string, string, string, func(store.SessionArtifact, io.Reader) error) error + DeleteSessionArtifact(context.Context, string, string, string) error +} + +func WithSessionArtifacts(s SessionArtifactStore) Option { + return func(h *Handler) { h.artifacts = s } +} + +func (h *Handler) artifactsReady(w http.ResponseWriter, r *http.Request, list bool) bool { + if h.artifacts == nil { + writeError(w, http.StatusServiceUnavailable, "artifact_storage_unavailable", "Artifact storage is unavailable.") + return false + } + if !list && r.URL.RawQuery != "" { + writeStoreError(w, r, store.ErrInvalidInput) + return false + } + return true +} + +// @Summary List immutable Session artifacts +// @Description Lists published outputs independently of Environment availability. Sorting uses publication time and ID. The local default page size is 20; exact upstream defaults and error parity remain unverified. +// @Tags Artifacts +// @Produce json +// @Security BearerAuth +// @Param OpenAI-Beta header string true "agents=v1" +// @Param session_id path string true "Session ID" +// @Param environment_id query string false "Producing Environment ID" +// @Param after query string false "Last immutable artifact ID" +// @Param limit query int false "Page size" minimum(1) maximum(100) default(20) +// @Param order query string false "Publication order" Enums(asc,desc) default(desc) +// @Success 200 {object} v1.SessionArtifactList +// @Failure 400,401,404,500,503 {object} v1.ErrorResponse +// @Router /agents/sessions/{session_id}/artifacts [get] +func (h *Handler) listSessionArtifacts(w http.ResponseWriter, r *http.Request) { + if !h.artifactsReady(w, r, true) { + return + } + options, ok := readPage(w, r, "environment_id") + if !ok { + return + } + page, err := h.artifacts.ListSessionArtifacts(r.Context(), tenantID(r), chi.URLParam(r, "session_id"), r.URL.Query().Get("environment_id"), options.after, options.limit, options.ascending) + if err != nil { + writeStoreError(w, r, err) + return + } + response := v1.SessionArtifactList{Data: make([]v1.SessionArtifact, 0, len(page.Artifacts)), HasMore: page.NextCursor != ""} + for _, artifact := range page.Artifacts { + response.Data = append(response.Data, artifactResponse(artifact)) + } + writeJSON(w, http.StatusOK, response) +} + +// @Summary Retrieve immutable artifact metadata +// @Tags Artifacts +// @Produce json +// @Security BearerAuth +// @Param OpenAI-Beta header string true "agents=v1" +// @Param session_id path string true "Session ID" +// @Param artifact_id path string true "Artifact ID" +// @Success 200 {object} v1.SessionArtifact +// @Failure 400,401,404,500,503 {object} v1.ErrorResponse +// @Router /agents/sessions/{session_id}/artifacts/{artifact_id} [get] +func (h *Handler) getSessionArtifact(w http.ResponseWriter, r *http.Request) { + if !h.artifactsReady(w, r, false) { + return + } + artifact, err := h.artifacts.GetSessionArtifact(r.Context(), tenantID(r), chi.URLParam(r, "session_id"), chi.URLParam(r, "artifact_id")) + if err != nil { + writeStoreError(w, r, err) + return + } + writeJSON(w, http.StatusOK, artifactResponse(artifact)) +} + +// @Summary Delete a published artifact +// @Description Deletes the published copy without modifying its original workspace file. Already admitted content reads may finish; later reads reject. +// @Tags Artifacts +// @Produce json +// @Security BearerAuth +// @Param OpenAI-Beta header string true "agents=v1" +// @Param session_id path string true "Session ID" +// @Param artifact_id path string true "Artifact ID" +// @Success 200 {object} v1.SessionArtifactDeleted +// @Failure 400,401,404,500,503 {object} v1.ErrorResponse +// @Router /agents/sessions/{session_id}/artifacts/{artifact_id} [delete] +func (h *Handler) deleteSessionArtifact(w http.ResponseWriter, r *http.Request) { + if !h.artifactsReady(w, r, false) { + return + } + id := chi.URLParam(r, "artifact_id") + if err := h.artifacts.DeleteSessionArtifact(r.Context(), tenantID(r), chi.URLParam(r, "session_id"), id); err != nil { + writeStoreError(w, r, err) + return + } + writeJSON(w, http.StatusOK, v1.SessionArtifactDeleted{ID: id, Object: "agent.session.artifact.deleted", Deleted: true}) +} + +// @Summary Download immutable artifact bytes +// @Description Streams stored bytes after tenant and Session authorization, including after Environment expiration. Exact upstream headers and Range behavior remain unverified. +// @Tags Artifacts +// @Produce octet-stream +// @Security BearerAuth +// @Param OpenAI-Beta header string true "agents=v1" +// @Param session_id path string true "Session ID" +// @Param artifact_id path string true "Artifact ID" +// @Success 200 {file} binary +// @Failure 400,401,404,500,503 {object} v1.ErrorResponse +// @Router /agents/sessions/{session_id}/artifacts/{artifact_id}/content [get] +func (h *Handler) sessionArtifactContent(w http.ResponseWriter, r *http.Request) { + if !h.artifactsReady(w, r, false) { + return + } + serveStoredContent(w, r, func(ctx context.Context, consume func(string, int64, io.Reader) error) error { + return h.artifacts.ReadSessionArtifact(ctx, tenantID(r), chi.URLParam(r, "session_id"), chi.URLParam(r, "artifact_id"), func(a store.SessionArtifact, body io.Reader) error { + return consume(path.Base(a.Path), a.SizeBytes, body) + }) + }) +} + +func artifactResponse(a store.SessionArtifact) v1.SessionArtifact { + return v1.SessionArtifact{ID: a.ID, CreatedAt: a.CreatedAt.Unix(), EnvironmentID: a.EnvironmentID, + Object: "agent.session.artifact", Path: a.Path, SessionID: a.SessionID, SizeBytes: a.SizeBytes, TurnID: a.TurnID} +} diff --git a/services/agents-api/internal/api/session_artifacts_test.go b/services/agents-api/internal/api/session_artifacts_test.go new file mode 100644 index 00000000..4a57706e --- /dev/null +++ b/services/agents-api/internal/api/session_artifacts_test.go @@ -0,0 +1,108 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +type artifactFixture struct { + artifact store.SessionArtifact + tenant, session, id, environment, cursor string + limit int + ascending bool + err error + calls int +} + +func (f *artifactFixture) GetSessionArtifact(_ context.Context, tenant, session, id string) (store.SessionArtifact, error) { + f.calls++ + f.tenant, f.session, f.id = tenant, session, id + return f.artifact, f.err +} + +func (f *artifactFixture) ListSessionArtifacts(_ context.Context, tenant, session, environment, cursor string, limit int, ascending bool) (store.ArtifactPage, error) { + f.calls++ + f.tenant, f.session, f.environment, f.cursor, f.limit, f.ascending = tenant, session, environment, cursor, limit, ascending + return store.ArtifactPage{Artifacts: []store.SessionArtifact{f.artifact}, NextCursor: f.artifact.ID}, f.err +} + +func (f *artifactFixture) ReadSessionArtifact(ctx context.Context, tenant, session, id string, consume func(store.SessionArtifact, io.Reader) error) error { + a, err := f.GetSessionArtifact(ctx, tenant, session, id) + if err != nil { + return err + } + return consume(a, bytes.NewReader([]byte{0, 255, 1})) +} + +func (f *artifactFixture) DeleteSessionArtifact(ctx context.Context, tenant, session, id string) error { + _, err := f.GetSessionArtifact(ctx, tenant, session, id) + return err +} + +type artifactResponseRecorder struct{ *httptest.ResponseRecorder } + +func (*artifactResponseRecorder) SetWriteDeadline(time.Time) error { return nil } + +func TestSessionArtifactRoutesAndPublicProjection(t *testing.T) { + f := &artifactFixture{artifact: store.SessionArtifact{ID: "artifact", SessionID: "session", EnvironmentID: "environment", TurnID: "turn", Path: "/workspace/outputs/a.bin", SizeBytes: 3, CreatedAt: time.Unix(123, 456)}} + h, _, tenant := testHandler(t, WithSessionArtifacts(f)) + request := func(method, suffix, beta string) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, "/v1/agents/sessions/session/artifacts"+suffix, nil) + r.Header.Set("Authorization", "Bearer test-api-key") + r.Header.Set("OpenAI-Beta", beta) + r.Header.Set("X-Tenant-ID", "untrusted") + w := &artifactResponseRecorder{httptest.NewRecorder()} + h.ServeHTTP(w, r) + return w.ResponseRecorder + } + w := request("GET", "/artifact", "agents=v1") + var got v1.SessionArtifact + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &got) != nil || got.Object != "agent.session.artifact" || got.CreatedAt != 123 || got.SizeBytes != 3 || got.Path != f.artifact.Path || got.TurnID != "turn" || got.EnvironmentID != "environment" || got.SessionID != "session" || got.ID != "artifact" { + t.Fatalf("metadata projection: %d %s", w.Code, w.Body) + } + if f.tenant != tenant || f.session != "session" || f.id != "artifact" { + t.Fatalf("untrusted request scope: %+v", f) + } + w = request("GET", "?environment_id=environment&after=previous&limit=1&order=asc", "agents=v1") + var page v1.SessionArtifactList + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &page) != nil || !page.HasMore || len(page.Data) != 1 || f.environment != "environment" || f.cursor != "previous" || f.limit != 1 || !f.ascending { + t.Fatalf("filtered page: %d %s %+v", w.Code, w.Body, f) + } + w = request("GET", "/artifact/content", "agents=v1") + if w.Code != 200 || !bytes.Equal(w.Body.Bytes(), []byte{0, 255, 1}) || w.Header().Get("Content-Type") != "application/octet-stream" || w.Header().Get("Content-Length") != "3" { + t.Fatalf("content: %d %s %v", w.Code, w.Body, w.Header()) + } + w = request("DELETE", "/artifact", "agents=v1") + var deleted v1.SessionArtifactDeleted + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &deleted) != nil || !deleted.Deleted || deleted.ID != "artifact" || deleted.Object != "agent.session.artifact.deleted" { + t.Fatalf("delete: %d %s", w.Code, w.Body) + } + for _, suffix := range []string{"?limit=0", "?limit=101", "?order=random", "?environment_id=a&environment_id=b", "?limit=1&limit=2", "?unknown=x", "/artifact?unknown=x", "/artifact/content?unknown=x"} { + before := f.calls + if w := request("GET", suffix, "agents=v1"); w.Code != 400 || f.calls != before { + t.Fatalf("invalid query reached storage: %s %d", suffix, w.Code) + } + } + for _, route := range []struct{ method, suffix string }{{"GET", ""}, {"GET", "/artifact"}, {"GET", "/artifact/content"}, {"DELETE", "/artifact"}} { + before := f.calls + if w := request(route.method, route.suffix, ""); w.Code != 400 || f.calls != before { + t.Fatalf("missing Beta accepted: %s %s %d", route.method, route.suffix, w.Code) + } + f.err = store.ErrNotFound + if w := request(route.method, route.suffix, "agents=v1"); w.Code != 404 { + t.Fatalf("not-found mapping: %s %s %d", route.method, route.suffix, w.Code) + } + } + if w := request(http.MethodPost, "", "agents=v1"); w.Code != 405 { + t.Fatalf("invented create route: %d", w.Code) + } +} diff --git a/services/agents-api/internal/api/source_files_content.go b/services/agents-api/internal/api/source_files_content.go index 1275b947..5848c44c 100644 --- a/services/agents-api/internal/api/source_files_content.go +++ b/services/agents-api/internal/api/source_files_content.go @@ -25,6 +25,14 @@ func (h *Handler) sourceFileContent(w http.ResponseWriter, r *http.Request) { if !h.sourceFilesReady(w, r) { return } + serveStoredContent(w, r, func(ctx context.Context, consume func(string, int64, io.Reader) error) error { + return h.sourceFiles.ReadSourceFile(ctx, tenantID(r), chi.URLParam(r, "file_id"), func(file store.SourceFile, body io.Reader) error { + return consume(file.Filename, file.SizeBytes, body) + }) + }) +} + +func serveStoredContent(w http.ResponseWriter, r *http.Request, read func(context.Context, func(string, int64, io.Reader) error) error) { deadline := time.Now().Add(sourceTransferTimeout) if http.NewResponseController(w).SetWriteDeadline(deadline) != nil { writeError(w, http.StatusServiceUnavailable, "file_transfer_unavailable", "Bounded file transfer is unavailable.") @@ -33,16 +41,16 @@ func (h *Handler) sourceFileContent(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithDeadline(r.Context(), deadline) defer cancel() started := false - err := h.sourceFiles.ReadSourceFile(ctx, tenantID(r), chi.URLParam(r, "file_id"), func(file store.SourceFile, body io.Reader) error { + err := read(ctx, func(filename string, size int64, body io.Reader) error { w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": file.Filename})) - w.Header().Set("Content-Length", strconv.FormatInt(file.SizeBytes, 10)) + w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": filename})) + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) w.Header().Set("Cache-Control", "no-store") w.Header().Set("X-Content-Type-Options", "nosniff") started = true w.WriteHeader(http.StatusOK) n, err := io.CopyBuffer(w, body, make([]byte, 256<<10)) - if err == nil && n != file.SizeBytes { + if err == nil && n != size { err = io.ErrUnexpectedEOF } return err diff --git a/services/agents-api/internal/db/queries/session_artifacts.sql b/services/agents-api/internal/db/queries/session_artifacts.sql new file mode 100644 index 00000000..a18beae7 --- /dev/null +++ b/services/agents-api/internal/db/queries/session_artifacts.sql @@ -0,0 +1,44 @@ +-- name: StageSessionArtifact :exec +INSERT INTO session_artifacts (id, session_id, turn_id, environment_id, path, size_bytes, body_oid, sha256) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8); + +-- name: PublishTurnArtifacts :exec +UPDATE session_artifacts SET created_at = $3 +WHERE session_id = $1 AND turn_id = $2 AND created_at IS NULL; + +-- name: DeleteUnpublishedTurnArtifacts :exec +WITH removed AS ( + DELETE FROM session_artifacts WHERE session_id = $1 AND turn_id = $2 AND created_at IS NULL RETURNING body_oid +) +SELECT lo_unlink(body_oid) FROM removed; + +-- name: DeleteSessionArtifacts :exec +WITH removed AS ( + DELETE FROM session_artifacts WHERE session_id = $1 RETURNING body_oid +) +SELECT lo_unlink(body_oid) FROM removed; + +-- name: GetSessionArtifact :one +SELECT a.* FROM session_artifacts a JOIN sessions s ON s.id = a.session_id +WHERE s.tenant_id = $1 AND s.deleted_at IS NULL AND a.session_id = $2 AND a.id = $3 AND a.created_at IS NOT NULL; + +-- name: DeleteSessionArtifact :one +DELETE FROM session_artifacts a USING sessions s +WHERE s.id = a.session_id AND s.tenant_id = $1 AND s.deleted_at IS NULL +AND a.session_id = $2 AND a.id = $3 AND a.created_at IS NOT NULL +RETURNING a.body_oid; + +-- name: ListSessionArtifacts :many +SELECT a.* FROM session_artifacts a JOIN sessions s ON s.id = a.session_id +WHERE s.tenant_id = sqlc.arg(tenant_id) AND s.deleted_at IS NULL + AND a.session_id = sqlc.arg(session_id) AND a.created_at IS NOT NULL + AND (sqlc.narg(environment_id)::uuid IS NULL OR a.environment_id = sqlc.narg(environment_id)::uuid) + AND (sqlc.narg(after_created)::timestamptz IS NULL + OR (NOT sqlc.arg(ascending)::boolean AND (a.created_at, a.id) < (sqlc.narg(after_created)::timestamptz, sqlc.arg(after_id)::uuid)) + OR (sqlc.arg(ascending)::boolean AND (a.created_at, a.id) > (sqlc.narg(after_created)::timestamptz, sqlc.arg(after_id)::uuid))) +ORDER BY + CASE WHEN sqlc.arg(ascending)::boolean THEN a.created_at END ASC, + CASE WHEN sqlc.arg(ascending)::boolean THEN a.id END ASC, + CASE WHEN NOT sqlc.arg(ascending)::boolean THEN a.created_at END DESC, + CASE WHEN NOT sqlc.arg(ascending)::boolean THEN a.id END DESC +LIMIT sqlc.arg(page_limit); diff --git a/services/agents-api/internal/db/sqlc/models.go b/services/agents-api/internal/db/sqlc/models.go index c28b221e..6082eb23 100644 --- a/services/agents-api/internal/db/sqlc/models.go +++ b/services/agents-api/internal/db/sqlc/models.go @@ -121,6 +121,18 @@ type Session struct { CreatorID pgtype.Text `json:"creator_id"` } +type SessionArtifact struct { + ID pgtype.UUID `json:"id"` + SessionID pgtype.UUID `json:"session_id"` + TurnID pgtype.UUID `json:"turn_id"` + EnvironmentID pgtype.UUID `json:"environment_id"` + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + BodyOid pgtype.Uint32 `json:"body_oid"` + Sha256 string `json:"sha256"` + CreatedAt pgtype.Timestamptz `json:"created_at"` +} + type SessionDevice struct { SessionID pgtype.UUID `json:"session_id"` DeviceID pgtype.UUID `json:"device_id"` diff --git a/services/agents-api/internal/db/sqlc/session_artifacts.sql.go b/services/agents-api/internal/db/sqlc/session_artifacts.sql.go new file mode 100644 index 00000000..54e8c107 --- /dev/null +++ b/services/agents-api/internal/db/sqlc/session_artifacts.sql.go @@ -0,0 +1,199 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: session_artifacts.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const deleteSessionArtifact = `-- name: DeleteSessionArtifact :one +DELETE FROM session_artifacts a USING sessions s +WHERE s.id = a.session_id AND s.tenant_id = $1 AND s.deleted_at IS NULL +AND a.session_id = $2 AND a.id = $3 AND a.created_at IS NOT NULL +RETURNING a.body_oid +` + +type DeleteSessionArtifactParams struct { + TenantID pgtype.UUID `json:"tenant_id"` + SessionID pgtype.UUID `json:"session_id"` + ID pgtype.UUID `json:"id"` +} + +func (q *Queries) DeleteSessionArtifact(ctx context.Context, arg DeleteSessionArtifactParams) (pgtype.Uint32, error) { + row := q.db.QueryRow(ctx, deleteSessionArtifact, arg.TenantID, arg.SessionID, arg.ID) + var body_oid pgtype.Uint32 + err := row.Scan(&body_oid) + return body_oid, err +} + +const deleteSessionArtifacts = `-- name: DeleteSessionArtifacts :exec +WITH removed AS ( + DELETE FROM session_artifacts WHERE session_id = $1 RETURNING body_oid +) +SELECT lo_unlink(body_oid) FROM removed +` + +func (q *Queries) DeleteSessionArtifacts(ctx context.Context, sessionID pgtype.UUID) error { + _, err := q.db.Exec(ctx, deleteSessionArtifacts, sessionID) + return err +} + +const deleteUnpublishedTurnArtifacts = `-- name: DeleteUnpublishedTurnArtifacts :exec +WITH removed AS ( + DELETE FROM session_artifacts WHERE session_id = $1 AND turn_id = $2 AND created_at IS NULL RETURNING body_oid +) +SELECT lo_unlink(body_oid) FROM removed +` + +type DeleteUnpublishedTurnArtifactsParams struct { + SessionID pgtype.UUID `json:"session_id"` + TurnID pgtype.UUID `json:"turn_id"` +} + +func (q *Queries) DeleteUnpublishedTurnArtifacts(ctx context.Context, arg DeleteUnpublishedTurnArtifactsParams) error { + _, err := q.db.Exec(ctx, deleteUnpublishedTurnArtifacts, arg.SessionID, arg.TurnID) + return err +} + +const getSessionArtifact = `-- name: GetSessionArtifact :one +SELECT a.id, a.session_id, a.turn_id, a.environment_id, a.path, a.size_bytes, a.body_oid, a.sha256, a.created_at FROM session_artifacts a JOIN sessions s ON s.id = a.session_id +WHERE s.tenant_id = $1 AND s.deleted_at IS NULL AND a.session_id = $2 AND a.id = $3 AND a.created_at IS NOT NULL +` + +type GetSessionArtifactParams struct { + TenantID pgtype.UUID `json:"tenant_id"` + SessionID pgtype.UUID `json:"session_id"` + ID pgtype.UUID `json:"id"` +} + +func (q *Queries) GetSessionArtifact(ctx context.Context, arg GetSessionArtifactParams) (SessionArtifact, error) { + row := q.db.QueryRow(ctx, getSessionArtifact, arg.TenantID, arg.SessionID, arg.ID) + var i SessionArtifact + err := row.Scan( + &i.ID, + &i.SessionID, + &i.TurnID, + &i.EnvironmentID, + &i.Path, + &i.SizeBytes, + &i.BodyOid, + &i.Sha256, + &i.CreatedAt, + ) + return i, err +} + +const listSessionArtifacts = `-- name: ListSessionArtifacts :many +SELECT a.id, a.session_id, a.turn_id, a.environment_id, a.path, a.size_bytes, a.body_oid, a.sha256, a.created_at FROM session_artifacts a JOIN sessions s ON s.id = a.session_id +WHERE s.tenant_id = $1 AND s.deleted_at IS NULL + AND a.session_id = $2 AND a.created_at IS NOT NULL + AND ($3::uuid IS NULL OR a.environment_id = $3::uuid) + AND ($4::timestamptz IS NULL + OR (NOT $5::boolean AND (a.created_at, a.id) < ($4::timestamptz, $6::uuid)) + OR ($5::boolean AND (a.created_at, a.id) > ($4::timestamptz, $6::uuid))) +ORDER BY + CASE WHEN $5::boolean THEN a.created_at END ASC, + CASE WHEN $5::boolean THEN a.id END ASC, + CASE WHEN NOT $5::boolean THEN a.created_at END DESC, + CASE WHEN NOT $5::boolean THEN a.id END DESC +LIMIT $7 +` + +type ListSessionArtifactsParams struct { + TenantID pgtype.UUID `json:"tenant_id"` + SessionID pgtype.UUID `json:"session_id"` + EnvironmentID pgtype.UUID `json:"environment_id"` + AfterCreated pgtype.Timestamptz `json:"after_created"` + Ascending bool `json:"ascending"` + AfterID pgtype.UUID `json:"after_id"` + PageLimit int32 `json:"page_limit"` +} + +func (q *Queries) ListSessionArtifacts(ctx context.Context, arg ListSessionArtifactsParams) ([]SessionArtifact, error) { + rows, err := q.db.Query(ctx, listSessionArtifacts, + arg.TenantID, + arg.SessionID, + arg.EnvironmentID, + arg.AfterCreated, + arg.Ascending, + arg.AfterID, + arg.PageLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []SessionArtifact{} + for rows.Next() { + var i SessionArtifact + if err := rows.Scan( + &i.ID, + &i.SessionID, + &i.TurnID, + &i.EnvironmentID, + &i.Path, + &i.SizeBytes, + &i.BodyOid, + &i.Sha256, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const publishTurnArtifacts = `-- name: PublishTurnArtifacts :exec +UPDATE session_artifacts SET created_at = $3 +WHERE session_id = $1 AND turn_id = $2 AND created_at IS NULL +` + +type PublishTurnArtifactsParams struct { + SessionID pgtype.UUID `json:"session_id"` + TurnID pgtype.UUID `json:"turn_id"` + CreatedAt pgtype.Timestamptz `json:"created_at"` +} + +func (q *Queries) PublishTurnArtifacts(ctx context.Context, arg PublishTurnArtifactsParams) error { + _, err := q.db.Exec(ctx, publishTurnArtifacts, arg.SessionID, arg.TurnID, arg.CreatedAt) + return err +} + +const stageSessionArtifact = `-- name: StageSessionArtifact :exec +INSERT INTO session_artifacts (id, session_id, turn_id, environment_id, path, size_bytes, body_oid, sha256) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +` + +type StageSessionArtifactParams struct { + ID pgtype.UUID `json:"id"` + SessionID pgtype.UUID `json:"session_id"` + TurnID pgtype.UUID `json:"turn_id"` + EnvironmentID pgtype.UUID `json:"environment_id"` + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + BodyOid pgtype.Uint32 `json:"body_oid"` + Sha256 string `json:"sha256"` +} + +func (q *Queries) StageSessionArtifact(ctx context.Context, arg StageSessionArtifactParams) error { + _, err := q.db.Exec(ctx, stageSessionArtifact, + arg.ID, + arg.SessionID, + arg.TurnID, + arg.EnvironmentID, + arg.Path, + arg.SizeBytes, + arg.BodyOid, + arg.Sha256, + ) + return err +} diff --git a/services/agents-api/internal/execution/artifacts.go b/services/agents-api/internal/execution/artifacts.go new file mode 100644 index 00000000..3a92912d --- /dev/null +++ b/services/agents-api/internal/execution/artifacts.go @@ -0,0 +1,36 @@ +package execution + +import ( + "context" + "io" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +func (d *Dispatcher) captureCompletedArtifacts(ctx context.Context, peer *gateway.Session, session store.Session, environment store.Environment, bound store.ExecutionDevice, turnID string, result Result, status string) (Result, string) { + if status != store.TurnCompleted || !LocalWorkspaceConfiguration(environment.Configuration) { + return result, status + } + owner, cancel := context.WithTimeout(ctx, 180*time.Second) + defer cancel() + err := d.withPreparedWorkspace(owner, peer, session, environment, bound, func(ctx context.Context, handle string) error { + return peer.ExportWorkspaceOutputs(ctx, proto.WorkspaceExportPayload{Handle: handle, EnvironmentID: environment.ID}, func(body io.Reader) error { + return d.Store.StageTurnArtifacts(ctx, session.TenantID, session.ID, turnID, environment.ID, body) + }) + }) + if err == nil { + return result, status + } + // Do not expose native diagnostics or publish partial output after a failed capture. + result.ErrorCode = "artifact_capture_failed" + status = store.TurnFailed + check, stop := context.WithTimeout(context.Background(), 5*time.Second) + defer stop() + if turn, err := d.Store.GetTurn(check, session.TenantID, session.ID, turnID); err == nil && !turn.CancelRequestedAt.IsZero() { + status = store.TurnCancelled + } + return result, status +} diff --git a/services/agents-api/internal/execution/directory_preparation.go b/services/agents-api/internal/execution/directory_preparation.go index e0dffdb4..97ffd981 100644 --- a/services/agents-api/internal/execution/directory_preparation.go +++ b/services/agents-api/internal/execution/directory_preparation.go @@ -10,20 +10,32 @@ import ( ) func (d *Dispatcher) readPreparedDirectory(ctx context.Context, peer *gateway.Session, session store.Session, environment store.Environment, bound store.ExecutionDevice, read proto.WorkspaceReadPayload) directoryReadResult { - unavailable := directoryReadResult{err: ErrExecutionUnavailable} owner, cancel := context.WithTimeout(ctx, 45*time.Second) defer cancel() + var result directoryReadResult + err := d.withPreparedWorkspace(owner, peer, session, environment, bound, func(ctx context.Context, handle string) error { + read.Handle = handle + result = readEnvironmentDirectory(ctx, peer, read) + return result.err + }) + if err != nil { + result.err = err + } + return result +} + +func (d *Dispatcher) withPreparedWorkspace(owner context.Context, peer *gateway.Session, session store.Session, environment store.Environment, bound store.ExecutionDevice, consume func(context.Context, string) error) error { req := proto.PromptRequestPayload{AgentKind: session.Engine, AgentStateKey: "agents-api-" + session.ID, StrictResume: true, ReleaseOnCompletion: true, WorkspaceReadOnly: true} release, err := d.configurePreparedEnvironment(owner, session, environment, bound, &req) if release != nil { defer release() } if err != nil { - return unavailable + return ErrExecutionUnavailable } prepared, err := newPreparedStart(peer) if err != nil { - return unavailable + return ErrExecutionUnavailable } releaseAttempted := false defer func() { @@ -40,15 +52,14 @@ func (d *Dispatcher) readPreparedDirectory(ctx context.Context, peer *gateway.Se } stop() if err != nil { - return unavailable + return ErrExecutionUnavailable } - read.Handle = prepared.handle - result := readEnvironmentDirectory(owner, peer, read) + err = consume(owner, prepared.handle) releaseAttempted = true if prepared.releaseDirectory() != nil { - return unavailable + return ErrExecutionUnavailable } - return result + return err } func (p *preparedStart) awaitDirectoryReady(ctx context.Context) error { diff --git a/services/agents-api/internal/execution/prepared_dispatch.go b/services/agents-api/internal/execution/prepared_dispatch.go index 319b9298..fe7c78f2 100644 --- a/services/agents-api/internal/execution/prepared_dispatch.go +++ b/services/agents-api/internal/execution/prepared_dispatch.go @@ -99,6 +99,7 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, tenantID, sessionI req.Prompt = strings.Join(messages, "\n\n") through := run.Reservation.Receipts[len(run.Reservation.Receipts)-1].Sequence result, status := d.deliver(owner, tenantID, sessionID, peer, req, through, prepared) + result, status = d.captureCompletedArtifacts(owner, peer, session, environment, bound, req.RunID, result, status) run.Turn, err = d.finishRun(tenantID, sessionID, req.RunID, snapshot.Agent.Model, result, status) return run, err } diff --git a/services/agents-api/internal/execution/support.go b/services/agents-api/internal/execution/support.go index f8fca7d0..cd8fdb91 100644 --- a/services/agents-api/internal/execution/support.go +++ b/services/agents-api/internal/execution/support.go @@ -163,8 +163,8 @@ func engineCapabilities(peer *gateway.Session, engine string, snapshot Snapshot) return fail("device must advertise preparation and remote_environment") } if snapshot.Environment != nil && snapshot.Environment.Type == "openai_hosted" { - if !caps.Preparation || !caps.LocalEnvironment { - return fail("device must advertise preparation and local_environment") + if !caps.Preparation || !caps.LocalEnvironment || !caps.WorkspaceReadPreparation || !caps.WorkspaceOutputExport { + return fail("device must advertise local preparation, workspace reads and output export") } if (snapshot.Environment.Network == nil || snapshot.Environment.Network.Access != "disabled") && !caps.LocalEnvironmentNetworkPolicy { return fail("device must advertise local_environment_network_policy") diff --git a/services/agents-api/internal/store/artifact_capture.go b/services/agents-api/internal/store/artifact_capture.go new file mode 100644 index 00000000..4eeab6bb --- /dev/null +++ b/services/agents-api/internal/store/artifact_capture.go @@ -0,0 +1,135 @@ +package store + +import ( + "archive/tar" + "context" + "encoding/hex" + "encoding/json" + "errors" + "io" + "io/fs" + "strings" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +const MaxArtifactBytes int64 = 200 << 20 +const MaxArtifactBatchBytes int64 = 500 << 20 + +// StageTurnArtifacts stores a complete export privately without locking admission during transfer. +func (s *Store) StageTurnArtifacts(ctx context.Context, tenantID, sessionID, turnID, environmentID string, input io.Reader) error { + lookup, err := turnLookup(tenantID, sessionID, turnID) + if err != nil { + return err + } + environment, err := parseID(environmentID) + if err != nil || input == nil { + return ErrInvalidInput + } + // Authorize before reading caller-controlled bytes or allocating storage. + owned, err := s.GetSessionEnvironment(ctx, tenantID, sessionID) + if err != nil { + return err + } + if owned.ID != environmentID { + return ErrNotFound + } + var configuration struct { + Type string `json:"type"` + } + if err := json.Unmarshal(owned.Configuration, &configuration); err != nil { + return err + } + if configuration.Type != "openai_hosted" { + return ErrInvalidInput + } + tx, err := s.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + rows, err := captureArtifactArchive(ctx, tx, input) + if err != nil { + return err + } + q := s.queries.WithTx(tx) + locked, err := q.LockSession(ctx, sqlc.LockSessionParams{TenantID: lookup.TenantID, ID: lookup.SessionID}) + if errors.Is(err, pgx.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return err + } + if locked.DeletedAt.Valid { + return ErrNotFound + } + turn, err := q.GetTurn(ctx, lookup) + if err != nil { + return err + } + if turn.Status != TurnInProgress || turn.CancelRequestedAt.Valid { + return ErrTurnConflict + } + for _, row := range rows { + row.SessionID, row.TurnID, row.EnvironmentID = lookup.SessionID, lookup.ID, environment + if err := q.StageSessionArtifact(ctx, row); err != nil { + return err + } + } + return tx.Commit(ctx) +} + +func captureArtifactArchive(ctx context.Context, tx pgx.Tx, input io.Reader) ([]sqlc.StageSessionArtifactParams, error) { + archive := tar.NewReader(input) + objects := tx.LargeObjects() + rows := make([]sqlc.StageSessionArtifactParams, 0) + seen := make(map[string]bool) + var total int64 + for { + header, err := archive.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if header.Typeflag != tar.TypeReg || !strings.HasPrefix(header.Name, "outputs/") || !fs.ValidPath(header.Name) || strings.ContainsAny(header.Name, "\\\x00\r\n") || len(header.Name) > 4096 || header.Size < 0 || header.Size > MaxArtifactBytes || header.Size > MaxArtifactBatchBytes-total || len(rows) >= 4096 || seen[header.Name] { + return nil, ErrInvalidInput + } + seen[header.Name] = true + total += header.Size + oid, err := objects.Create(ctx, 0) + if err != nil { + return nil, err + } + body, err := objects.Open(ctx, oid, pgx.LargeObjectModeWrite) + if err != nil { + return nil, err + } + writer := newSourceFileWriter(body) + if _, err := io.CopyN(writer, archive, header.Size); err != nil { + return nil, err + } + if err := body.Close(); err != nil { + return nil, err + } + rows = append(rows, sqlc.StageSessionArtifactParams{ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, Path: "/workspace/" + header.Name, SizeBytes: writer.size, BodyOid: pgtype.Uint32{Uint32: oid, Valid: true}, Sha256: hex.EncodeToString(writer.hash.Sum(nil))}) + } + // Require transport EOF after the archive trailer, including confirmed helper exit. + padding, err := io.ReadAll(io.LimitReader(input, 32769)) + if err != nil { + return nil, err + } + if len(padding) > 32768 { + return nil, ErrInvalidInput + } + for _, b := range padding { + if b != 0 { + return nil, ErrInvalidInput + } + } + return rows, nil +} diff --git a/services/agents-api/internal/store/artifact_lifecycle.go b/services/agents-api/internal/store/artifact_lifecycle.go new file mode 100644 index 00000000..ff031fdf --- /dev/null +++ b/services/agents-api/internal/store/artifact_lifecycle.go @@ -0,0 +1,14 @@ +package store + +import ( + "context" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" +) + +func settleTurnArtifacts(ctx context.Context, q *sqlc.Queries, turn sqlc.Turn) error { + if turn.Status == TurnCompleted { + return q.PublishTurnArtifacts(ctx, sqlc.PublishTurnArtifactsParams{SessionID: turn.SessionID, TurnID: turn.ID, CreatedAt: turn.CompletedAt}) + } + return q.DeleteUnpublishedTurnArtifacts(ctx, sqlc.DeleteUnpublishedTurnArtifactsParams{SessionID: turn.SessionID, TurnID: turn.ID}) +} diff --git a/services/agents-api/internal/store/local_artifact_export_test.go b/services/agents-api/internal/store/local_artifact_export_test.go new file mode 100644 index 00000000..50c49ebf --- /dev/null +++ b/services/agents-api/internal/store/local_artifact_export_test.go @@ -0,0 +1,53 @@ +package store_test + +import ( + "archive/tar" + "bytes" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +func completeLocalArtifactExport(t *testing.T, h *dispatchHarness, environment store.Environment) { + t.Helper() + prepared := h.read(proto.TypeExecutionPrepare) + var request proto.ExecutionPreparePayload + if prepared.DecodePayload(&request) != nil || !proto.ValidWorkspaceReadPreparation(request.Configuration) || request.Configuration.LocalEnvironment == nil || request.Configuration.LocalEnvironment.ID != environment.ID { + t.Fatal("export did not reuse the bound read-only preparation") + } + handle := acknowledgePreparation(h, prepared.ID) + h.write(prepared.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 2, State: "ready"}) + begin := h.read(proto.TypeWorkspaceExport) + var export proto.WorkspaceExportPayload + if begin.DecodePayload(&export) != nil || export.Step != "begin" || export.Handle != handle || export.EnvironmentID != environment.ID { + t.Fatal("export lost preparation authority") + } + var data bytes.Buffer + w := tar.NewWriter(&data) + if err := w.WriteHeader(&tar.Header{Name: "outputs/result.bin", Size: 3, Mode: 0600, Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte{0, 255, 1}); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + h.write(begin.ID, proto.TypeWorkspaceExportResult, proto.WorkspaceExportResultPayload{Outcome: "chunk", Data: data.Bytes()}) + next := h.read(proto.TypeWorkspaceExport) + if next.DecodePayload(&export) != nil || export.Step != "next" || export.Offset != int64(data.Len()) { + t.Fatal("export did not await native completion") + } + page, err := h.s.ListSessionArtifacts(t.Context(), h.tenant, h.session.ID, "", "", 20, false) + if err != nil || len(page.Artifacts) != 0 { + t.Fatal("capture published before native completion", err) + } + h.write(begin.ID, proto.TypeWorkspaceExportResult, proto.WorkspaceExportResultPayload{Outcome: "completed", Offset: export.Offset}) + release := h.read(proto.TypeExecutionRelease) + var close proto.ExecutionReleasePayload + if release.DecodePayload(&close) != nil || release.ID != prepared.ID || close.Handle != handle { + t.Fatal("export did not release its own read preparation") + } + h.write(prepared.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 3, State: "released"}) +} diff --git a/services/agents-api/internal/store/local_environment_worker_test.go b/services/agents-api/internal/store/local_environment_worker_test.go index 18792397..d00a29da 100644 --- a/services/agents-api/internal/store/local_environment_worker_test.go +++ b/services/agents-api/internal/store/local_environment_worker_test.go @@ -22,6 +22,7 @@ func localWorker(t *testing.T, scoped, execute bool) (*dispatchHarness, *executi } caps := proto.AgentKindCapabilities{LocalEnvironment: true, Preparation: true, WorkspaceReadPreparation: true} if execute { + caps.WorkspaceOutputExport = true caps.Streaming, caps.Steering, caps.DurableTurns, caps.DurableInputReceipts = true, true, true, true caps.WebSearchControl, caps.TextVerbosity, caps.ExecutionControls = true, true, true caps.SubagentControl, caps.ToolObservations = true, true @@ -140,6 +141,7 @@ func TestLocalEnvironmentWorkerSchedulesPreparationWithoutRemoteResolver(t *test } h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 3, State: "started", RunID: start.RunID}) h.write(start.RunID, proto.TypeDone, proto.DonePayload{Content: "complete", Metadata: map[string]any{proto.DoneMetaAgentSessionID: "local-native-history"}}) + completeLocalArtifactExport(t, h, environment) awaitDaemonRemoteCondition(t, t.Context(), 5*time.Second, "local completion", func() bool { turn, err := h.s.GetTurn(t.Context(), h.tenant, h.session.ID, start.RunID) return err == nil && turn.Status == store.TurnCompleted @@ -152,4 +154,8 @@ func TestLocalEnvironmentWorkerSchedulesPreparationWithoutRemoteResolver(t *test if err != nil || bound.EnvironmentID != environment.ID || bound.NativeSessionID != "local-native-history" { t.Fatal("local native identity was not retained", err) } + artifacts, err := h.s.ListSessionArtifacts(t.Context(), h.tenant, h.session.ID, environment.ID, "", 20, false) + if err != nil || len(artifacts.Artifacts) != 1 || artifacts.Artifacts[0].Path != "/workspace/outputs/result.bin" || artifacts.Artifacts[0].TurnID != start.RunID || artifacts.Artifacts[0].SizeBytes != 3 { + t.Fatalf("completed turn did not publish output: %+v %v", artifacts, err) + } } diff --git a/services/agents-api/internal/store/session_artifacts.go b/services/agents-api/internal/store/session_artifacts.go new file mode 100644 index 00000000..3c586658 --- /dev/null +++ b/services/agents-api/internal/store/session_artifacts.go @@ -0,0 +1,159 @@ +package store + +import ( + "context" + "errors" + "io" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +type SessionArtifact struct { + ID string + SessionID string + TurnID string + EnvironmentID string + Path string + SizeBytes int64 + CreatedAt time.Time +} + +type ArtifactPage struct { + Artifacts []SessionArtifact + NextCursor string +} + +func (s *Store) GetSessionArtifact(ctx context.Context, tenantID, sessionID, artifactID string) (SessionArtifact, error) { + lookup, err := artifactLookup(tenantID, sessionID, artifactID) + if err != nil { + return SessionArtifact{}, err + } + row, err := s.queries.GetSessionArtifact(ctx, lookup) + if errors.Is(err, pgx.ErrNoRows) { + return SessionArtifact{}, ErrNotFound + } + return artifactFromRow(row), err +} + +func (s *Store) ListSessionArtifacts(ctx context.Context, tenantID, sessionID, environmentID, cursor string, limit int, ascending bool) (ArtifactPage, error) { + if limit < 1 || limit > 100 { + return ArtifactPage{}, ErrInvalidInput + } + if _, err := s.GetSession(ctx, tenantID, sessionID); err != nil { + return ArtifactPage{}, err + } + tenant, _ := parseID(tenantID) + session, _ := parseID(sessionID) + params := sqlc.ListSessionArtifactsParams{TenantID: tenant, SessionID: session, PageLimit: int32(limit + 1), Ascending: ascending, AfterID: pgtype.UUID{Valid: true}} + if environmentID != "" { + var err error + params.EnvironmentID, err = parseID(environmentID) + if err != nil { + return ArtifactPage{}, err + } + } + if cursor != "" { + after, err := s.GetSessionArtifact(ctx, tenantID, sessionID, cursor) + if err != nil { + return ArtifactPage{}, err + } + params.AfterCreated = pgtype.Timestamptz{Time: after.CreatedAt, Valid: true} + params.AfterID, _ = parseID(after.ID) + } + rows, err := s.queries.ListSessionArtifacts(ctx, params) + if err != nil { + return ArtifactPage{}, err + } + page := ArtifactPage{Artifacts: make([]SessionArtifact, 0, min(limit, len(rows)))} + if len(rows) > limit { + page.NextCursor = uuid.UUID(rows[limit-1].ID.Bytes).String() + rows = rows[:limit] + } + for _, row := range rows { + page.Artifacts = append(page.Artifacts, artifactFromRow(row)) + } + return page, nil +} + +// ReadSessionArtifact keeps an admitted snapshot available across concurrent deletion. +func (s *Store) ReadSessionArtifact(ctx context.Context, tenantID, sessionID, artifactID string, consume func(SessionArtifact, io.Reader) error) error { + lookup, err := artifactLookup(tenantID, sessionID, artifactID) + if err != nil { + return err + } + if consume == nil { + return ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + row, err := s.queries.WithTx(tx).GetSessionArtifact(ctx, lookup) + if errors.Is(err, pgx.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return err + } + objects := tx.LargeObjects() + body, err := objects.Open(ctx, row.BodyOid.Uint32, pgx.LargeObjectModeRead) + if err != nil { + return err + } + if err := consume(artifactFromRow(row), body); err != nil { + return err + } + if err := body.Close(); err != nil { + return err + } + return tx.Commit(ctx) +} + +func (s *Store) DeleteSessionArtifact(ctx context.Context, tenantID, sessionID, artifactID string) error { + lookup, err := artifactLookup(tenantID, sessionID, artifactID) + if err != nil { + return err + } + tx, err := s.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + q := s.queries.WithTx(tx) + // Use the same lock order as whole-Session deletion and Turn publication. + locked, err := q.LockSession(ctx, sqlc.LockSessionParams{TenantID: lookup.TenantID, ID: lookup.SessionID}) + if errors.Is(err, pgx.ErrNoRows) || err == nil && locked.DeletedAt.Valid { + return ErrNotFound + } + if err != nil { + return err + } + oid, err := q.DeleteSessionArtifact(ctx, sqlc.DeleteSessionArtifactParams(lookup)) + if errors.Is(err, pgx.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return err + } + objects := tx.LargeObjects() + if err := objects.Unlink(ctx, oid.Uint32); err != nil { + return err + } + return tx.Commit(ctx) +} + +func artifactLookup(tenantID, sessionID, artifactID string) (sqlc.GetSessionArtifactParams, error) { + ids, err := turnLookup(tenantID, sessionID, artifactID) + return sqlc.GetSessionArtifactParams{TenantID: ids.TenantID, SessionID: ids.SessionID, ID: ids.ID}, err +} + +func artifactFromRow(row sqlc.SessionArtifact) SessionArtifact { + return SessionArtifact{ID: uuid.UUID(row.ID.Bytes).String(), SessionID: uuid.UUID(row.SessionID.Bytes).String(), + TurnID: uuid.UUID(row.TurnID.Bytes).String(), EnvironmentID: uuid.UUID(row.EnvironmentID.Bytes).String(), + Path: row.Path, SizeBytes: row.SizeBytes, CreatedAt: row.CreatedAt.Time} +} diff --git a/services/agents-api/internal/store/session_artifacts_test.go b/services/agents-api/internal/store/session_artifacts_test.go new file mode 100644 index 00000000..2884295d --- /dev/null +++ b/services/agents-api/internal/store/session_artifacts_test.go @@ -0,0 +1,261 @@ +package store + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "io" + "reflect" + "testing" + "time" + + "github.com/google/uuid" +) + +func artifactArchive(t *testing.T, files map[string][]byte) []byte { + t.Helper() + var data bytes.Buffer + w := tar.NewWriter(&data) + for name, body := range files { + if err := w.WriteHeader(&tar.Header{Name: name, Mode: 0600, Size: int64(len(body)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := w.Write(body); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return data.Bytes() +} + +func artifactTurn(t *testing.T, s *Store, kind string) (tenant, session, environment, turn string) { + t.Helper() + tenant = uuid.NewString() + created, err := s.CreateSession(t.Context(), tenant, environmentInput("artifact", kind, "/workspace")) + if err != nil { + t.Fatal(err) + } + env, err := s.GetSessionEnvironment(t.Context(), tenant, created.ID) + if err != nil { + t.Fatal(err) + } + input := submitMessage(t, s, tenant, created.ID, "artifact-turn") + transition(t, s, tenant, created.ID, input.TurnID, TurnQueued, TurnInProgress) + return tenant, created.ID, env.ID, input.TurnID +} + +func TestSessionArtifactsPublishVersionScopeAndLifetime(t *testing.T) { + s, pool := testStore(t) + tenant, session, environment, turn := artifactTurn(t, s, "openai_hosted") + before := sourceObjectCount(t, pool) + data := bytes.Repeat([]byte("immutable\x00"), 100000) + archive := artifactArchive(t, map[string][]byte{"outputs/a.bin": data, "outputs/nested/empty": {}}) + if err := s.StageTurnArtifacts(t.Context(), tenant, session, turn, environment, bytes.NewReader(archive)); err != nil { + t.Fatal(err) + } + page, err := s.ListSessionArtifacts(t.Context(), tenant, session, "", "", 100, true) + if err != nil || len(page.Artifacts) != 0 { + t.Fatalf("private capture visible: %+v %v", page, err) + } + completed := transition(t, s, tenant, session, turn, TurnInProgress, TurnCompleted) + page, err = s.ListSessionArtifacts(t.Context(), tenant, session, environment, "", 100, true) + if err != nil || len(page.Artifacts) != 2 { + t.Fatalf("published capture: %+v %v", page, err) + } + for _, a := range page.Artifacts { + if a.SessionID != session || a.TurnID != turn || a.EnvironmentID != environment || !a.CreatedAt.Equal(completed.CompletedAt) { + t.Fatalf("publication metadata: %+v", a) + } + foreign := uuid.NewString() + if _, err := s.GetSessionArtifact(t.Context(), foreign, session, a.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("foreign metadata: %v", err) + } + if _, err := s.GetSessionArtifact(t.Context(), tenant, uuid.NewString(), a.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("wrong session metadata: %v", err) + } + if err := s.DeleteSessionArtifact(t.Context(), foreign, session, a.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("foreign delete: %v", err) + } + if err := s.ReadSessionArtifact(t.Context(), foreign, session, a.ID, func(SessionArtifact, io.Reader) error { + t.Error("foreign read reached content") + return nil + }); !errors.Is(err, ErrNotFound) { + t.Fatalf("foreign read: %v", err) + } + } + if _, err := s.ListSessionArtifacts(t.Context(), uuid.NewString(), session, "", "", 100, false); !errors.Is(err, ErrNotFound) { + t.Fatalf("foreign list: %v", err) + } + if empty, err := s.ListSessionArtifacts(t.Context(), tenant, session, uuid.NewString(), "", 100, false); err != nil || len(empty.Artifacts) != 0 { + t.Fatalf("environment filter: %+v %v", empty, err) + } + // A later completed Turn publishes another immutable version of the same path. + next := submitMessage(t, s, tenant, session, "version-two") + transition(t, s, tenant, session, next.TurnID, TurnQueued, TurnInProgress) + if err := s.StageTurnArtifacts(t.Context(), tenant, session, next.TurnID, environment, bytes.NewReader(artifactArchive(t, map[string][]byte{"outputs/a.bin": []byte("new")}))); err != nil { + t.Fatal(err) + } + transition(t, s, tenant, session, next.TurnID, TurnInProgress, TurnCompleted) + all, err := s.ListSessionArtifacts(t.Context(), tenant, session, "", "", 100, true) + if err != nil || len(all.Artifacts) != 3 { + t.Fatal(all, err) + } + for _, asc := range []bool{true, false} { + var got []SessionArtifact + cursor := "" + for { + part, err := s.ListSessionArtifacts(t.Context(), tenant, session, environment, cursor, 1, asc) + if err != nil { + t.Fatal(err) + } + got = append(got, part.Artifacts...) + if part.NextCursor == "" { + break + } + if len(got) > 3 { + t.Fatal("pagination repeated artifacts") + } + cursor = part.NextCursor + } + want := append([]SessionArtifact(nil), all.Artifacts...) + if !asc { + want[0], want[2] = want[2], want[0] + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("ordered pages differ: %+v %+v", got, want) + } + } + // Expiration is a controlled fixture; stored reads must not touch Runtime. + if _, err := pool.Exec(t.Context(), "UPDATE environments SET status='expired' WHERE id=$1", environment); err != nil { + t.Fatal(err) + } + for _, a := range page.Artifacts { + if err := New(pool).ReadSessionArtifact(t.Context(), tenant, session, a.ID, func(meta SessionArtifact, r io.Reader) error { + if err := s.DeleteSessionArtifact(t.Context(), tenant, session, a.ID); err != nil { + return err + } + body, err := io.ReadAll(r) + want := data + if a.Path == "/workspace/outputs/nested/empty" { + want = nil + } + if meta != a || !bytes.Equal(body, want) { + t.Error("expired/deleted artifact damaged admitted snapshot") + } + return err + }); err != nil { + t.Fatal(err) + } + if _, err := s.GetSessionArtifact(t.Context(), tenant, session, a.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("deleted metadata retained: %v", err) + } + } + if err := s.DeleteSession(t.Context(), tenant, session); err != nil { + t.Fatal(err) + } + if count := sourceObjectCount(t, pool); count != before { + t.Fatalf("objects leaked: %d -> %d", before, count) + } +} + +type artifactReadError struct{} + +func (artifactReadError) Read([]byte) (int, error) { return 0, io.ErrUnexpectedEOF } + +func TestSessionArtifactsRejectIncompleteAndUnownedCapture(t *testing.T) { + s, pool := testStore(t) + tenant, session, environment, turn := artifactTurn(t, s, "openai_hosted") + before := sourceObjectCount(t, pool) + valid := artifactArchive(t, map[string][]byte{"outputs/a": []byte("data")}) + for name, body := range map[string]io.Reader{ + "transport-failure-after-valid-tar": io.MultiReader(bytes.NewReader(valid), artifactReadError{}), + "truncated-body": bytes.NewReader(valid[:513]), + "trailing-data": io.MultiReader(bytes.NewReader(valid), bytes.NewReader([]byte("not archive padding"))), + "traversal": bytes.NewReader(artifactArchive(t, map[string][]byte{"outputs/../secret": []byte("no")})), + "private-root": bytes.NewReader(artifactArchive(t, map[string][]byte{"secrets/key": []byte("no")})), + } { + t.Run(name, func(t *testing.T) { + if err := s.StageTurnArtifacts(t.Context(), tenant, session, turn, environment, body); err == nil { + t.Fatal("invalid capture accepted") + } + if count := sourceObjectCount(t, pool); count != before { + t.Fatalf("rollback leaked objects: %d -> %d", before, count) + } + }) + } + for _, ids := range [][4]string{{uuid.NewString(), session, turn, environment}, {tenant, session, turn, uuid.NewString()}} { + if err := s.StageTurnArtifacts(t.Context(), ids[0], ids[1], ids[2], ids[3], artifactReadError{}); !errors.Is(err, ErrNotFound) { + t.Fatalf("unauthorized capture reached reader: %v", err) + } + } + st, ss, se, sr := artifactTurn(t, s, "self_hosted") + if err := s.StageTurnArtifacts(t.Context(), st, ss, sr, se, artifactReadError{}); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("self_hosted publication allowed: %v", err) + } +} + +func TestSessionArtifactsDiscardTerminalPrivateCapture(t *testing.T) { + for _, status := range []string{TurnFailed, TurnCancelled} { + t.Run(status, func(t *testing.T) { + s, pool := testStore(t) + tenant, session, environment, turn := artifactTurn(t, s, "openai_hosted") + before := sourceObjectCount(t, pool) + body := artifactArchive(t, map[string][]byte{"outputs/a": []byte("private")}) + if err := s.StageTurnArtifacts(t.Context(), tenant, session, turn, environment, bytes.NewReader(body)); err != nil { + t.Fatal(err) + } + transition(t, s, tenant, session, turn, TurnInProgress, status) + if count := sourceObjectCount(t, pool); count != before { + t.Fatalf("terminal capture leaked objects: %d -> %d", before, count) + } + if err := s.StageTurnArtifacts(t.Context(), tenant, session, turn, environment, bytes.NewReader(body)); !errors.Is(err, ErrTurnConflict) { + t.Fatalf("late capture accepted: %v", err) + } + if count := sourceObjectCount(t, pool); count != before { + t.Fatalf("late capture leaked objects: %d -> %d", before, count) + } + }) + } +} + +func TestSessionArtifactTransferDoesNotBlockDeletionOrCancellation(t *testing.T) { + for _, operation := range []string{"delete", "cancel"} { + t.Run(operation, func(t *testing.T) { + s, pool := testStore(t) + tenant, session, environment, turn := artifactTurn(t, s, "openai_hosted") + before := sourceObjectCount(t, pool) + reader, writer := io.Pipe() + defer reader.Close() + defer writer.Close() + result := make(chan error, 1) + go func() { result <- s.StageTurnArtifacts(t.Context(), tenant, session, turn, environment, reader) }() + // A complete TAR arrives, but transport has not acknowledged success yet. + if _, err := writer.Write(artifactArchive(t, map[string][]byte{"outputs/a": []byte("partial")})); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + want := ErrNotFound + if operation == "delete" { + if err := s.DeleteSession(ctx, tenant, session); err != nil { + t.Fatalf("transfer blocked deletion: %v", err) + } + } else { + if _, err := s.RequestCancel(ctx, tenant, session, "cancel-capture"); err != nil { + t.Fatalf("transfer blocked cancellation: %v", err) + } + want = ErrTurnConflict + } + writer.Close() + if err := <-result; !errors.Is(err, want) { + t.Fatalf("late publication after %s: %v", operation, err) + } + if count := sourceObjectCount(t, pool); count != before { + t.Fatalf("late capture leaked objects: %d -> %d", before, count) + } + }) + } +} diff --git a/services/agents-api/internal/store/session_deletion.go b/services/agents-api/internal/store/session_deletion.go index 9bfc411e..dfefb5a7 100644 --- a/services/agents-api/internal/store/session_deletion.go +++ b/services/agents-api/internal/store/session_deletion.go @@ -15,6 +15,9 @@ func (s *Store) DeleteSession(ctx context.Context, tenantID, sessionID string) e if err := cancelSessionWork(ctx, q, session); err != nil { return err } + if err := q.DeleteSessionArtifacts(ctx, session); err != nil { + return err + } return q.MarkSessionDeleted(ctx, session) }) } diff --git a/services/agents-api/internal/store/session_events.go b/services/agents-api/internal/store/session_events.go index 23b731d2..37aa4763 100644 --- a/services/agents-api/internal/store/session_events.go +++ b/services/agents-api/internal/store/session_events.go @@ -41,6 +41,9 @@ func recordTurnChange(ctx context.Context, q *sqlc.Queries, row sqlc.Turn, creat return nil } if terminalStatus(row.Status) { + if err := settleTurnArtifacts(ctx, q, row); err != nil { + return err + } unfinished, err := q.FinishSessionItems(ctx, sqlc.FinishSessionItemsParams{SessionID: row.SessionID, TurnID: row.ID}) if err != nil { return err diff --git a/services/agents-api/migrations/000040_session_artifacts.sql b/services/agents-api/migrations/000040_session_artifacts.sql new file mode 100644 index 00000000..21f94f0e --- /dev/null +++ b/services/agents-api/migrations/000040_session_artifacts.sql @@ -0,0 +1,26 @@ +-- +goose Up +CREATE TABLE session_artifacts ( + id uuid PRIMARY KEY, + session_id uuid NOT NULL REFERENCES sessions(id), + turn_id uuid NOT NULL, + environment_id uuid NOT NULL REFERENCES environments(id), + path text NOT NULL CHECK (octet_length(path) BETWEEN 20 AND 4110), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 0 AND 209715200), + body_oid oid NOT NULL UNIQUE, + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + created_at timestamptz, + UNIQUE (turn_id, path), + FOREIGN KEY (session_id, turn_id) REFERENCES turns(session_id, id) +); +CREATE INDEX session_artifacts_page_idx ON session_artifacts (session_id, created_at DESC, id DESC) +WHERE created_at IS NOT NULL; + +-- +goose Down +-- +goose StatementBegin +DO $$ BEGIN + IF EXISTS (SELECT 1 FROM session_artifacts) THEN + RAISE EXCEPTION 'delete session artifacts through the service before downgrade'; + END IF; +END $$; +-- +goose StatementEnd +DROP TABLE session_artifacts; From caa5482163f40fa4203387186eb50372a14b2073 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Fri, 18 Sep 2026 12:37:45 +0800 Subject: [PATCH 3/6] test(agents-api): verify artifacts through pinned SDK and raw HTTP --- .../tests/official_session_artifacts.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 services/agents-api/tests/official_session_artifacts.py diff --git a/services/agents-api/tests/official_session_artifacts.py b/services/agents-api/tests/official_session_artifacts.py new file mode 100644 index 00000000..b86f25af --- /dev/null +++ b/services/agents-api/tests/official_session_artifacts.py @@ -0,0 +1,75 @@ +"""Pinned SDK and raw HTTP checks for known immutable Session output versions.""" + +from openai import NotFoundError + + +def verify_session_artifacts(client, foreign, http, session_id, environment_id, expected): + resource = client.beta.agents.sessions.artifacts + endpoint = str(client.base_url).rstrip("/") + "/agents/sessions/" + session_id + "/artifacts" + headers = {"Authorization": "Bearer " + client.api_key, "OpenAI-Beta": "agents=v1"} + wanted = {(turn, path): data for turn, files in expected.items() for path, data in files.items()} + all_items = list(resource.list(session_id, limit=100, order="asc")) + assert len(all_items) == len(wanted), "Missing or duplicate published output" + assert {(item.turn_id, item.path) for item in all_items} == set(wanted), "Wrong output versions" + fields = {"id", "created_at", "environment_id", "object", "path", "session_id", "size_bytes", "turn_id"} + for item in all_items: + metadata = item.to_dict() + assert set(metadata) == fields, "Wrong public metadata or private storage fields leaked" + assert item.object == "agent.session.artifact" and item.session_id == session_id + assert item.environment_id == environment_id and type(item.created_at) is int and item.created_at > 0 + data = wanted[(item.turn_id, item.path)] + assert item.size_bytes == len(data), "Wrong immutable size" + assert resource.retrieve(item.id, session_id=session_id).to_dict() == metadata + with resource.with_streaming_response.content(item.id, session_id=session_id) as response: + assert response.read() == data, "SDK content differs from completed output" + response = http.get(endpoint + "/" + item.id, headers=headers) + assert response.status_code == 200 and response.json() == metadata + response = http.get(endpoint + "/" + item.id + "/content", headers=headers) + assert response.status_code == 200 and response.content == data + assert response.headers["content-type"].startswith("application/octet-stream") + + ascending_ids = [item.id for item in all_items] + for order in ("asc", "desc"): + wanted_ids = ascending_ids if order == "asc" else list(reversed(ascending_ids)) + sdk = resource.list(session_id, environment_id=environment_id, limit=1, order=order) + assert [item.id for item in sdk] == wanted_ids, "SDK cursor continuation changed order" + params = {"environment_id": environment_id, "limit": 2, "order": order} + seen = [] + for _ in range(len(wanted) + 1): + response = http.get(endpoint, headers=headers, params=params) + assert response.status_code == 200 + page = response.json() + assert isinstance(page["data"], list) and type(page["has_more"]) is bool + assert len(page["data"]) <= 2 + seen.extend(item["id"] for item in page["data"]) + if not page["has_more"]: + break + assert page["data"], "Empty page claims continuation" + params["after"] = page["data"][-1]["id"] + else: + raise AssertionError("Artifact cursor did not terminate") + assert seen == wanted_ids, "Raw HTTP pagination changed order or completeness" + assert [item.id for item in resource.list(session_id)] == list(reversed(ascending_ids)) + assert [item.id for item in resource.list(session_id, after=None, environment_id=None, limit=None)] == list(reversed(ascending_ids)) + + assert http.get(endpoint, headers={"Authorization": headers["Authorization"]}).status_code == 400 + assert http.get(endpoint, headers={"OpenAI-Beta": "agents=v1"}).status_code == 401 + for query in ("limit=0", "limit=101", "order=wrong", "environment_id=a&environment_id=b"): + assert http.get(endpoint + "?" + query, headers=headers).status_code == 400 + assert http.post(endpoint, headers=headers, json={}).status_code == 405 + other = foreign.beta.agents.sessions.artifacts + probes = [lambda: other.list(session_id)] + for item in all_items: + probes.extend(( + lambda item=item: other.retrieve(item.id, session_id=session_id), + lambda item=item: other.content(item.id, session_id=session_id), + lambda item=item: other.delete(item.id, session_id=session_id), + )) + for probe in probes: + try: + probe() + except NotFoundError: + pass + else: + raise AssertionError("Foreign tenant accessed published artifacts") + return all_items From b838f815546944f85533bb03cd823b312e9384ef Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Fri, 18 Sep 2026 12:56:08 +0800 Subject: [PATCH 4/6] fix(agents-api): reserve later inputs while capturing outputs --- CONTRIBUTING.md | 4 ++ .../internal/db/queries/session_artifacts.sql | 4 ++ .../agents-api/internal/db/sqlc/models.go | 23 +++++------ .../internal/db/sqlc/scheduling.sql.go | 3 +- .../internal/db/sqlc/session_artifacts.sql.go | 18 +++++++++ .../internal/db/sqlc/session_events.sql.go | 3 +- .../internal/db/sqlc/turn_reads.sql.go | 3 +- .../agents-api/internal/db/sqlc/turns.sql.go | 12 ++++-- .../internal/execution/artifacts.go | 15 +++++-- .../execution/directory_preparation.go | 2 +- .../internal/store/artifact_lifecycle.go | 39 +++++++++++++++++++ .../internal/store/environment_inputs.go | 4 +- .../store/local_artifact_export_test.go | 5 +++ .../migrations/000040_session_artifacts.sql | 2 + 14 files changed, 112 insertions(+), 25 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 876388d2..1d42502d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -434,6 +434,10 @@ publication ordered with Turn completion, and authorize stored reads independent of Environment availability so published outputs can survive its expiration. Capture bytes into private PostgreSQL large objects without a Session admission lock; after confirmed export, lock and recheck the live Turn before staging metadata. +Before capture, seal native input under that lock using the private capture marker. +Later messages reuse the existing Environment input reservation and await the next +Turn; the public Turn stays in progress until publication settles. Cancellation +retains the existing Turn/reservation semantics. Do not introduce a second queue. Publish metadata in the same transaction as Turn completion. Failed/cancelled Turns discard private objects, and Session deletion removes both private and published copies. Reuse the source-file snapshot reader pattern and common content response; diff --git a/services/agents-api/internal/db/queries/session_artifacts.sql b/services/agents-api/internal/db/queries/session_artifacts.sql index a18beae7..5d69cfa7 100644 --- a/services/agents-api/internal/db/queries/session_artifacts.sql +++ b/services/agents-api/internal/db/queries/session_artifacts.sql @@ -1,3 +1,7 @@ +-- name: BeginTurnArtifactCapture :execrows +UPDATE turns SET artifact_capture_started = true +WHERE session_id = $1 AND id = $2 AND NOT artifact_capture_started; + -- name: StageSessionArtifact :exec INSERT INTO session_artifacts (id, session_id, turn_id, environment_id, path, size_bytes, body_oid, sha256) VALUES ($1, $2, $3, $4, $5, $6, $7, $8); diff --git a/services/agents-api/internal/db/sqlc/models.go b/services/agents-api/internal/db/sqlc/models.go index 6082eb23..c812c18e 100644 --- a/services/agents-api/internal/db/sqlc/models.go +++ b/services/agents-api/internal/db/sqlc/models.go @@ -180,17 +180,18 @@ type SubagentIdentity struct { } type Turn struct { - ID pgtype.UUID `json:"id"` - SessionID pgtype.UUID `json:"session_id"` - Status string `json:"status"` - CreatedAt pgtype.Timestamptz `json:"created_at"` - StartedAt pgtype.Timestamptz `json:"started_at"` - CompletedAt pgtype.Timestamptz `json:"completed_at"` - CancelRequestedAt pgtype.Timestamptz `json:"cancel_requested_at"` - Outcome []byte `json:"outcome"` - EventCount int32 `json:"event_count"` - EventBytes int64 `json:"event_bytes"` - TokenUsage []byte `json:"token_usage"` + ID pgtype.UUID `json:"id"` + SessionID pgtype.UUID `json:"session_id"` + Status string `json:"status"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + StartedAt pgtype.Timestamptz `json:"started_at"` + CompletedAt pgtype.Timestamptz `json:"completed_at"` + CancelRequestedAt pgtype.Timestamptz `json:"cancel_requested_at"` + Outcome []byte `json:"outcome"` + EventCount int32 `json:"event_count"` + EventBytes int64 `json:"event_bytes"` + TokenUsage []byte `json:"token_usage"` + ArtifactCaptureStarted bool `json:"artifact_capture_started"` } type TurnEvent struct { diff --git a/services/agents-api/internal/db/sqlc/scheduling.sql.go b/services/agents-api/internal/db/sqlc/scheduling.sql.go index 7c1f5444..4a748867 100644 --- a/services/agents-api/internal/db/sqlc/scheduling.sql.go +++ b/services/agents-api/internal/db/sqlc/scheduling.sql.go @@ -12,7 +12,7 @@ import ( ) const getLatestSessionTurn = `-- name: GetLatestSessionTurn :one -SELECT id, session_id, status, created_at, started_at, completed_at, cancel_requested_at, outcome, event_count, event_bytes, token_usage FROM turns WHERE session_id = $1 ORDER BY created_at DESC, id DESC LIMIT 1 +SELECT id, session_id, status, created_at, started_at, completed_at, cancel_requested_at, outcome, event_count, event_bytes, token_usage, artifact_capture_started FROM turns WHERE session_id = $1 ORDER BY created_at DESC, id DESC LIMIT 1 ` func (q *Queries) GetLatestSessionTurn(ctx context.Context, sessionID pgtype.UUID) (Turn, error) { @@ -30,6 +30,7 @@ func (q *Queries) GetLatestSessionTurn(ctx context.Context, sessionID pgtype.UUI &i.EventCount, &i.EventBytes, &i.TokenUsage, + &i.ArtifactCaptureStarted, ) return i, err } diff --git a/services/agents-api/internal/db/sqlc/session_artifacts.sql.go b/services/agents-api/internal/db/sqlc/session_artifacts.sql.go index 54e8c107..dbbb94d4 100644 --- a/services/agents-api/internal/db/sqlc/session_artifacts.sql.go +++ b/services/agents-api/internal/db/sqlc/session_artifacts.sql.go @@ -11,6 +11,24 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const beginTurnArtifactCapture = `-- name: BeginTurnArtifactCapture :execrows +UPDATE turns SET artifact_capture_started = true +WHERE session_id = $1 AND id = $2 AND NOT artifact_capture_started +` + +type BeginTurnArtifactCaptureParams struct { + SessionID pgtype.UUID `json:"session_id"` + ID pgtype.UUID `json:"id"` +} + +func (q *Queries) BeginTurnArtifactCapture(ctx context.Context, arg BeginTurnArtifactCaptureParams) (int64, error) { + result, err := q.db.Exec(ctx, beginTurnArtifactCapture, arg.SessionID, arg.ID) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + const deleteSessionArtifact = `-- name: DeleteSessionArtifact :one DELETE FROM session_artifacts a USING sessions s WHERE s.id = a.session_id AND s.tenant_id = $1 AND s.deleted_at IS NULL diff --git a/services/agents-api/internal/db/sqlc/session_events.sql.go b/services/agents-api/internal/db/sqlc/session_events.sql.go index 1151cea7..c1290c50 100644 --- a/services/agents-api/internal/db/sqlc/session_events.sql.go +++ b/services/agents-api/internal/db/sqlc/session_events.sql.go @@ -144,7 +144,7 @@ func (q *Queries) SessionEventCursor(ctx context.Context, arg SessionEventCursor } const sessionEventTurn = `-- name: SessionEventTurn :one -SELECT id, session_id, status, created_at, started_at, completed_at, cancel_requested_at, outcome, event_count, event_bytes, token_usage FROM turns WHERE session_id = $1 AND id = $2 +SELECT id, session_id, status, created_at, started_at, completed_at, cancel_requested_at, outcome, event_count, event_bytes, token_usage, artifact_capture_started FROM turns WHERE session_id = $1 AND id = $2 ` type SessionEventTurnParams struct { @@ -167,6 +167,7 @@ func (q *Queries) SessionEventTurn(ctx context.Context, arg SessionEventTurnPara &i.EventCount, &i.EventBytes, &i.TokenUsage, + &i.ArtifactCaptureStarted, ) return i, err } diff --git a/services/agents-api/internal/db/sqlc/turn_reads.sql.go b/services/agents-api/internal/db/sqlc/turn_reads.sql.go index 745de06d..b2137b46 100644 --- a/services/agents-api/internal/db/sqlc/turn_reads.sql.go +++ b/services/agents-api/internal/db/sqlc/turn_reads.sql.go @@ -12,7 +12,7 @@ import ( ) const listTurns = `-- name: ListTurns :many -SELECT t.id, t.session_id, t.status, t.created_at, t.started_at, t.completed_at, t.cancel_requested_at, t.outcome, t.event_count, t.event_bytes, t.token_usage FROM turns t JOIN sessions s ON s.id = t.session_id +SELECT t.id, t.session_id, t.status, t.created_at, t.started_at, t.completed_at, t.cancel_requested_at, t.outcome, t.event_count, t.event_bytes, t.token_usage, t.artifact_capture_started FROM turns t JOIN sessions s ON s.id = t.session_id WHERE s.tenant_id = $1 AND t.session_id = $2 AND ($3::timestamptz IS NULL OR (NOT $4::boolean AND (t.created_at, t.id) < ($3::timestamptz, $5::uuid)) @@ -62,6 +62,7 @@ func (q *Queries) ListTurns(ctx context.Context, arg ListTurnsParams) ([]Turn, e &i.EventCount, &i.EventBytes, &i.TokenUsage, + &i.ArtifactCaptureStarted, ); err != nil { return nil, err } diff --git a/services/agents-api/internal/db/sqlc/turns.sql.go b/services/agents-api/internal/db/sqlc/turns.sql.go index 79af189d..1c718729 100644 --- a/services/agents-api/internal/db/sqlc/turns.sql.go +++ b/services/agents-api/internal/db/sqlc/turns.sql.go @@ -12,7 +12,7 @@ import ( ) const createTurn = `-- name: CreateTurn :one -INSERT INTO turns(id, session_id) VALUES ($1, $2) RETURNING id, session_id, status, created_at, started_at, completed_at, cancel_requested_at, outcome, event_count, event_bytes, token_usage +INSERT INTO turns(id, session_id) VALUES ($1, $2) RETURNING id, session_id, status, created_at, started_at, completed_at, cancel_requested_at, outcome, event_count, event_bytes, token_usage, artifact_capture_started ` type CreateTurnParams struct { @@ -35,6 +35,7 @@ func (q *Queries) CreateTurn(ctx context.Context, arg CreateTurnParams) (Turn, e &i.EventCount, &i.EventBytes, &i.TokenUsage, + &i.ArtifactCaptureStarted, ) return i, err } @@ -112,7 +113,7 @@ func (q *Queries) FindInputBatch(ctx context.Context, arg FindInputBatchParams) } const getActiveTurn = `-- name: GetActiveTurn :one -SELECT id, session_id, status, created_at, started_at, completed_at, cancel_requested_at, outcome, event_count, event_bytes, token_usage FROM turns WHERE session_id = $1 AND status IN ('queued', 'in_progress', 'waiting') +SELECT id, session_id, status, created_at, started_at, completed_at, cancel_requested_at, outcome, event_count, event_bytes, token_usage, artifact_capture_started FROM turns WHERE session_id = $1 AND status IN ('queued', 'in_progress', 'waiting') ` func (q *Queries) GetActiveTurn(ctx context.Context, sessionID pgtype.UUID) (Turn, error) { @@ -130,12 +131,13 @@ func (q *Queries) GetActiveTurn(ctx context.Context, sessionID pgtype.UUID) (Tur &i.EventCount, &i.EventBytes, &i.TokenUsage, + &i.ArtifactCaptureStarted, ) return i, err } const getTurn = `-- name: GetTurn :one -SELECT t.id, t.session_id, t.status, t.created_at, t.started_at, t.completed_at, t.cancel_requested_at, t.outcome, t.event_count, t.event_bytes, t.token_usage FROM turns t JOIN sessions s ON s.id = t.session_id +SELECT t.id, t.session_id, t.status, t.created_at, t.started_at, t.completed_at, t.cancel_requested_at, t.outcome, t.event_count, t.event_bytes, t.token_usage, t.artifact_capture_started FROM turns t JOIN sessions s ON s.id = t.session_id WHERE s.tenant_id = $1 AND t.session_id = $2 AND t.id = $3 ` @@ -160,6 +162,7 @@ func (q *Queries) GetTurn(ctx context.Context, arg GetTurnParams) (Turn, error) &i.EventCount, &i.EventBytes, &i.TokenUsage, + &i.ArtifactCaptureStarted, ) return i, err } @@ -279,7 +282,7 @@ WHERE id = $3 AND session_id = $4 AND status = $5 AND status IN ('queued', 'in_progress', 'waiting') AND ($1::text <> 'in_progress' OR cancel_requested_at IS NULL) -RETURNING id, session_id, status, created_at, started_at, completed_at, cancel_requested_at, outcome, event_count, event_bytes, token_usage +RETURNING id, session_id, status, created_at, started_at, completed_at, cancel_requested_at, outcome, event_count, event_bytes, token_usage, artifact_capture_started ` type TransitionTurnParams struct { @@ -311,6 +314,7 @@ func (q *Queries) TransitionTurn(ctx context.Context, arg TransitionTurnParams) &i.EventCount, &i.EventBytes, &i.TokenUsage, + &i.ArtifactCaptureStarted, ) return i, err } diff --git a/services/agents-api/internal/execution/artifacts.go b/services/agents-api/internal/execution/artifacts.go index 3a92912d..187f7d4b 100644 --- a/services/agents-api/internal/execution/artifacts.go +++ b/services/agents-api/internal/execution/artifacts.go @@ -2,6 +2,7 @@ package execution import ( "context" + "errors" "io" "time" @@ -16,16 +17,22 @@ func (d *Dispatcher) captureCompletedArtifacts(ctx context.Context, peer *gatewa } owner, cancel := context.WithTimeout(ctx, 180*time.Second) defer cancel() - err := d.withPreparedWorkspace(owner, peer, session, environment, bound, func(ctx context.Context, handle string) error { - return peer.ExportWorkspaceOutputs(ctx, proto.WorkspaceExportPayload{Handle: handle, EnvironmentID: environment.ID}, func(body io.Reader) error { - return d.Store.StageTurnArtifacts(ctx, session.TenantID, session.ID, turnID, environment.ID, body) + err := d.Store.BeginTurnArtifactCapture(owner, session.TenantID, session.ID, turnID, result.AppliedThrough) + if err == nil { + err = d.withPreparedWorkspace(owner, peer, session, environment, bound, func(ctx context.Context, handle string) error { + return peer.ExportWorkspaceOutputs(ctx, proto.WorkspaceExportPayload{Handle: handle, EnvironmentID: environment.ID}, func(body io.Reader) error { + return d.Store.StageTurnArtifacts(ctx, session.TenantID, session.ID, turnID, environment.ID, body) + }) }) - }) + } if err == nil { return result, status } // Do not expose native diagnostics or publish partial output after a failed capture. result.ErrorCode = "artifact_capture_failed" + if errors.Is(err, store.ErrUnappliedInputs) { + result.ErrorCode = "input_not_applied" + } status = store.TurnFailed check, stop := context.WithTimeout(context.Background(), 5*time.Second) defer stop() diff --git a/services/agents-api/internal/execution/directory_preparation.go b/services/agents-api/internal/execution/directory_preparation.go index 97ffd981..c82cee88 100644 --- a/services/agents-api/internal/execution/directory_preparation.go +++ b/services/agents-api/internal/execution/directory_preparation.go @@ -19,7 +19,7 @@ func (d *Dispatcher) readPreparedDirectory(ctx context.Context, peer *gateway.Se return result.err }) if err != nil { - result.err = err + return directoryReadResult{err: err} } return result } diff --git a/services/agents-api/internal/store/artifact_lifecycle.go b/services/agents-api/internal/store/artifact_lifecycle.go index ff031fdf..be0b004f 100644 --- a/services/agents-api/internal/store/artifact_lifecycle.go +++ b/services/agents-api/internal/store/artifact_lifecycle.go @@ -2,10 +2,49 @@ package store import ( "context" + "errors" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" ) +// BeginTurnArtifactCapture separates native completion from bounded output publication. +// Later messages use the existing reservation path instead of the finished executor. +func (s *Store) BeginTurnArtifactCapture(ctx context.Context, tenantID, sessionID, turnID string, appliedThrough int64) error { + lookup, err := turnLookup(tenantID, sessionID, turnID) + if err != nil { + return err + } + if appliedThrough < 0 { + return ErrInvalidInput + } + return s.withSession(ctx, tenantID, sessionID, func(ctx context.Context, q *sqlc.Queries, session pgtype.UUID) error { + turn, err := q.GetTurn(ctx, lookup) + if errors.Is(err, pgx.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return err + } + if turn.Status != TurnInProgress || turn.CancelRequestedAt.Valid { + return ErrTurnConflict + } + pending, err := q.HasUnappliedMessages(ctx, sqlc.HasUnappliedMessagesParams{SessionID: session, TurnID: lookup.ID, Sequence: appliedThrough}) + if err != nil { + return err + } + if pending { + return ErrUnappliedInputs + } + count, err := q.BeginTurnArtifactCapture(ctx, sqlc.BeginTurnArtifactCaptureParams{SessionID: session, ID: lookup.ID}) + if err == nil && count != 1 { + return ErrTurnConflict + } + return err + }) +} + func settleTurnArtifacts(ctx context.Context, q *sqlc.Queries, turn sqlc.Turn) error { if turn.Status == TurnCompleted { return q.PublishTurnArtifacts(ctx, sqlc.PublishTurnArtifactsParams{SessionID: turn.SessionID, TurnID: turn.ID, CreatedAt: turn.CompletedAt}) diff --git a/services/agents-api/internal/store/environment_inputs.go b/services/agents-api/internal/store/environment_inputs.go index 2389cbba..e1edcedb 100644 --- a/services/agents-api/internal/store/environment_inputs.go +++ b/services/agents-api/internal/store/environment_inputs.go @@ -88,7 +88,7 @@ func (s *Store) ReserveEnvironmentInput(ctx context.Context, tenantID, sessionID if err := checkEnvironmentInputGate(ctx, q, session, key, encoded); err != nil { return err } - if _, err := q.GetActiveTurn(ctx, session); err == nil { + if active, err := q.GetActiveTurn(ctx, session); err == nil && !active.ArtifactCaptureStarted { result = EnvironmentInputReservation{SessionID: sessionID, State: EnvironmentInputAdmitted} for position, input := range batch { receipt, err := admitInput(ctx, q, tenantID, session, key, int32(position), input) @@ -98,7 +98,7 @@ func (s *Store) ReserveEnvironmentInput(ctx context.Context, tenantID, sessionID result.Receipts = append(result.Receipts, receipt) } return nil - } else if !errors.Is(err, pgx.ErrNoRows) { + } else if err != nil && !errors.Is(err, pgx.ErrNoRows) { return err } row, err := q.CreateEnvironmentInputReservation(ctx, sqlc.CreateEnvironmentInputReservationParams{ diff --git a/services/agents-api/internal/store/local_artifact_export_test.go b/services/agents-api/internal/store/local_artifact_export_test.go index 50c49ebf..f94f8080 100644 --- a/services/agents-api/internal/store/local_artifact_export_test.go +++ b/services/agents-api/internal/store/local_artifact_export_test.go @@ -3,6 +3,7 @@ package store_test import ( "archive/tar" "bytes" + "encoding/json" "testing" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" @@ -43,6 +44,10 @@ func completeLocalArtifactExport(t *testing.T, h *dispatchHarness, environment s if err != nil || len(page.Artifacts) != 0 { t.Fatal("capture published before native completion", err) } + pending, err := h.s.ReserveEnvironmentInput(t.Context(), h.tenant, h.session.ID, "during-artifact-capture", []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"run after the completed native execution"}`)}}) + if err != nil || pending.State != store.EnvironmentInputPending || len(pending.Receipts) != 0 { + t.Fatalf("input during artifact capture was assigned to the finished executor: %+v %v", pending, err) + } h.write(begin.ID, proto.TypeWorkspaceExportResult, proto.WorkspaceExportResultPayload{Outcome: "completed", Offset: export.Offset}) release := h.read(proto.TypeExecutionRelease) var close proto.ExecutionReleasePayload diff --git a/services/agents-api/migrations/000040_session_artifacts.sql b/services/agents-api/migrations/000040_session_artifacts.sql index 21f94f0e..ac739682 100644 --- a/services/agents-api/migrations/000040_session_artifacts.sql +++ b/services/agents-api/migrations/000040_session_artifacts.sql @@ -1,4 +1,5 @@ -- +goose Up +ALTER TABLE turns ADD COLUMN artifact_capture_started boolean NOT NULL DEFAULT false; CREATE TABLE session_artifacts ( id uuid PRIMARY KEY, session_id uuid NOT NULL REFERENCES sessions(id), @@ -24,3 +25,4 @@ DO $$ BEGIN END $$; -- +goose StatementEnd DROP TABLE session_artifacts; +ALTER TABLE turns DROP COLUMN artifact_capture_started; From adfc4ba96d83b05ba86523bcc1874434a469341f Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Fri, 18 Sep 2026 13:16:00 +0800 Subject: [PATCH 5/6] fix(agents-api): keep directory reads available during output capture --- CONTRIBUTING.md | 4 ++- contracts/agents-api/README.md | 8 +++--- .../execution/environment_directory.go | 9 ++++--- .../store/local_artifact_export_test.go | 25 ++++++++++++++++++- .../store/local_environment_worker_test.go | 4 +-- services/agents-api/internal/store/turns.go | 3 +++ 6 files changed, 42 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1d42502d..e53a8bb7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -436,7 +436,9 @@ Capture bytes into private PostgreSQL large objects without a Session admission lock; after confirmed export, lock and recheck the live Turn before staging metadata. Before capture, seal native input under that lock using the private capture marker. Later messages reuse the existing Environment input reservation and await the next -Turn; the public Turn stays in progress until publication settles. Cancellation +Turn; the public Turn stays in progress until publication settles. Directory reads +during capture use an independent authorized read-only preparation, not the released +native Run; they do not request model credentials or mutate the workspace. Cancellation retains the existing Turn/reservation semantics. Do not introduce a second queue. Publish metadata in the same transaction as Turn completion. Failed/cancelled Turns discard private objects, and Session deletion removes both private and published diff --git a/contracts/agents-api/README.md b/contracts/agents-api/README.md index 660cd6bd..fb460f4a 100644 --- a/contracts/agents-api/README.md +++ b/contracts/agents-api/README.md @@ -57,9 +57,9 @@ dependencies, risk and effort. Parsar cutover and its business Team loop are sep This inventory is based on the pinned Python source, not our generated OpenAPI. It contains 42 distinct HTTP operations in 15 resource classes, excluding async -duplicates, overloads and client-side helpers. Twenty operations currently have -handlers; that count is not a compatibility score. Even those operations implement -only part of the upstream input, configuration and event variants. +duplicates, overloads and client-side helpers. Handlers below are partial coverage, +not a compatibility score. They implement only part of the upstream input, +configuration and event variants. Paths below are SDK resource paths beneath `client.beta.agents`. Method names use the Python SDK. Vault HTTP paths start at `/vaults`, not `/agents/vaults`. @@ -71,7 +71,7 @@ the Python SDK. Vault HTTP paths start at `/vaults`, not `/agents/vaults`. | sessions.events | create, stream | Text/cancel/function-result admission and live events; function-action state snapshots supported | | sessions.turns | retrieve, list | Implemented reads; lifecycle conformance still partial | | sessions.items | list | Partial Item variants | -| sessions.artifacts | retrieve, list, delete, content | Missing | +| sessions.artifacts | retrieve, list, delete, content | Codex/Docker hosted output capture and immutable stored reads/deletion, including after Environment expiry; exact upstream defaults/errors, unchanged-file republishing and cancellation-edge parity remain unverified | | sessions.subagents | retrieve, list | Missing | | sessions.subagents.items | list | Missing | | sessions.subagents.turns | retrieve, list | Missing | diff --git a/services/agents-api/internal/execution/environment_directory.go b/services/agents-api/internal/execution/environment_directory.go index d0473bd0..31719ddf 100644 --- a/services/agents-api/internal/execution/environment_directory.go +++ b/services/agents-api/internal/execution/environment_directory.go @@ -90,16 +90,19 @@ func (w *Worker) runDirectoryRead(owner context.Context, request directoryReadRe return } } + // Capture retains the public Turn after its native Run has been released. + prepare := reserved || session.LastTurn != nil && session.LastTurn.ArtifactCaptureStarted bound, err := w.dispatcher.Store.GetSessionDevice(check, session.TenantID, session.ID) - if err != nil || !environmentDeviceMatches(session, environment, bound, placement) || !w.directoryDeviceReady(bound.ID, session.Engine, placement, reserved) { + if err != nil || !environmentDeviceMatches(session, environment, bound, placement) || !w.directoryDeviceReady(bound.ID, session.Engine, placement, prepare) { return } peer, err := w.dispatcher.Registry.LookupDevice(bound.ID) if err != nil { return } - read := proto.WorkspaceReadPayload{EnvironmentID: environment.ID, RunID: run, Path: request.path, MaxEntries: proto.WorkspaceDirectoryMaxEntries} - if !reserved { + read := proto.WorkspaceReadPayload{EnvironmentID: environment.ID, Path: request.path, MaxEntries: proto.WorkspaceDirectoryMaxEntries} + if !prepare { + read.RunID = run result = readEnvironmentDirectory(owner, peer, read) return } diff --git a/services/agents-api/internal/store/local_artifact_export_test.go b/services/agents-api/internal/store/local_artifact_export_test.go index f94f8080..f259d964 100644 --- a/services/agents-api/internal/store/local_artifact_export_test.go +++ b/services/agents-api/internal/store/local_artifact_export_test.go @@ -7,10 +7,11 @@ import ( "testing" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" ) -func completeLocalArtifactExport(t *testing.T, h *dispatchHarness, environment store.Environment) { +func completeLocalArtifactExport(t *testing.T, h *dispatchHarness, worker *execution.Worker, environment store.Environment) { t.Helper() prepared := h.read(proto.TypeExecutionPrepare) var request proto.ExecutionPreparePayload @@ -44,6 +45,7 @@ func completeLocalArtifactExport(t *testing.T, h *dispatchHarness, environment s if err != nil || len(page.Artifacts) != 0 { t.Fatal("capture published before native completion", err) } + completeCaptureDirectoryRead(t, h, worker, environment) pending, err := h.s.ReserveEnvironmentInput(t.Context(), h.tenant, h.session.ID, "during-artifact-capture", []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"run after the completed native execution"}`)}}) if err != nil || pending.State != store.EnvironmentInputPending || len(pending.Receipts) != 0 { t.Fatalf("input during artifact capture was assigned to the finished executor: %+v %v", pending, err) @@ -56,3 +58,24 @@ func completeLocalArtifactExport(t *testing.T, h *dispatchHarness, environment s } h.write(prepared.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 3, State: "released"}) } + +func completeCaptureDirectoryRead(t *testing.T, h *dispatchHarness, worker *execution.Worker, environment store.Environment) { + t.Helper() + result := startDirectoryRead(t.Context(), worker, environment) + frame := h.read(proto.TypeExecutionPrepare) + var prepare proto.ExecutionPreparePayload + if frame.DecodePayload(&prepare) != nil || !proto.ValidWorkspaceReadPreparation(prepare.Configuration) || prepare.Configuration.LocalEnvironment == nil || prepare.Configuration.LocalEnvironment.ID != environment.ID { + t.Fatal("directory read during capture lost read-only authority") + } + handle := acknowledgePreparation(h, frame.ID) + h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 2, State: "ready"}) + read := h.read(proto.TypeWorkspaceRead) + var request proto.WorkspaceReadPayload + if read.DecodePayload(&request) != nil || request.Handle != handle || request.RunID != "" || request.EnvironmentID != environment.ID { + t.Fatal("directory read during capture used a finished native Run") + } + completeDirectoryRead(t, h, frame.ID, read.ID, false, false) + if got := awaitDirectoryResult(t, result); got.err != nil || len(got.value.Entries) != 1 { + t.Fatal("directory read failed during paused output capture", got.err) + } +} diff --git a/services/agents-api/internal/store/local_environment_worker_test.go b/services/agents-api/internal/store/local_environment_worker_test.go index d00a29da..8f18b182 100644 --- a/services/agents-api/internal/store/local_environment_worker_test.go +++ b/services/agents-api/internal/store/local_environment_worker_test.go @@ -111,7 +111,7 @@ func TestLocalEnvironmentWorkerRejectsGeneralDeviceDespiteCapability(t *testing. } func TestLocalEnvironmentWorkerSchedulesPreparationWithoutRemoteResolver(t *testing.T) { - h, _, environment := localWorker(t, true, true) + h, worker, environment := localWorker(t, true, true) reservation, err := h.s.ReserveEnvironmentInput(t.Context(), h.tenant, h.session.ID, "local-input", []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"first"}`)}}) if err != nil { t.Fatal(err) @@ -141,7 +141,7 @@ func TestLocalEnvironmentWorkerSchedulesPreparationWithoutRemoteResolver(t *test } h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 3, State: "started", RunID: start.RunID}) h.write(start.RunID, proto.TypeDone, proto.DonePayload{Content: "complete", Metadata: map[string]any{proto.DoneMetaAgentSessionID: "local-native-history"}}) - completeLocalArtifactExport(t, h, environment) + completeLocalArtifactExport(t, h, worker, environment) awaitDaemonRemoteCondition(t, t.Context(), 5*time.Second, "local completion", func() bool { turn, err := h.s.GetTurn(t.Context(), h.tenant, h.session.ID, start.RunID) return err == nil && turn.Status == store.TurnCompleted diff --git a/services/agents-api/internal/store/turns.go b/services/agents-api/internal/store/turns.go index 4e1824a7..e301acfa 100644 --- a/services/agents-api/internal/store/turns.go +++ b/services/agents-api/internal/store/turns.go @@ -36,6 +36,8 @@ type Turn struct { CancelRequestedAt time.Time Usage json.RawMessage Outcome json.RawMessage + // ArtifactCaptureStarted is private Runtime coordination, never a wire field. + ArtifactCaptureStarted bool `json:"-"` } type TurnTransition struct { @@ -153,5 +155,6 @@ func turnFromRow(row sqlc.Turn) Turn { ID: uuid.UUID(row.ID.Bytes).String(), SessionID: uuid.UUID(row.SessionID.Bytes).String(), Status: row.Status, CreatedAt: row.CreatedAt.Time, StartedAt: row.StartedAt.Time, CompletedAt: row.CompletedAt.Time, CancelRequestedAt: row.CancelRequestedAt.Time, Outcome: json.RawMessage(row.Outcome), Usage: json.RawMessage(row.TokenUsage), + ArtifactCaptureStarted: row.ArtifactCaptureStarted, } } From 46f6312220177e5cc7320d2e6ab299d4ce1e8072 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Fri, 18 Sep 2026 13:24:42 +0800 Subject: [PATCH 6/6] test(agents-api): isolate process-wide descriptor assertions --- packages/codex-executor/src/directory.rs | 4 ++++ packages/codex-executor/src/directory_tests.rs | 2 -- packages/codex-executor/src/export_tests.rs | 6 ++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/codex-executor/src/directory.rs b/packages/codex-executor/src/directory.rs index 6cbd6373..0d77f898 100644 --- a/packages/codex-executor/src/directory.rs +++ b/packages/codex-executor/src/directory.rs @@ -62,3 +62,7 @@ pub(crate) fn observe(fd: &OwnedFd, limit: usize) -> io::Result { #[cfg(test)] #[path = "directory_tests.rs"] mod tests; + +// Tests counting process-wide descriptors must exclude concurrent fixture I/O. +#[cfg(test)] +pub(crate) static TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); diff --git a/packages/codex-executor/src/directory_tests.rs b/packages/codex-executor/src/directory_tests.rs index 264072d3..39f923dc 100644 --- a/packages/codex-executor/src/directory_tests.rs +++ b/packages/codex-executor/src/directory_tests.rs @@ -5,8 +5,6 @@ use std::os::unix::fs::symlink; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; -static TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - struct Fixture { path: PathBuf, _guard: std::sync::MutexGuard<'static, ()>, diff --git a/packages/codex-executor/src/export_tests.rs b/packages/codex-executor/src/export_tests.rs index 61c8091f..9fd81bfa 100644 --- a/packages/codex-executor/src/export_tests.rs +++ b/packages/codex-executor/src/export_tests.rs @@ -5,6 +5,7 @@ use std::os::unix::{fs::symlink, net::UnixListener}; #[test] fn captures_nested_binary_empty_and_long_names_without_other_workspace_files() { + let _guard = crate::directory::TEST_LOCK.lock().unwrap(); let root = tempfile::tempdir().unwrap(); fs::create_dir_all(root.path().join("outputs/nested")).unwrap(); fs::write(root.path().join("private.txt"), "not an output").unwrap(); @@ -32,6 +33,7 @@ fn captures_nested_binary_empty_and_long_names_without_other_workspace_files() { #[test] fn absent_outputs_is_an_empty_archive() { + let _guard = crate::directory::TEST_LOCK.lock().unwrap(); let root = tempfile::tempdir().unwrap(); let mut bytes = Vec::new(); outputs(root.path(), &mut bytes).unwrap(); @@ -46,6 +48,7 @@ fn absent_outputs_is_an_empty_archive() { #[test] fn rejects_links_and_special_files_without_exposing_their_contents() { + let _guard = crate::directory::TEST_LOCK.lock().unwrap(); for kind in ["root-symlink", "file-symlink", "hardlink", "socket"] { let root = tempfile::tempdir().unwrap(); let other = tempfile::tempdir().unwrap(); @@ -80,6 +83,7 @@ fn rejects_links_and_special_files_without_exposing_their_contents() { #[test] fn enforces_file_and_total_bytes_without_loading_bodies() { + let _guard = crate::directory::TEST_LOCK.lock().unwrap(); for sizes in [ vec![MAX_FILE_BYTES + 1], vec![MAX_FILE_BYTES, MAX_FILE_BYTES, 101 << 20], @@ -98,6 +102,7 @@ fn enforces_file_and_total_bytes_without_loading_bodies() { #[test] fn rejects_a_changed_file_or_directory_during_export() { + let _guard = crate::directory::TEST_LOCK.lock().unwrap(); struct Mutate<'a> { root: &'a Path, directory: bool, @@ -140,6 +145,7 @@ fn rejects_a_changed_file_or_directory_during_export() { #[test] fn rejects_truncated_traversal_and_broken_destination() { + let _guard = crate::directory::TEST_LOCK.lock().unwrap(); let root = tempfile::tempdir().unwrap(); fs::create_dir(root.path().join("outputs")).unwrap(); for index in 0..=MAX_ENTRIES {