Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions src/agent_env/explorer/ui/src/lib/collected-zip-files.smoke.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,36 @@ function main(): void {
assert(byUri['fake://home/c'] === 'elsewhere/c.txt', 'an absolute key elsewhere loses its leading slash');
}

// No member name climbs out of the archive or names a root or a drive, whatever the key holds.
{
const files = collectedZipFiles(
{
'../../x': 'fake://home/x',
'/abs/../../y': 'fake://home/y',
'a/../../../z': 'fake://home/z',
'..\\..\\w.txt': 'fake://home/w',
'C:\\Users\\op\\.ssh\\v': 'fake://home/v',
'C:evil': 'fake://home/e',
'./out/./u.txt': 'fake://home/u',
'..': 'fake://home/dots',
},
'/app',
);
const byUri = Object.fromEntries(files.map(f => [f.objectUrl, f.path]));
assert(byUri['fake://home/x'] === 'x', 'leading .. segments are dropped');
assert(byUri['fake://home/y'] === 'y', 'an absolute key outside base_path climbs no higher than the root');
assert(byUri['fake://home/z'] === 'z', 'a .. past the start is dropped');
assert(byUri['fake://home/w'] === 'w.txt', 'backslashes separate like slashes');
assert(byUri['fake://home/v'] === 'Users/op/.ssh/v', 'a drive prefix is dropped');
assert(byUri['fake://home/e'] === 'evil', 'a drive-relative prefix is dropped');
assert(byUri['fake://home/u'] === 'out/u.txt', '. segments are dropped');
assert(byUri['fake://home/dots'] === 'artifact', 'a key with nothing left is named artifact');
assert(
files.every(f => !f.path.split('/').some(part => part === '..' || part === '') && !/^[A-Za-z]:/.test(f.path)),
'every member name is a clean relative path',
);
}

if (failures > 0) {
console.error(`\n${failures} assertion(s) failed`);
process.exit(1);
Expand Down
18 changes: 13 additions & 5 deletions src/agent_env/explorer/ui/src/lib/collected-zip-files.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,19 @@ export function collectedZipFiles(
return Object.entries(artifacts)
.filter(([, uri]) => typeof uri === 'string' && uri !== '')
.map(([key, uri]) => ({
path: !isAbsoluteArtifactPath(key)
? key
: key.startsWith(`${base}/`)
? key.slice(base.length + 1)
: key.replace(/^[/\\]+/, ''),
path: archiveName(
!isAbsoluteArtifactPath(key) ? key : key.startsWith(`${base}/`) ? key.slice(base.length + 1) : key,
),
objectUrl: uri,
}));
}

/** A key as a member name that stays inside the archive when extracted: an agent names its manifest's entries. */
export function archiveName(key: string): string {
const parts: string[] = [];
for (const part of key.replace(/^[A-Za-z]:/, '').split(/[\\/]+/)) {
if (part === '..') parts.pop();
else if (part !== '' && part !== '.') parts.push(part);
}
return parts.join('/') || 'artifact';
}
55 changes: 40 additions & 15 deletions src/agent_env/task_step/task_steps/collect_artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
Artifact paths come from one of these sources (in priority order):
1. Manifest (`manifest_step_id`): a prior prompt step's declared artifacts from
that step's PromptResponse.structured_output["artifacts"] (logical name -> abs path);
collected files are keyed by logical name.
collected files are keyed by logical name, its separators and dot segments dropped.
2. Seed data: context.metadata["seed"][artifacts_key] — a comma-separated
list of filenames (e.g., "Dockerfile,run.sh,codebase.patch")
3. Static fallback: artifact_paths list in the step definition
Expand All @@ -27,6 +27,8 @@
in /app — without assuming a common directory. base_path only applies to
relative entries and to the no-list enumeration fallback.

Only regular files are read, and a link is not one.

This allows the same step definition to collect different artifacts per seed.

Usage in task JSON:
Expand Down Expand Up @@ -71,8 +73,10 @@
import logging
import os
import json
import posixpath
import re
import shlex
import stat
import tempfile
import time
from typing import ClassVar, Optional
Expand Down Expand Up @@ -352,20 +356,20 @@ def _resolve_items(self, context: TaskStepContext) -> list[tuple[str, str, str]]
manifest = (structured.get("artifacts") if isinstance(structured, dict) else None) or {}
items: list[tuple[str, str, str]] = []
skipped_nonfile: list[str] = []
base = posixpath.join(posixpath.normpath(self.base_path or "/"), "")
keys: set[str] = set()
for name, path in manifest.items():
if not (isinstance(path, str) and path.startswith("/")):
skipped_nonfile.append(name) # literal value (e.g. coverage_state) or relative path
continue
if "*" in path:
continue # unresolved glob (e.g. per-iteration debug logs) — can't fetch a literal *
path = posixpath.normpath(path)
if os.path.basename(path) in self.exclude_basenames:
continue
rel = (
path[len(self.base_path) + 1:]
if path.startswith(self.base_path + "/")
else os.path.basename(path)
)
items.append((name, path, rel))
key = _unused(_manifest_key(name, path), keys)
keys.add(key)
items.append((key, path, path[len(base):] if path.startswith(base) else posixpath.basename(path)))
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
if skipped_nonfile:
logger.debug(
"collect(manifest=%s): skipped non-absolute manifest entries "
Expand Down Expand Up @@ -445,17 +449,21 @@ async def _resolve_live_sandbox(self, provider, sandbox_id: str):
) from err

async def _get_file_size(self, sandbox, container: Optional[str], source_path: str) -> int:
"""Get file size on the agent's filesystem. Returns -1 only if the file is genuinely
absent; a `stat` that fails for any other reason RAISES so a failed size check is a
hard failure rather than a silent 'missing' skip that drops an existing deliverable.
"""Size of the regular file at ``source_path`` on the agent's filesystem. Returns -1 only if the file is
genuinely absent; a `stat` that fails for any other reason RAISES so a failed size check is a hard failure
rather than a silent 'missing' skip that drops an existing deliverable, and so does anything but a regular
file (a link, a device, a FIFO or a directory), which could stream without end.
VM-mode wraps in `docker exec <container>`; container-mode runs `stat` directly."""
args = _exec_args(sandbox, container, ("stat", "-c", "%s", source_path))
args = _exec_args(sandbox, container, ("stat", "-c", "%f:%s", source_path))
exit_code, stdout, stderr = await sandbox.exec_with_output(*args)
if exit_code != 0:
if _looks_absent(stderr):
return -1
raise RuntimeError(f"could not size {source_path} (stat exit {exit_code}): {stderr[:200]}")
return int(stdout.strip())
mode, _, size = stdout.strip().rpartition(":")
if not stat.S_ISREG(int(mode, 16)):
raise RuntimeError(f"{source_path} is not a regular file ({stat.filemode(int(mode, 16))}); refusing to read it")
return int(size)

async def _collect_file(
self, sandbox, container: Optional[str], source_path: str, object_name: str, store, artifact_id: str,
Expand Down Expand Up @@ -558,9 +566,9 @@ async def _controller_file_size(self, deployed_env, path: str) -> int:
as a missing file and silently dropping an existing deliverable. ``wc -c`` is
portable across the Ubuntu and macOS desktop VMs; ``stat``'s flags are not. A
transport exception from ``_controller_call`` propagates unchanged."""
out = await self._controller_call(
deployed_env, "cua_bash", {"script": f"wc -c < {shlex.quote(path)}"}
)
quoted = shlex.quote(path)
script = f'[ ! -e {quoted} ] || [ -f {quoted} ] || {{ echo "not a regular file" >&2; exit 1; }}; wc -c < {quoted}'
out = await self._controller_call(deployed_env, "cua_bash", {"script": script})
# A present file (even 0 bytes) prints its size to stdout; empty stdout means wc failed.
text = _cua_bash_stdout(out).strip()
if text:
Expand Down Expand Up @@ -1050,3 +1058,20 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:
)

return context


def _manifest_key(name: str, path: str) -> str:
"""The agent's name for a manifest entry as a key no reader takes for a path: its separators and dot segments
dropped, else the file's own name. Consumers write files under these keys, a zip of them among them."""
parts = [part for part in re.split(r"[\\/]+", name) if part not in ("", ".", "..")]
return "_".join(parts) or posixpath.basename(path)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved


def _unused(key: str, keys: set[str]) -> str:
"""``key``, or the first ``key (n)`` not in ``keys``: two manifest names can clean to the same key."""
n = 1
unused = key
while unused in keys:
n += 1
unused = f"{key} ({n})"
return unused
9 changes: 7 additions & 2 deletions tst/integration/computer/desktop_example_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow]

RECORDING = Path(__file__).parent / "recording"
# How far apart, summed over R, G and B, a screenshot's pixel in the clicked cell is from the one before the click once
# the app has repainted the cell.
CELL_REPAINTED = 48


def test_the_desktop_examples_computer_passes_the_probe_at_full(state):
Expand Down Expand Up @@ -68,8 +71,10 @@ def test_a_scripted_drive_is_recorded_on_the_clock_its_screenshots_carry(state):
before = _when(shots[n - 1]["captured_at"]).timestamp()
_, unclicked = min(frames, key=lambda frame: abs(frame[0] - before))
changed = next(t for t, image in frames if t > before and corner(image) != corner(unclicked))
# The cell changes after the screenshot before its click, by the one after it, both on the computer's clock.
assert changed <= _when(shots[n]["captured_at"]).timestamp() + interval
# The app repaints the cell some time after the click, so the click's own screenshot can still show it unclicked.
# The cell changes after the screenshot before its click, by the first screenshot that shows it, on the same clock.
shown = next(s for s in shots[n:] if sum(abs(a - b) for a, b in zip(s["cell"], shots[n - 1]["cell"])) > CELL_REPAINTED)
assert changed <= _when(shown["captured_at"]).timestamp() + interval, (shown["step"], changed - before)


@skip_without_model_endpoint()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""A scripted drive of the Fixture Desk, the conformance fixture started on the computer as the probe starts it, while
the desktop records: the clicks, typing and scroll a person would make, each answered with a screenshot. A test finds each moment in the video by its send time, its screenshot's capture time
and thumbnail, and the clicked grid cell's box."""
and thumbnail, its pixel in the clicked grid cell, and that cell's box."""
import asyncio
import base64
import io
Expand All @@ -17,9 +17,13 @@
for name in ("fixture.py", "desk.py")}


def _thumb(result):
def _image(result):
data = next(c.data for c in result.content if c.type == "image")
return Image.open(io.BytesIO(base64.b64decode(data))).convert("L").resize(THUMB).tobytes().hex()
return Image.open(io.BytesIO(base64.b64decode(data)))


def _thumb(result):
return _image(result).convert("L").resize(THUMB).tobytes().hex()


async def verify(mcp_url, env_url):
Expand Down Expand Up @@ -54,6 +58,7 @@ def centre(box):
if result.isError:
return [{"id": "drove", "result": False, "description": f"{tool}: {result.content[0].text}"}]
shots.append({"step": tool, "args": args, "sent_at": sent_at,
"captured_at": result.structuredContent["frame"]["captured_at"], "thumb": _thumb(result)})
"captured_at": result.structuredContent["frame"]["captured_at"], "thumb": _thumb(result),
"cell": list(_image(result).convert("RGB").getpixel((clicked[0] + 8, clicked[1] + 8)))})
await asyncio.sleep(0.7)
return [{"id": "drove", "result": True, "description": json.dumps({"shots": shots, "clicked_cell": clicked})}]
2 changes: 1 addition & 1 deletion tst/unit/store/fakes.py
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ async def exec_script(self, script: str) -> str:

async def exec_with_output(self, *args):
if "stat" in args:
return 0, str(len(COLLECTED[args[-1]])), ""
return 0, f"81a4:{len(COLLECTED[args[-1]])}", ""
return 0, "ok", ""

async def exec(self, *args):
Expand Down
83 changes: 83 additions & 0 deletions tst/unit/task_step/test_collect_artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import asyncio
import json
import os
import subprocess
import tempfile
import threading
from unittest.mock import AsyncMock, MagicMock, patch
Expand Down Expand Up @@ -516,6 +517,26 @@ def test_get_file_size_other_error_raises(self):
except RuntimeError as e:
assert "could not size" in str(e)

@pytest.mark.parametrize("mode, shown", [("a1ff", "lrwxrwxrwx"), ("21b6", "crw-rw-rw-"), ("11a4", "prw-r--r--"),
("41ed", "drwxr-xr-x"), ("c1ed", "srwxr-xr-x")],
ids=["link", "device", "fifo", "directory", "socket"])
def test_get_file_size_refuses_anything_but_a_regular_file(self, mode, shown):
"""By the raw mode bits stat prints, which no locale translates, as it does a file type's name."""
step = CollectArtifactsTaskStep(id="c", version=None, agent_name="solver")
sandbox = MagicMock()
sandbox.mode = "container"
sandbox.exec_with_output = AsyncMock(return_value=(0, f"{mode}:0\n", ""))
with pytest.raises(RuntimeError, match=f"not a regular file \\({shown}\\)"):
_run(step._get_file_size(sandbox, None, "/app/artifact/report.pdf"))
assert sandbox.exec_with_output.await_args.args[:3] == ("stat", "-c", "%f:%s")

def test_get_file_size_sizes_a_regular_file_even_when_empty(self):
step = CollectArtifactsTaskStep(id="c", version=None, agent_name="solver")
sandbox = MagicMock()
sandbox.mode = "container"
sandbox.exec_with_output = AsyncMock(side_effect=[(0, "81a4:0\n", ""), (0, "8180:4096\n", "")])
assert [_run(step._get_file_size(sandbox, None, "/x")) for _ in range(2)] == [0, 4096]

def test_unreachable_sandbox_is_clear_error(self):
"""A re-run targeting an expired/unreachable sandbox fails with a clear message."""
step = CollectArtifactsTaskStep(
Expand Down Expand Up @@ -599,6 +620,43 @@ def test_manifest_mode_keys_by_logical_name_and_filters(self):
assert items["gold_patch"] == ("/app/workspace/patches/gold_patch.diff", "patches/gold_patch.diff")
assert items["dockerfile"] == ("/app/workspace/Dockerfile", "Dockerfile")

def test_manifest_entries_outside_base_path_are_named_by_their_file(self):
step = CollectArtifactsTaskStep(id="collect", version=None, base_path="/app/workspace", manifest_step_id="m9")
ctx = _manifest_ctx({"kept": "/app/workspace/out/./report.md", "note": "/app/handoff/REJECTED.md"})
assert step._resolve_items(ctx) == [
("kept", "/app/workspace/out/report.md", "out/report.md"),
("note", "/app/handoff/REJECTED.md", "REJECTED.md"),
]

def test_manifest_names_become_keys_with_no_separators_or_dot_segments(self):
step = CollectArtifactsTaskStep(id="collect", version=None, base_path="/app/workspace", manifest_step_id="m9")
ctx = _manifest_ctx({
"../../.bashrc": "/app/workspace/payload",
"logs/run.json": "/app/workspace/logs/run.json",
"..\\..\\evil": "/app/workspace/evil",
"..": "/app/workspace/dots.txt",
})
assert [k for k, _, _ in step._resolve_items(ctx)] == [".bashrc", "logs_run.json", "evil", "dots.txt"]

def test_manifest_names_that_clean_to_one_key_each_keep_a_key_of_their_own(self):
step = CollectArtifactsTaskStep(id="collect", version=None, base_path="/app/workspace", manifest_step_id="m9")
ctx = _manifest_ctx({
"logs/run.json": "/app/workspace/a.json",
"logs_run.json": "/app/workspace/b.json",
"../logs/run.json": "/app/workspace/c.json",
})
assert [(k, src) for k, src, _ in step._resolve_items(ctx)] == [
("logs_run.json", "/app/workspace/a.json"),
("logs_run.json (2)", "/app/workspace/b.json"),
("logs_run.json (3)", "/app/workspace/c.json"),
]

@pytest.mark.parametrize("base_path", ["/app/./workspace", "/app/workspace/", "/app//workspace"])
def test_manifest_entries_are_matched_against_base_path_normalized(self, base_path):
step = CollectArtifactsTaskStep(id="collect", version=None, base_path=base_path, manifest_step_id="m9")
ctx = _manifest_ctx({"report": "/app/workspace/out/report.md"})
assert step._resolve_items(ctx) == [("report", "/app/workspace/out/report.md", "out/report.md")]

def test_enumeration_drops_excluded_basenames(self):
step = CollectArtifactsTaskStep(
id="collect", version=None, base_path="/app/workspace",
Expand Down Expand Up @@ -936,6 +994,31 @@ async def call(deployed_env, tool_name, arguments):
except RuntimeError as e:
assert "refusing to upload a partial artifact" in str(e)

@pytest.mark.parametrize("make, size", [
(lambda p, tmp: p.write_bytes(b"hello"), 5),
(lambda p, tmp: None, -1),
(lambda p, tmp: p.symlink_to(tmp / "gone"), -1),
(lambda p, tmp: p.symlink_to("/dev/zero"), None),
(lambda p, tmp: os.mkfifo(p), None),
(lambda p, tmp: p.mkdir(), None),
], ids=["regular", "absent", "dangling-link", "link-to-dev-zero", "fifo", "directory"])
def test_the_size_probe_sizes_only_a_regular_file(self, tmp_path, make, size):
"""The probe's own script, run by a real shell as cua_bash runs it on the VM."""
target = tmp_path / "deliverable"
make(target, tmp_path)
step = CollectArtifactsTaskStep(id="c", version=None, env_id="ubuntu-cua")

async def call(deployed_env, tool_name, arguments):
done = subprocess.run(["bash", "-c", arguments["script"]], capture_output=True, text=True, timeout=10)
return json.dumps({"output": done.stdout, "error": done.stderr, "returncode": done.returncode})

with patch.object(step, "_controller_call", side_effect=call):
if size is None:
with pytest.raises(RuntimeError, match="not a regular file"):
_run(step._controller_file_size(_cua_record(), str(target)))
else:
assert _run(step._controller_file_size(_cua_record(), str(target))) == size

def test_absent_file_raises_filenotfound(self):
"""A missing file (wc errors with 'No such file') is reported absent so the caller
skips it leniently."""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ async def exec_with_output(self, *args):
return 0, "bundle\n", ""
return 0, "".join(f"{p}\n" for p in FILES), ""
if "stat" in args:
return 0, str(len(next(iter(FILES.values())))), ""
return 0, f"81a4:{len(next(iter(FILES.values())))}", ""
return 0, "", ""

async def exec(self, *args):
Expand Down
Loading
Loading