diff --git a/src/agent_runtime/labs.py b/src/agent_runtime/labs.py index a51937d..6833a72 100644 --- a/src/agent_runtime/labs.py +++ b/src/agent_runtime/labs.py @@ -59,7 +59,25 @@ MAX_STEPS = 20 MAX_ARGS = 100 MAX_ARG_LENGTH = 4096 -GIT_TIMEOUT = 300 +GIT_TIMEOUT = 180 # seconds per git command + +# Fetching a lab over the network. HTTP/1.1, because HTTP/2 through some proxies and VPNs fails +# with "Error in the HTTP2 framing layer" (seen 2026-09-19 on macOS, fetching from GitHub); +# a shallow fetch is no slower over HTTP/1.1. Under 1 KB/s for 30 s counts as stalled, so a dead +# connection fails in half a minute instead of at the timeout. Network failures are retried; +# a missing commit or repository is not. +FETCH_CONFIG = ( + "-c", "http.version=HTTP/1.1", + "-c", "http.lowSpeedLimit=1000", + "-c", "http.lowSpeedTime=30", +) # fmt: skip +FETCH_ATTEMPTS = 3 +FETCH_BACKOFF = (2.0, 5.0) +_PERMANENT_FETCH_ERROR = re.compile( + r"not our ref|couldn't find remote ref|repository '[^']*' not found|could not read username" + r"|authentication failed", + re.IGNORECASE, +) class LabError(Exception): @@ -188,6 +206,14 @@ def checkout_dir(lab_key: str, commit: str) -> Path: return settings.labs_dir / lab_key / commit[:12] +def _command_name(args: list[str]) -> str: + """The git subcommand in `args`, skipping leading `-c key=value` options.""" + i = 0 + while i < len(args) and args[i] == "-c": + i += 2 + return args[i] if i < len(args) else "git" + + def _git(args: list[str], cwd: Path | None = None) -> str: env = dict( os.environ, @@ -207,12 +233,33 @@ def _git(args: list[str], cwd: Path | None = None) -> str: except FileNotFoundError as error: raise LabError("git is not installed", 500) from error except subprocess.TimeoutExpired as error: - raise LabError("git timed out fetching the lab", 504) from error + raise LabError(f"git {_command_name(args)} timed out", 504) from error if result.returncode != 0: - raise LabError(f"git {args[0]} failed: {(result.stderr or result.stdout).strip()}", 502) + detail = (result.stderr or result.stdout).strip() + raise LabError(f"git {_command_name(args)} failed: {detail}", 502) return result.stdout.strip() +def _fetch(repo: str, commit: str, cwd: Path) -> None: + """`git fetch` exactly `commit`, retrying network failures (see FETCH_CONFIG).""" + for attempt in range(1, FETCH_ATTEMPTS + 1): + try: + fetch = ["fetch", "--quiet", "--depth", "1", "--no-tags", repo, commit] + _git([*FETCH_CONFIG, *fetch], cwd) + return + except LabError as error: + if _PERMANENT_FETCH_ERROR.search(str(error)): + raise + if attempt == FETCH_ATTEMPTS: + host = urlsplit(repo).hostname or repo + raise LabError( + f"{error} (tried {attempt} times; check that this computer can reach {host}, " + "for example through its proxy or VPN)", + error.status, + ) from error + time.sleep(FETCH_BACKOFF[attempt - 1]) + + def fetch_commit(repo: str, commit: str, dest: Path) -> Path: """Check out exactly `commit` of `repo` into `dest`, reusing a finished checkout.""" if (dest / ".git").is_dir(): @@ -223,7 +270,7 @@ def fetch_commit(repo: str, commit: str, dest: Path) -> Path: shutil.rmtree(partial, ignore_errors=True) partial.mkdir(parents=True) _git(["init", "--quiet"], cwd=partial) - _git(["fetch", "--quiet", "--depth", "1", "--no-tags", repo, commit], cwd=partial) + _fetch(repo, commit, partial) _git( ["-c", "advice.detachedHead=false", "checkout", "--quiet", "--detach", "FETCH_HEAD"], cwd=partial, diff --git a/tests/test_labs.py b/tests/test_labs.py index dc4bb6e..99a7dfc 100644 --- a/tests/test_labs.py +++ b/tests/test_labs.py @@ -349,3 +349,67 @@ def test_pairing_request_passes_the_scope_to_the_prompt(self): assert initiate.call_args.args == (SITE, "actions") assert prompt.call_args.args[2] == "actions" assert bad.status_code == 422 + + +FRAMING_ERROR = "fatal: unable to access 'https://github.com/x/': Error in the HTTP2 framing layer" + + +class TestFetchOverUnreliableNetworks: + """Fetching a lab through proxies and VPNs (HTTP/2 framing errors, stalls).""" + + def _completed(self, code: int, stderr: str = "") -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess([], code, "", stderr) + + def test_fetch_uses_http_1_1_and_gives_up_on_a_stalled_connection(self, lab_settings, tmp_path): + calls = [] + + def run(args, **kwargs): + calls.append(args) + return self._completed(0) + + with patch.object(labs.subprocess, "run", side_effect=run): + labs._fetch("https://github.com/SciMigo/x", "a" * 40, tmp_path) + assert calls[0][:8] == [ + "git", + "-c", + "http.version=HTTP/1.1", + "-c", + "http.lowSpeedLimit=1000", + "-c", + "http.lowSpeedTime=30", + "fetch", + ] + + def test_network_errors_are_retried(self, lab_settings, tmp_path): + framing = FRAMING_ERROR + results = [self._completed(128, framing), self._completed(128, framing), self._completed(0)] + with ( + patch.object(labs.subprocess, "run", side_effect=lambda *a, **k: results.pop(0)), + patch.object(labs.time, "sleep") as sleep, + ): + labs._fetch("https://github.com/SciMigo/x", "a" * 40, tmp_path) + assert results == [] and sleep.call_count == 2 + + def test_gives_up_after_three_attempts_and_says_what_to_check(self, lab_settings, tmp_path): + framing = FRAMING_ERROR + with ( + patch.object(labs.subprocess, "run", return_value=self._completed(128, framing)) as run, + patch.object(labs.time, "sleep"), + pytest.raises(labs.LabError) as error, + ): + labs._fetch("https://github.com/SciMigo/x", "a" * 40, tmp_path) + assert run.call_count == labs.FETCH_ATTEMPTS + message = str(error.value) + assert message.startswith("git fetch failed:") and "HTTP2 framing" in message + assert "tried 3 times" in message and "github.com" in message and "proxy" in message + assert error.value.status == 502 + + def test_a_missing_commit_is_not_retried(self, lab_settings, tmp_path): + missing = "fatal: remote error: upload-pack: not our ref " + "a" * 40 + with ( + patch.object(labs.subprocess, "run", return_value=self._completed(128, missing)) as run, + patch.object(labs.time, "sleep") as sleep, + pytest.raises(labs.LabError, match="not our ref"), + ): + labs._fetch("https://github.com/SciMigo/x", "a" * 40, tmp_path) + assert run.call_count == 1 and sleep.call_count == 0