Lab fetch: HTTP/1.1, fail fast on a stalled connection, retry network errors

src/agent_runtime/labs.py

@@ -59,7 +59,25 @@
 MAX_STEPS = 20
 MAX_ARGS = 100
 MAX_ARG_LENGTH = 4096
-GIT_TIMEOUT = 300
+GIT_TIMEOUT = 180  # seconds per git command
+
+# Fetching a lab over the network. HTTP/1.1, because HTTP/2 through some proxies and VPNs fails
+# with "Error in the HTTP2 framing layer" (seen 2026-09-19 on macOS, fetching from GitHub);
+# a shallow fetch is no slower over HTTP/1.1. Under 1 KB/s for 30 s counts as stalled, so a dead
+# connection fails in half a minute instead of at the timeout. Network failures are retried;
+# a missing commit or repository is not.
+FETCH_CONFIG = (
+    "-c", "http.version=HTTP/1.1",
+    "-c", "http.lowSpeedLimit=1000",
+    "-c", "http.lowSpeedTime=30",
+)  # fmt: skip
+FETCH_ATTEMPTS = 3
+FETCH_BACKOFF = (2.0, 5.0)
+_PERMANENT_FETCH_ERROR = re.compile(
+    r"not our ref|couldn't find remote ref|repository '[^']*' not found|could not read username"
+    r"|authentication failed",
+    re.IGNORECASE,
+)
 
 
 class LabError(Exception):
@@ -188,6 +206,14 @@ def checkout_dir(lab_key: str, commit: str) -> Path:
     return settings.labs_dir / lab_key / commit[:12]
 
 
+def _command_name(args: list[str]) -> str:
+    """The git subcommand in `args`, skipping leading `-c key=value` options."""
+    i = 0
+    while i < len(args) and args[i] == "-c":
+        i += 2
+    return args[i] if i < len(args) else "git"
+
+
 def _git(args: list[str], cwd: Path | None = None) -> str:
     env = dict(
         os.environ,
@@ -207,12 +233,33 @@ def _git(args: list[str], cwd: Path | None = None) -> str:
     except FileNotFoundError as error:
         raise LabError("git is not installed", 500) from error
     except subprocess.TimeoutExpired as error:
-        raise LabError("git timed out fetching the lab", 504) from error
+        raise LabError(f"git {_command_name(args)} timed out", 504) from error
     if result.returncode != 0:
-        raise LabError(f"git {args[0]} failed: {(result.stderr or result.stdout).strip()}", 502)
+        detail = (result.stderr or result.stdout).strip()
+        raise LabError(f"git {_command_name(args)} failed: {detail}", 502)
     return result.stdout.strip()
 
 
+def _fetch(repo: str, commit: str, cwd: Path) -> None:
+    """`git fetch` exactly `commit`, retrying network failures (see FETCH_CONFIG)."""
+    for attempt in range(1, FETCH_ATTEMPTS + 1):
+        try:
+            fetch = ["fetch", "--quiet", "--depth", "1", "--no-tags", repo, commit]
+            _git([*FETCH_CONFIG, *fetch], cwd)
+            return
+        except LabError as error:
+            if _PERMANENT_FETCH_ERROR.search(str(error)):
+                raise
+            if attempt == FETCH_ATTEMPTS:
+                host = urlsplit(repo).hostname or repo
+                raise LabError(
+                    f"{error} (tried {attempt} times; check that this computer can reach {host}, "
+                    "for example through its proxy or VPN)",
+                    error.status,
+                ) from error
+            time.sleep(FETCH_BACKOFF[attempt - 1])
+
+
 def fetch_commit(repo: str, commit: str, dest: Path) -> Path:
     """Check out exactly `commit` of `repo` into `dest`, reusing a finished checkout."""
     if (dest / ".git").is_dir():
@@ -223,7 +270,7 @@ def fetch_commit(repo: str, commit: str, dest: Path) -> Path:
     shutil.rmtree(partial, ignore_errors=True)
     partial.mkdir(parents=True)
     _git(["init", "--quiet"], cwd=partial)
-    _git(["fetch", "--quiet", "--depth", "1", "--no-tags", repo, commit], cwd=partial)
+    _fetch(repo, commit, partial)
     _git(
         ["-c", "advice.detachedHead=false", "checkout", "--quiet", "--detach", "FETCH_HEAD"],
         cwd=partial,

tests/test_labs.py

@@ -349,3 +349,67 @@ def test_pairing_request_passes_the_scope_to_the_prompt(self):
         assert initiate.call_args.args == (SITE, "actions")
         assert prompt.call_args.args[2] == "actions"
         assert bad.status_code == 422
+
+
+FRAMING_ERROR = "fatal: unable to access 'https://github.com/x/': Error in the HTTP2 framing layer"
+
+
+class TestFetchOverUnreliableNetworks:
+    """Fetching a lab through proxies and VPNs (HTTP/2 framing errors, stalls)."""
+
+    def _completed(self, code: int, stderr: str = "") -> subprocess.CompletedProcess[str]:
+        return subprocess.CompletedProcess([], code, "", stderr)
+
+    def test_fetch_uses_http_1_1_and_gives_up_on_a_stalled_connection(self, lab_settings, tmp_path):
+        calls = []
+
+        def run(args, **kwargs):
+            calls.append(args)
+            return self._completed(0)
+
+        with patch.object(labs.subprocess, "run", side_effect=run):
+            labs._fetch("https://github.com/SciMigo/x", "a" * 40, tmp_path)
+        assert calls[0][:8] == [
+            "git",
+            "-c",
+            "http.version=HTTP/1.1",
+            "-c",
+            "http.lowSpeedLimit=1000",
+            "-c",
+            "http.lowSpeedTime=30",
+            "fetch",
+        ]
+
+    def test_network_errors_are_retried(self, lab_settings, tmp_path):
+        framing = FRAMING_ERROR
+        results = [self._completed(128, framing), self._completed(128, framing), self._completed(0)]
+        with (
+            patch.object(labs.subprocess, "run", side_effect=lambda *a, **k: results.pop(0)),
+            patch.object(labs.time, "sleep") as sleep,
+        ):
+            labs._fetch("https://github.com/SciMigo/x", "a" * 40, tmp_path)
+        assert results == [] and sleep.call_count == 2
+
+    def test_gives_up_after_three_attempts_and_says_what_to_check(self, lab_settings, tmp_path):
+        framing = FRAMING_ERROR
+        with (
+            patch.object(labs.subprocess, "run", return_value=self._completed(128, framing)) as run,
+            patch.object(labs.time, "sleep"),
+            pytest.raises(labs.LabError) as error,
+        ):
+            labs._fetch("https://github.com/SciMigo/x", "a" * 40, tmp_path)
+        assert run.call_count == labs.FETCH_ATTEMPTS
+        message = str(error.value)
+        assert message.startswith("git fetch failed:") and "HTTP2 framing" in message
+        assert "tried 3 times" in message and "github.com" in message and "proxy" in message
+        assert error.value.status == 502
+
+    def test_a_missing_commit_is_not_retried(self, lab_settings, tmp_path):
+        missing = "fatal: remote error: upload-pack: not our ref " + "a" * 40
+        with (
+            patch.object(labs.subprocess, "run", return_value=self._completed(128, missing)) as run,
+            patch.object(labs.time, "sleep") as sleep,
+            pytest.raises(labs.LabError, match="not our ref"),
+        ):
+            labs._fetch("https://github.com/SciMigo/x", "a" * 40, tmp_path)
+        assert run.call_count == 1 and sleep.call_count == 0