Retry the full deploy check once after a lost connection

frontend/HOSTED_RUNTIME.md

@@ -72,7 +72,13 @@ order:
 4. Verify the unaliased frontend metadata, then execute a real 2024 national threshold
    calculation followed by a California income-tax calculation. Promotion requires the
    reviewed package and data identities, verified H5 bytes, numeric results, and the
-   expected national-to-state cache transition.
+   expected national-to-state cache transition. If the connection is refused, closes,
+   resets or times out before a complete response arrives, the check waits until the
+   backend answers a metadata request (at most 860 seconds after the first request),
+   then sends the calculation once more; the retried response must meet the same
+   requirements. A DNS or TLS-certificate failure, an HTTP error, or a response that
+   fails them is never retried, so a slow calculation still fails: the proxy answers
+   HTTP 504 after waiting 780 seconds.
 5. Promote the Vercel deployment and execute a California calculation through both
    `calibration-diagnostics.vercel.app` and the mounted `microcosm.institute` route.
 

frontend/scripts/deployment_smoke.py

@@ -16,6 +16,32 @@
 RELEASE_PATH = Path(__file__).with_name("hosted_release.json")
 NATIONAL_VARIABLE = "spm_unit_spm_threshold"
 STATE_VARIABLE = "ca_income_tax"
+# Socket timeout for one request. The Vercel route allows 800 s and its proxy
+# answers 504 after waiting 780 s for Modal, so a slow backend produces an HTTP
+# error with a body before this timeout.
+REQUEST_TIMEOUT_SECONDS = 790
+# The backend runs one request at a time and finishes a calculation whose
+# client has gone, so a retried full check waits until the backend answers a
+# metadata request. backend/modal_app.py stops a call after 800 s; the other
+# 60 s allow for container start. Counted from the first full-check request.
+BACKEND_FREE_DEADLINE_SECONDS = 860
+BACKEND_PROBE_PAUSE_SECONDS = 10
+# Failures that leave no complete HTTP response: the connection was refused,
+# closed, reset or aborted (RemoteDisconnected is a ConnectionResetError), or
+# it timed out. urllib wraps such a failure while connecting or sending in a
+# URLError. A URLError for a DNS, TLS-certificate or URL problem is not one,
+# and HTTPError, a URLError that carries a response, is handled first.
+_CONNECTION_FAILURES = (ConnectionError, TimeoutError)
+
+
+class ConnectionFailedError(RuntimeError):
+    """The connection was refused, closed, reset or timed out without a response."""
+
+
+def _is_connection_failure(error: Exception | None) -> bool:
+    if isinstance(error, URLError):
+        return isinstance(error.reason, _CONNECTION_FAILURES)
+    return isinstance(error, _CONNECTION_FAILURES)
 
 
 def _record(value: Any, name: str) -> dict[str, Any]:
@@ -130,6 +156,7 @@ def _request_json(
     headers: dict[str, str],
     *,
     attempts: int,
+    timeout: float = REQUEST_TIMEOUT_SECONDS,
 ) -> dict[str, Any]:
     target = f"{url}?{urlencode(query)}"
     last_error: Exception | None = None
@@ -139,7 +166,7 @@ def _request_json(
                 target,
                 headers={"Cache-Control": "no-cache", **headers},
             )
-            with urlopen(request, timeout=790) as response:
+            with urlopen(request, timeout=timeout) as response:
                 if response.status != 200:
                     raise ValueError(f"Deployment returned HTTP {response.status}.")
                 content_type = response.headers.get("Content-Type", "")
@@ -149,10 +176,20 @@ def _request_json(
         except HTTPError as exc:
             detail = exc.read(1_000).decode("utf-8", errors="replace")
             last_error = RuntimeError(f"Deployment returned HTTP {exc.code}: {detail}")
-        except (URLError, TimeoutError, ValueError, json.JSONDecodeError) as exc:
+        except (
+            URLError,
+            *_CONNECTION_FAILURES,
+            ValueError,
+            json.JSONDecodeError,
+        ) as exc:
             last_error = exc
         if attempt + 1 < attempts:
             time.sleep(5 * (attempt + 1))
+    if _is_connection_failure(last_error):
+        raise ConnectionFailedError(
+            "Deployment check failed: the connection ended before a complete "
+            f"response arrived ({type(last_error).__name__}: {last_error})."
+        ) from last_error
     raise RuntimeError(f"Deployment check failed: {last_error}") from last_error
 
 
@@ -177,6 +214,84 @@ def _request_and_validate(
     raise RuntimeError(f"Deployment validation failed: {last_error}") from last_error
 
 
+def _validate_body(
+    body: dict[str, Any], validate: Callable[[dict[str, Any]], None]
+) -> None:
+    try:
+        validate(body)
+    except (TypeError, ValueError) as exc:
+        raise RuntimeError(f"Deployment validation failed: {exc}") from exc
+
+
+def _wait_until_backend_is_free(
+    url: str,
+    headers: dict[str, str],
+    validate_metadata_body: Callable[[dict[str, Any]], None],
+    deadline: float,
+) -> None:
+    """Return once the backend answers a valid metadata request.
+
+    The backend serves one request at a time, so this answer arrives only after
+    the calculation whose connection was lost has finished. Only another lost
+    connection is retried; any HTTP error or invalid metadata fails at once.
+    """
+    last_probe_failure: Exception | None = None
+    while (remaining := deadline - time.monotonic()) > 0:
+        try:
+            body = _request_json(
+                url,
+                [("metadata", "1")],
+                headers,
+                attempts=1,
+                timeout=min(REQUEST_TIMEOUT_SECONDS, remaining),
+            )
+        except ConnectionFailedError as probe_failure:
+            last_probe_failure = probe_failure
+            print(f"Backend metadata request: {probe_failure}", flush=True)
+            pause = min(BACKEND_PROBE_PAUSE_SECONDS, deadline - time.monotonic())
+            time.sleep(max(0.0, pause))
+            continue
+        _validate_body(body, validate_metadata_body)
+        return
+    raise RuntimeError(
+        "The backend was still busy or unreachable "
+        f"{BACKEND_FREE_DEADLINE_SECONDS} s after the full check was sent, so it "
+        f"was not retried. Last metadata request: {last_probe_failure}"
+    )
+
+
+def _run_full_check(
+    url: str,
+    query: list[tuple[str, str]],
+    headers: dict[str, str],
+    validate_calculation_body: Callable[[dict[str, Any]], None],
+    validate_metadata_body: Callable[[dict[str, Any]], None],
+) -> None:
+    """Run the national and state calculation, retrying once after a lost connection.
+
+    Each response, first or retried, must pass the same validation on its own.
+    A response that fails validation and an HTTP error are never retried.
+    """
+    deadline = time.monotonic() + BACKEND_FREE_DEADLINE_SECONDS
+    try:
+        body = _request_json(url, query, headers, attempts=1)
+    except ConnectionFailedError as first_failure:
+        print(
+            f"::warning::{first_failure} Waiting for the backend to finish that "
+            "calculation, then sending the full check once more.",
+            flush=True,
+        )
+        _wait_until_backend_is_free(url, headers, validate_metadata_body, deadline)
+        print("The backend is free; sending the full check once more.", flush=True)
+        try:
+            body = _request_json(url, query, headers, attempts=1)
+        except RuntimeError as retry_failure:
+            raise RuntimeError(
+                f"The retried full check failed: {retry_failure}"
+            ) from retry_failure
+    _validate_body(body, validate_calculation_body)
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--url", required=True)
@@ -217,18 +332,27 @@ def main() -> int:
                 ("release", expected["release_id"]),
             ]
         )
-        _request_and_validate(
-            args.url,
-            query,
-            headers,
-            lambda body: validate_calculation(
+
+        def validate_calculation_body(body: dict[str, Any]) -> None:
+            validate_calculation(
                 body,
                 args.source_commit,
                 variables,
                 require_scope_switch=args.check == "full",
-            ),
-            attempts=1 if args.check == "full" else 6,
-        )
+            )
+
+        if args.check == "full":
+            _run_full_check(
+                args.url,
+                query,
+                headers,
+                validate_calculation_body,
+                lambda body: validate_metadata(body, args.source_commit),
+            )
+        else:
+            _request_and_validate(
+                args.url, query, headers, validate_calculation_body, attempts=6
+            )
     print(f"Validated {args.check} response from {args.url}")
     return 0