fix(security-tracker-stats): retry 5xx and default cache to temp root

plugins/magpie-security/skills/tracker-stats-dashboard/SKILL.md

@@ -18,7 +18,7 @@ when_to_use: |
 capability: capability:stats
 surface_hash: sha256:779340f8eb985eeb
 license: Apache-2.0
-measured_tokens: 4591
+measured_tokens: 4609
 ---
 
 <!-- SPDX-License-Identifier: Apache-2.0
@@ -207,7 +207,8 @@ If the adopter passes nothing, surface the resolved output path and cache state
 2. **Check cache freshness.** Inspect
    `<cache>/issues.json` mtime, where `<cache>` is the
    `tracker_stats_cache` value from the step 1 config (else
-   `${TRACKER_STATS_CACHE:-/tmp/tracker-stats-cache}`, the fetch scripts' default).
+   `${TRACKER_STATS_CACHE:-<platform temp dir>/tracker-stats-cache}`, the
+   fetch scripts' default — `$TMPDIR` on POSIX, `%TEMP%` on Windows).
    Step 3 passes the same `<cache>` so the check and the fetch agree.
    If older than 24 h, propose a fresh fetch; if missing or
    the user passed `clear-cache`, do a fresh fetch unconditionally.

plugins/magpie-setup/templates/security-tracker-stats.md

@@ -60,7 +60,10 @@ review).
 ## Cache directory
 
 ```yaml
-tracker_stats_cache: /tmp/tracker-stats-cache
+# Left unset, the fetch scripts default to
+# `<platform temp dir>/tracker-stats-cache` ($TMPDIR on POSIX, %TEMP% on
+# Windows) — the one location a sandboxed run can always write.
+# tracker_stats_cache: /path/to/cache
 ```
 
 Where the fetch scripts persist their cache. Safe to delete (forces a

tools/security-tracker-stats-dashboard/README.md

@@ -57,6 +57,7 @@ tools/security-tracker-stats-dashboard/
 ├── README.md             (this file)
 ├── default-config.yaml   (config schema + adopter-overridable defaults)
 ├── render.py             (renders cached data to HTML; reads config)
+├── gh_retry.py           (runs gh, retrying transient failures; used by the fetch scripts)
 ├── fetch_issues.py       (gh issue list, incl. body + closedByPRs -> issues.json)
 ├── fetch_roster.py       (gh api collaborators -> roster.txt)
 ├── fetch_bodies.py       (body + closedByPRs from issues.json -> issue_extra.json)
@@ -76,8 +77,8 @@ Env knobs (all optional):
 | Var | Default | Notes |
 |---|---|---|
 | `TRACKER_STATS_REPO` | *(e.g. `airflow-s/airflow-s`)* | `<tracker>` repo slug |
-| `TRACKER_STATS_OUT` | `/tmp/airflow_s_monthly.html` | output HTML path |
-| `TRACKER_STATS_CACHE` | `/tmp/tracker-stats-cache` | fetch cache dir |
+| `TRACKER_STATS_OUT` | `$TMPDIR/airflow_s_monthly.html` | output HTML path |
+| `TRACKER_STATS_CACHE` | `$TMPDIR/tracker-stats-cache` | fetch cache dir |
 | `TRACKER_STATS_CONFIG` | *(unset)* | path to a YAML overlay file |
 | `TRACKER_STATS_BUCKETS` | *(from config: `monthly`)* | `monthly` or `quarterly` |
 | `TRACKER_STATS_START` | *(from config: `null`)* | `YYYY-MM` or `YYYY-Qn` |

tools/security-tracker-stats-dashboard/fetch_bodies.py

@@ -30,6 +30,7 @@
 import json
 import os
 import subprocess
+import tempfile
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
 EXTRA_FIELDS = ("body", "closedByPullRequestsReferences")
@@ -85,7 +86,9 @@ def fetch(repo, n):
 
 
 def main():
-    root = os.environ.get("TRACKER_STATS_CACHE", "/tmp/tracker-stats-cache")
+    # Default under the platform temp root, not a hardcoded `/tmp`: a sandboxed run
+    # is granted its own scratch directory and cannot write `/tmp`. See fetch_issues.py.
+    root = os.environ.get("TRACKER_STATS_CACHE") or os.path.join(tempfile.gettempdir(), "tracker-stats-cache")
     repo = os.environ.get("TRACKER_STATS_REPO", "airflow-s/airflow-s")
     out = f"{root}/issue_extra.json"
 

tools/security-tracker-stats-dashboard/fetch_events.py

@@ -24,8 +24,11 @@
 import json
 import os
 import subprocess
+import tempfile
 
-ROOT = os.environ.get("TRACKER_STATS_CACHE", "/tmp/tracker-stats-cache")
+# Default under the platform temp root, not a hardcoded `/tmp`: a sandboxed run is
+# granted its own scratch directory and cannot write `/tmp`. See fetch_issues.py.
+ROOT = os.environ.get("TRACKER_STATS_CACHE") or os.path.join(tempfile.gettempdir(), "tracker-stats-cache")
 REPO = os.environ.get("TRACKER_STATS_REPO", "airflow-s/airflow-s")
 EVENTS_DIR = f"{ROOT}/events"
 

tools/security-tracker-stats-dashboard/fetch_issues.py

@@ -21,49 +21,56 @@
 
 import json
 import os
-import subprocess
+import tempfile
 
-ROOT = os.environ.get("TRACKER_STATS_CACHE", "/tmp/tracker-stats-cache")
-REPO = os.environ.get("TRACKER_STATS_REPO", "airflow-s/airflow-s")
+from gh_retry import run_gh
 
-os.makedirs(ROOT, exist_ok=True)
+# The cache defaults under the platform temp root rather than a hardcoded `/tmp`:
+# a sandboxed run is typically granted its own scratch directory and cannot write
+# `/tmp`, so `/tmp/tracker-stats-cache` made the first fetch step fail outright.
+# `tempfile.gettempdir()` honours `$TMPDIR` (POSIX) and `%TEMP%` (Windows).
+ROOT = os.environ.get("TRACKER_STATS_CACHE") or os.path.join(tempfile.gettempdir(), "tracker-stats-cache")
+REPO = os.environ.get("TRACKER_STATS_REPO", "airflow-s/airflow-s")
 
-print(f"Fetching issue list from {REPO} (state=all, limit 1000) ...")
-r = subprocess.run(
-    [
-        "gh",
-        "issue",
-        "list",
-        "--repo",
-        REPO,
-        "--state",
-        "all",
-        "--limit",
-        "1000",
-        "--json",
-        # `updatedAt` is consumed by fetch_events.py, which refuses to trust a cached
-        # events file older than the issue's last update. Dropping it here silently
-        # disables that guard, because the check falls back to "no timestamp -> trust
-        # the cache" and every relabelled issue keeps its stale event history.
-        # `body` and `closedByPullRequestsReferences` ride on this one paginated call
-        # so fetch_bodies.py can build issue_extra.json without a `gh issue view`
-        # per issue.
-        "number,title,state,stateReason,createdAt,closedAt,updatedAt,labels,comments,body,closedByPullRequestsReferences",
-    ],
-    capture_output=True,
-    text=True,
-    timeout=300,
-)
-if r.returncode != 0:
-    raise SystemExit(f"gh failed: {r.stderr}")
 
-issues = json.loads(r.stdout)
-with open(f"{ROOT}/issues.json", "w") as f:
-    json.dump(issues, f)
+def main() -> None:
+    os.makedirs(ROOT, exist_ok=True)
 
-print(f"Wrote {len(issues)} issues to {ROOT}/issues.json")
-if len(issues) >= 1000:
-    print(
-        "WARNING: the issue list hit the 1000-issue cap; older issues are missing "
-        "and every count in the dashboard is a floor. Raise --limit in fetch_issues.py."
+    print(f"Fetching issue list from {REPO} (state=all, limit 1000) ...")
+    r = run_gh(
+        [
+            "gh",
+            "issue",
+            "list",
+            "--repo",
+            REPO,
+            "--state",
+            "all",
+            "--limit",
+            "1000",
+            "--json",
+            # `updatedAt` is consumed by fetch_events.py, which refuses to trust a cached
+            # events file older than the issue's last update. Dropping it here silently
+            # disables that guard, because the check falls back to "no timestamp -> trust
+            # the cache" and every relabelled issue keeps its stale event history.
+            # `body` and `closedByPullRequestsReferences` ride on this one paginated call
+            # so fetch_bodies.py can build issue_extra.json without a `gh issue view`
+            # per issue.
+            "number,title,state,stateReason,createdAt,closedAt,updatedAt,labels,comments,body,closedByPullRequestsReferences",
+        ]
     )
+
+    issues = json.loads(r.stdout)
+    with open(f"{ROOT}/issues.json", "w") as f:
+        json.dump(issues, f)
+
+    print(f"Wrote {len(issues)} issues to {ROOT}/issues.json")
+    if len(issues) >= 1000:
+        print(
+            "WARNING: the issue list hit the 1000-issue cap; older issues are missing "
+            "and every count in the dashboard is a floor. Raise --limit in fetch_issues.py."
+        )
+
+
+if __name__ == "__main__":
+    main()

tools/security-tracker-stats-dashboard/fetch_prs.py

@@ -28,9 +28,12 @@
 import os
 import re
 import subprocess
+import tempfile
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
-ROOT = os.environ.get("TRACKER_STATS_CACHE", "/tmp/tracker-stats-cache")
+# Default under the platform temp root, not a hardcoded `/tmp`: a sandboxed run is
+# granted its own scratch directory and cannot write `/tmp`. See fetch_issues.py.
+ROOT = os.environ.get("TRACKER_STATS_CACHE") or os.path.join(tempfile.gettempdir(), "tracker-stats-cache")
 UPSTREAM = os.environ.get("TRACKER_STATS_UPSTREAM_REPO", "apache/airflow")
 if UPSTREAM in ("", "none", "null"):
     print("TRACKER_STATS_UPSTREAM_REPO is empty/none - skipping PR fetch.")

tools/security-tracker-stats-dashboard/fetch_roster.py

@@ -20,21 +20,21 @@
 """Dump the security-team roster (tracker repo's collaborators) to <cache>/roster.txt."""
 
 import os
-import subprocess
+import tempfile
 
-ROOT = os.environ.get("TRACKER_STATS_CACHE", "/tmp/tracker-stats-cache")
+from gh_retry import run_gh
+
+# Default under the platform temp root, not a hardcoded `/tmp`: a sandboxed run is
+# granted its own scratch directory and cannot write `/tmp`. See fetch_issues.py.
+ROOT = os.environ.get("TRACKER_STATS_CACHE") or os.path.join(tempfile.gettempdir(), "tracker-stats-cache")
 REPO = os.environ.get("TRACKER_STATS_REPO", "airflow-s/airflow-s")
 
 os.makedirs(ROOT, exist_ok=True)
 
-r = subprocess.run(
+r = run_gh(
     ["gh", "api", f"repos/{REPO}/collaborators", "--jq", ".[].login", "--paginate"],
-    capture_output=True,
-    text=True,
     timeout=60,
 )
-if r.returncode != 0:
-    raise SystemExit(f"gh failed: {r.stderr}")
 
 logins = [ln.strip() for ln in r.stdout.splitlines() if ln.strip()]
 with open(f"{ROOT}/roster.txt", "w") as f:

tools/security-tracker-stats-dashboard/gh_retry.py

@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Run `gh` with retries on transient failures; shared by the fetch scripts."""
+
+import re
+import subprocess
+import sys
+import time
+
+# `gh` reports a server-side hiccup as a non-zero exit with the status in stderr.
+# A single transient `HTTP 502` on the first call used to abort the entire
+# dashboard run, so retry the failures that are worth retrying and fail fast on
+# everything else (auth errors, a bad repo slug, an unknown flag).
+_TRANSIENT_FAILURE = re.compile(
+    r"HTTP\s+5\d\d"
+    r"|Bad Gateway"
+    r"|Service Unavailable"
+    r"|Gateway Time-?out"
+    r"|Internal Server Error"
+    r"|connection reset"
+    r"|timed out",
+    re.IGNORECASE,
+)
+
+ATTEMPTS = 3
+BASE_DELAY_SECONDS = 2.0
+
+
+def is_transient_failure(stderr: str) -> bool:
+    """True when a failed `gh` call looks like a retryable server-side error."""
+    return bool(_TRANSIENT_FAILURE.search(stderr or ""))
+
+
+def run_gh(
+    args: list[str],
+    *,
+    attempts: int = ATTEMPTS,
+    base_delay: float = BASE_DELAY_SECONDS,
+    timeout: int = 300,
+) -> subprocess.CompletedProcess:
+    """Run `gh`, retrying transient failures with exponential backoff.
+
+    Raises `SystemExit` with the last stderr once the attempts are exhausted, or
+    immediately for a failure that retrying cannot fix.
+    """
+    for attempt in range(1, attempts + 1):
+        try:
+            r = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
+        except subprocess.TimeoutExpired:
+            # A call that hangs past the timeout is a server-side stall, so it is
+            # retried like a 504 rather than aborting the run.
+            stderr = f"gh timed out after {timeout}s"
+        else:
+            if r.returncode == 0:
+                return r
+            stderr = r.stderr
+            if not is_transient_failure(stderr):
+                raise SystemExit(f"gh failed: {stderr}")
+        if attempt == attempts:
+            raise SystemExit(f"gh failed: {stderr}")
+        delay = base_delay * 2 ** (attempt - 1)
+        print(
+            f"gh failed (attempt {attempt}/{attempts}), retrying in {delay:g}s: "
+            f"{(stderr or '').strip()[:200]}",
+            file=sys.stderr,
+        )
+        time.sleep(delay)
+    raise AssertionError("unreachable: run_gh returns or raises on every attempt")

tools/security-tracker-stats-dashboard/render.py

@@ -19,7 +19,7 @@
 
 """
 Regenerate a tracker-stats dashboard. Reads cached issues+events+PR data
-from `$TRACKER_STATS_CACHE` (default `/tmp/tracker-stats-cache`) and writes
+from `$TRACKER_STATS_CACHE` (default `<tempdir>/tracker-stats-cache`) and writes
 a self-contained HTML page to `$TRACKER_STATS_OUT`.
 
 Configuration is loaded from `scripts/default-config.yaml`, optionally
@@ -52,6 +52,7 @@
 import re
 import statistics
 import sys
+import tempfile
 from collections import defaultdict
 
 # Make the src/ package importable when render.py is run as a standalone script.
@@ -104,8 +105,13 @@ def yaml_load(text):
 
 # --- Config loading -------------------------------------------------
 
-ROOT = os.environ.get("TRACKER_STATS_CACHE", "/tmp/tracker-stats-cache")
-OUT_PATH = os.environ.get("TRACKER_STATS_OUT", "/tmp/airflow_s_monthly.html")
+# Both defaults sit under the platform temp root, not a hardcoded `/tmp`: a
+# sandboxed run is granted its own scratch directory and can write neither.
+# See fetch_issues.py.
+ROOT = os.environ.get("TRACKER_STATS_CACHE") or os.path.join(tempfile.gettempdir(), "tracker-stats-cache")
+OUT_PATH = os.environ.get("TRACKER_STATS_OUT") or os.path.join(
+    tempfile.gettempdir(), "airflow_s_monthly.html"
+)
 HERE = os.path.dirname(os.path.abspath(__file__))
 DEFAULT_CONFIG_PATH = os.path.join(HERE, "default-config.yaml")
 

tools/security-tracker-stats-dashboard/run.sh

@@ -4,8 +4,8 @@
 # Usage: run.sh [output-path]
 #
 # Env overrides:
-#   TRACKER_STATS_CACHE          (default: /tmp/tracker-stats-cache)
-#   TRACKER_STATS_OUT            (default: /tmp/airflow_s_monthly.html - or arg $1)
+#   TRACKER_STATS_CACHE          (default: $TMPDIR/tracker-stats-cache)
+#   TRACKER_STATS_OUT            (default: $TMPDIR/airflow_s_monthly.html - or arg $1)
 #   TRACKER_STATS_REPO           tracker repo (default: airflow-s/airflow-s)
 #   TRACKER_STATS_BUCKETS        monthly | quarterly (overlay)
 #   TRACKER_STATS_START          "YYYY-MM" / "YYYY-Qn" (overlay)
@@ -55,4 +55,5 @@ echo "-> fetch_prs"
 echo "-> render"
 "${PY_CMD[@]}" "$HERE/render.py"
 
-echo "done: ${TRACKER_STATS_OUT:-/tmp/airflow_s_monthly.html}"
+# render.py prints the path it wrote ("Wrote <path>"), so it is not guessed here.
+echo "done"