feat!: move all images ref to public

.github/workflows/lint.yml

@@ -7,9 +7,10 @@ permissions:
   contents: read
 
 jobs:
-  validate:
-    name: validate
+  lint:
+    name: lint
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v7
 
@@ -18,7 +19,7 @@ jobs:
           node-version: 24
           cache: npm
 
-      - name: Install schema checker
+      - name: Install dependencies
         run: npm ci
 
       - name: Validate model schemas
@@ -27,41 +28,17 @@ jobs:
       - name: Test the catalog library
         run: npm run test:lib
 
-  validate-helm:
-    name: validate-helm
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    steps:
-      - uses: actions/checkout@v7
-        with:
-          fetch-depth: 0
-          # Compare the PR head, not GitHub's synthetic merge commit.
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - uses: actions/setup-node@v7
-        with:
-          node-version: 24
-          cache: npm
-
-      - name: Install validator dependencies
-        run: npm ci
-
       - name: Test Helm validator
         run: npm run test:helm
 
       - name: Install pinned Helm and kubeconform
         run: npm run helm:install
 
-      - name: Validate changed model variants against latest stable charts
-        env:
-          BASE_REF: ${{ github.event.pull_request.base.sha }}
-        run: npm run validate:helm -- --base "$BASE_REF" --chart-mode latest
+      - name: Validate every model variant against latest stable charts
+        run: npm run validate:helm
 
-      - name: Save Helm validation logs and manifests
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: helm-validation
-          path: artifacts/helm-validation/
-          if-no-files-found: ignore
-          retention-days: 14
+      # A catalog is public, so every image must be pullable by anyone. The
+      # runner is the outside world: no login, no VPN. A pull request also
+      # holds each pinned digest to what its tag resolves to now.
+      - name: Check every image is publicly pullable
+        run: npm run validate:images -- --match-digest

.github/workflows/pages.yml

@@ -34,6 +34,12 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
+      - name: Install pinned Helm and kubeconform
+        run: npm run helm:install
+
+      - name: validate
+        run: npm run validate
+
       # index.json is built here, from models/, and checked against the one live
       # on the site: a published version must not change (hack/build-site.js).
       - name: Build Pages site and catalog
@@ -43,7 +49,7 @@ jobs:
 
       - name: Upload Pages artifact
         if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
-        uses: actions/upload-pages-artifact@v4
+        uses: actions/upload-pages-artifact@v5
         with:
           path: site
 

README.md

@@ -67,6 +67,7 @@ models/<name>/
    ```sh
    npm run validate:schema
    npm run helm:install && npm run validate:helm -- models/my-model/my-model-1.0.0.yaml
+   npm run validate:images -- models/my-model/my-model-1.0.0.yaml
    PUBLISHED_INDEX=none npm run build:site   # open site/index.html
    ```
 
@@ -83,6 +84,7 @@ What a variant needs to know:
 | `requires.gpus` | per pod; `nodes` + `topology: lws` for multi-node, with `values.lws.size` equal to `nodes` |
 | `requires.vendor` | `nvidia` (default), `ascend`, `cambricon`, `hygon` or `amd`; one vendor per variant |
 | `requires.gpuProduct` | canonical card names, any one matches: the `nvidia.com/gpu.product` value for NVIDIA (`NVIDIA-H100-80GB-HBM3`), `<Vendor>-<model>` otherwise (`Ascend-910B3`). swiss maps names to each cluster's node labels |
+| `image` | on a public registry, pullable without a login |
 | `chart.version` | exact (`0.8.0`) or a [semver range](https://github.com/Masterminds/semver#checking-version-constraints), quoted (`">=0.8.0"`, `"^0.8.0"`) |
 | `values` | chart values. Site-specific keys (namespace, host paths, registry, route ConfigMaps) are refused by the schema |
 
@@ -102,9 +104,8 @@ tuning:
     report: my-model-h100-report.html
 ```
 
-A report must be a complete HTML document whose embedded JSON parses.
-`validate:schema` fails on a missing, empty or invalid one; the site skips
-linking it.
+A report must be a complete HTML document whose embedded JSON parses. `validate:schema` fails on a missing, empty or invalid one; the site skips
+linking it. The images it records must be the two variants' own (registry, repository and tag), which `validate:images` checks.
 
 ## Development
 
@@ -113,17 +114,23 @@ linking it.
 | `npm run validate:schema` | schemas, tuning references, report HTML (pre-commit and CI) |
 | `npm run test:lib` | tests for `hack/lib` |
 | `npm run helm:install` | pinned Helm and kubeconform, once |
-| `npm run validate:helm -- <file>` | render a version's variants against the charts (`--chart-mode declared` for what a deploy uses) |
+| `npm run validate:helm` | render every variant against the charts (`--chart-mode declared` for what a deploy uses) |
+| `npm run validate:images` | every image is publicly pullable without a login, by tag and by any pinned `digest`, and every perf report measured its variants' builds<br> a tag that has moved off its pinned digest is a warning; `--match-digest` makes it an error |
+| `npm run validate` | `validate:schema`, `validate:helm` and `validate:images` |
 | `npm run build:site` | the site and `index.json` into `site/` |
 | `npm test` | `hack/validate.sh`: naming and layout checks (needs `yq`) |
 | `./hack/serve.sh` | rebuild `index.json` and serve the tree |
 
+`validate:helm` and `validate:images` check every version file; pass files, or
+`--base REF` for what a branch changes, to narrow them.
+
 Needs Node.js 24.
 
-- **CI.** `lints` runs on pull requests: `validate:schema`, `test:lib`, and
-  `validate-helm` on changed version files against the latest stable charts
-  ([Helm validation](schema/crds/README.md)). `pages` builds the site on pull
-  requests and deploys it on push to `master`.
+- **CI.** `lints` runs on pull requests: `validate:schema`, `test:lib`,
+  `test:helm`, then `validate:helm` against the latest stable charts
+  ([Helm validation](schema/crds/README.md)) and `validate:images --match-digest`,
+  both on every version file. `pages` builds the site on pull requests and
+  deploys it on push to `master`.
 - **`index.json` is generated, not committed.** It is a pure function of
   `models/` and `catalog.yaml`, with no timestamp, so the same tree gives the
   same catalog ref in swiss.

hack/lib/helm-validation.js

@@ -35,7 +35,7 @@ function git(root, args) {
 function selectFiles(root, {all = false, base = "origin/HEAD", files = []} = {}) {
   const models = path.join(root, "models");
   const versions = modelNames(models).flatMap((name) => versionFiles(name, models));
-  if (all) return {files: versions, added: [], reason: "--all"};
+  if (all) return {files: versions, added: [], reason: "every version file"};
   if (files.length) {
     const selected = [...new Set(files.map((file) => path.relative(root, path.resolve(root, file)).split(path.sep).join("/")))].sort();
     if (selected.some((file) => !versions.includes(file))) throw new Error("explicit files must be existing model version YAMLs under models/");

hack/lib/images.js

@@ -0,0 +1,206 @@
+"use strict";
+// Whether an image the catalog names can be pulled by anyone: no login, from
+// the registry it names. A catalog is public; an image only one site can pull
+// is a variant only that site can deploy.
+
+const crypto = require("crypto");
+const dns = require("dns").promises;
+const net = require("net");
+const path = require("path");
+const { command } = require("./helm-validation");
+const { modelNames, versionFiles } = require("./catalog");
+const { jsonBlocks } = require("./report");
+
+const ACCEPT = [
+  "application/vnd.oci.image.index.v1+json",
+  "application/vnd.docker.distribution.manifest.list.v2+json",
+  "application/vnd.oci.image.manifest.v1+json",
+  "application/vnd.docker.distribution.manifest.v2+json",
+].join(", ");
+
+// A repository the way containerd reads it: no registry host (or docker.io)
+// means Docker Hub, and a bare name there is an official image under library/.
+function parseRepository(repository) {
+  const parts = repository.split("/");
+  const first = parts[0];
+  if (parts.length > 1 && (first.includes(".") || first.includes(":") || first === "localhost")) {
+    if (first !== "docker.io" && first !== "index.docker.io") return { registry: first, host: first, name: parts.slice(1).join("/") };
+    parts.shift();
+  }
+  return { registry: "docker.io", host: "registry-1.docker.io", name: parts.length === 1 ? `library/${parts[0]}` : parts.join("/") };
+}
+
+// RFC 1918, loopback, link-local and unique-local: nobody outside reaches them.
+function isPrivateAddress(ip) {
+  if (net.isIPv4(ip)) {
+    const [a, b] = ip.split(".").map(Number);
+    return a === 10 || a === 127 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || (a === 169 && b === 254);
+  }
+  const v6 = ip.toLowerCase();
+  return v6 === "::1" || /^f[cd]/.test(v6) || /^fe[89ab]/.test(v6);
+}
+
+// The private address a registry host resolves to, when it resolves to
+// nothing else; null for a host the internet can reach (or that does not
+// resolve, which the request itself reports). A proxy or VPN that reaches it
+// from here does not make it public.
+async function privateNetwork(host, lookup = dns.lookup) {
+  const name = host.replace(/:\d+$/, "").replace(/^\[|\]$/g, "");
+  let addresses;
+  try {
+    addresses = net.isIP(name) ? [name] : (await lookup(name, { all: true })).map((a) => a.address);
+  } catch {
+    return null;
+  }
+  return addresses.length && addresses.every(isPrivateAddress) ? addresses[0] : null;
+}
+
+function bearerChallenge(header) {
+  if (!/^\s*bearer\s/i.test(header || "")) return null;
+  const params = {};
+  for (const [, k, v] of header.matchAll(/(\w+)="([^"]*)"/g)) params[k.toLowerCase()] = v;
+  return params.realm ? params : null;
+}
+
+// The manifest digest ref (a tag or digest) names in repository, pulled
+// anonymously, as { digest }; or why it cannot be pulled, as { problem }.
+// HEAD first: Docker Hub does not count it against pulls.
+async function resolveManifest(repository, ref, { fetch = globalThis.fetch, lookup, tokens = new Map(), attempts = 3 } = {}) {
+  const { host, name } = parseRepository(repository);
+  const internal = await privateNetwork(host, lookup);
+  if (internal) return { problem: `${host} is on a private network (${internal})` };
+  const url = `https://${host}/v2/${name}/manifests/${ref}`;
+  const request = async (method) => {
+    const token = tokens.get(`${host}/${name}`);
+    return fetch(url, { method, headers: { Accept: ACCEPT, ...(token ? { Authorization: `Bearer ${token}` } : {}) } });
+  };
+  let lastError;
+  for (let i = 0; i < attempts; i++) {
+    try {
+      let r = await request("HEAD");
+      if (r.status === 405) r = await request("GET");
+      if (r.status === 401 && !tokens.has(`${host}/${name}`)) {
+        const challenge = bearerChallenge(r.headers.get("www-authenticate"));
+        if (!challenge) return { problem: "the registry requires a login" };
+        const q = new URLSearchParams({ scope: `repository:${name}:pull` });
+        if (challenge.service) q.set("service", challenge.service);
+        const t = await fetch(`${challenge.realm}?${q}`);
+        const body = t.ok ? await t.json() : {};
+        if (!(body.token || body.access_token)) return { problem: "the registry gives no anonymous pull token" };
+        tokens.set(`${host}/${name}`, body.token || body.access_token);
+        r = await request("HEAD");
+        if (r.status === 405) r = await request("GET");
+      }
+      if (r.ok) {
+        // Registries name the digest in a header; for one that does not, it is
+        // the digest of the manifest's bytes.
+        let digest = r.headers.get("docker-content-digest");
+        if (!digest) {
+          const body = await request("GET");
+          if (!body.ok) throw new Error(`HTTP ${body.status}`);
+          digest = `sha256:${crypto.createHash("sha256").update(Buffer.from(await body.arrayBuffer())).digest("hex")}`;
+        }
+        return { digest };
+      }
+      if (r.status === 401 || r.status === 403) return { problem: "not pullable without a login (private, or no such repository)" };
+      if (r.status === 404) return { problem: "no such manifest" };
+      if (r.status === 429 || r.status >= 500) throw new Error(`HTTP ${r.status}`);
+      return { problem: `HTTP ${r.status}` };
+    } catch (err) {
+      lastError = err;
+      await new Promise((resolve) => setTimeout(resolve, 500 * 2 ** i));
+    }
+  }
+  const reason = lastError && lastError.cause ? lastError.cause.code || lastError.cause.message : lastError && lastError.message;
+  return { problem: `could not reach ${host}: ${reason}` };
+}
+
+// How an image block fares pulled anonymously, as { level, message }. "error"
+// when its tag or pinned digest cannot be pulled; "warning" when the tag has
+// moved off a pinned digest that is still pullable; else "ok". With
+// matchDigest a moved tag is an error: a runtime pulls the digest and ignores
+// the tag, so the two name different builds. Tags move, so only a change
+// being made can be held to that.
+async function checkImage(image, { matchDigest = false, ...opts } = {}) {
+  const ref = `${image.repository}:${image.tag}`;
+  const tag = await resolveManifest(image.repository, image.tag, opts);
+  if (tag.problem) return { level: "error", message: `${ref}: ${tag.problem}` };
+  if (!image.digest) return { level: "ok", message: `${ref}: pullable, resolves to ${tag.digest}` };
+  const pinned = `${ref}@${image.digest}`;
+  if (tag.digest === image.digest) return { level: "ok", message: `${pinned}: pullable, tag matches the pinned digest` };
+  const moved = `the tag resolves to ${tag.digest} now, not the pinned digest`;
+  if (matchDigest) return { level: "error", message: `${pinned}: ${moved}` };
+  const digest = await resolveManifest(image.repository, image.digest, opts);
+  if (digest.problem) return { level: "error", message: `${pinned}: ${digest.problem}` };
+  return { level: "warning", message: `${pinned}: ${moved}; both are pullable` };
+}
+
+// The data block an AutoTune report renders from: its page mounts with
+// AutotuneReport.mountStandalone("autotune-report", "autotune-report-data").
+const REPORT_DATA = "autotune-report-data";
+
+// The images a perf report's benchmark ran, from the block it renders from:
+// the baseline's, and each tuning attempt's, which the page shows as its Image
+// row. Empty sets when it records none.
+function reportImages(html) {
+  const out = { baseline: new Set(), attempts: new Set() };
+  let data;
+  try {
+    data = JSON.parse(jsonBlocks(html).find((block) => block.id === REPORT_DATA).text);
+  } catch {
+    return out;
+  }
+  const image = (run) => run && run.launch && run.launch.config && run.launch.config.image;
+  for (const r of data.reports || []) {
+    const c = r && r.comparison;
+    if (!c) continue;
+    if (image(c.baseline)) out.baseline.add(image(c.baseline));
+    for (const a of c.attempts || []) if (image(a)) out.attempts.add(image(a));
+  }
+  return out;
+}
+
+// An image reference in full, the way containerd reads it, digest aside:
+// registry, repository and tag, so nginx and docker.io/library/nginx:latest are
+// one image, and the same tag on another registry or namespace is another.
+function imageRef(ref) {
+  const name = String(ref).split("@")[0];
+  const slash = name.lastIndexOf("/");
+  const colon = name.lastIndexOf(":");
+  const { registry, name: repository } = parseRepository(colon > slash ? name.slice(0, colon) : name);
+  return `${registry}/${repository}:${colon > slash ? name.slice(colon + 1) : "latest"}`;
+}
+
+// Why a tuned pair's report did not measure the images its two variants run.
+// The tuned build need only be among the attempts: a tuning run may try more
+// than one, and its best overall can be the baseline.
+function reportMismatches(html, { baseline, optimized }) {
+  const ran = reportImages(html);
+  const out = [];
+  for (const [role, want, got] of [["baseline", baseline, ran.baseline], ["tuned", optimized, ran.attempts]]) {
+    if (!got.size) out.push(`records no ${role} image`);
+    else if (![...got].some((x) => imageRef(x) === imageRef(want))) {
+      out.push(`the ${role} variant runs ${want}, but the report measured ${[...got].join(", ")}`);
+    }
+  }
+  return out;
+}
+
+// Version files a branch adds or changes since its merge-base with base, and
+// every version of a model whose metadata or perf report changed. Deleted
+// files have nothing left to check.
+function changedVersionFiles(root, base) {
+  const git = (args) => {
+    const result = command("git", args, { cwd: root });
+    if (!result.ok) throw new Error(result.log);
+    return result.stdout;
+  };
+  const mergeBase = git(["merge-base", base, "HEAD"]).trim();
+  const changed = git(["diff", "--name-only", "-z", "--diff-filter=AMR", mergeBase, "HEAD"]).split("\0").filter(Boolean);
+  const touched = new Set(changed.filter((f) => /^models\/[^/]+\/(metadata\.yaml|[^/]+\.html)$/.test(f)).map((f) => f.split("/")[1]));
+  const models = path.join(root, "models");
+  const versions = modelNames(models).flatMap((name) => versionFiles(name, models));
+  return versions.filter((file) => changed.includes(file) || touched.has(file.split("/")[1]));
+}
+
+module.exports = { parseRepository, isPrivateAddress, resolveManifest, checkImage, reportImages, imageRef, reportMismatches, changedVersionFiles };