diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2655b43..43d9b45 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,9 +7,10 @@ permissions: contents: read jobs: - validate: - name: validate + lint: + name: lint runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v7 @@ -18,7 +19,7 @@ jobs: node-version: 24 cache: npm - - name: Install schema checker + - name: Install dependencies run: npm ci - name: Validate model schemas @@ -27,41 +28,17 @@ jobs: - name: Test the catalog library run: npm run test:lib - validate-helm: - name: validate-helm - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - # Compare the PR head, not GitHub's synthetic merge commit. - ref: ${{ github.event.pull_request.head.sha }} - - - uses: actions/setup-node@v7 - with: - node-version: 24 - cache: npm - - - name: Install validator dependencies - run: npm ci - - name: Test Helm validator run: npm run test:helm - name: Install pinned Helm and kubeconform run: npm run helm:install - - name: Validate changed model variants against latest stable charts - env: - BASE_REF: ${{ github.event.pull_request.base.sha }} - run: npm run validate:helm -- --base "$BASE_REF" --chart-mode latest + - name: Validate every model variant against latest stable charts + run: npm run validate:helm - - name: Save Helm validation logs and manifests - if: always() - uses: actions/upload-artifact@v4 - with: - name: helm-validation - path: artifacts/helm-validation/ - if-no-files-found: ignore - retention-days: 14 + # A catalog is public, so every image must be pullable by anyone. The + # runner is the outside world: no login, no VPN. A pull request also + # holds each pinned digest to what its tag resolves to now. + - name: Check every image is publicly pullable + run: npm run validate:images -- --match-digest diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index d1642ec..7d6102a 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -34,6 +34,12 @@ jobs: - name: Install dependencies run: npm ci + - name: Install pinned Helm and kubeconform + run: npm run helm:install + + - name: validate + run: npm run validate + # index.json is built here, from models/, and checked against the one live # on the site: a published version must not change (hack/build-site.js). - name: Build Pages site and catalog @@ -43,7 +49,7 @@ jobs: - name: Upload Pages artifact if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' - uses: actions/upload-pages-artifact@v4 + uses: actions/upload-pages-artifact@v5 with: path: site diff --git a/README.md b/README.md index e9ec695..9b50cb9 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ models// ```sh npm run validate:schema npm run helm:install && npm run validate:helm -- models/my-model/my-model-1.0.0.yaml + npm run validate:images -- models/my-model/my-model-1.0.0.yaml PUBLISHED_INDEX=none npm run build:site # open site/index.html ``` @@ -83,6 +84,7 @@ What a variant needs to know: | `requires.gpus` | per pod; `nodes` + `topology: lws` for multi-node, with `values.lws.size` equal to `nodes` | | `requires.vendor` | `nvidia` (default), `ascend`, `cambricon`, `hygon` or `amd`; one vendor per variant | | `requires.gpuProduct` | canonical card names, any one matches: the `nvidia.com/gpu.product` value for NVIDIA (`NVIDIA-H100-80GB-HBM3`), `-` otherwise (`Ascend-910B3`). swiss maps names to each cluster's node labels | +| `image` | on a public registry, pullable without a login | | `chart.version` | exact (`0.8.0`) or a [semver range](https://github.com/Masterminds/semver#checking-version-constraints), quoted (`">=0.8.0"`, `"^0.8.0"`) | | `values` | chart values. Site-specific keys (namespace, host paths, registry, route ConfigMaps) are refused by the schema | @@ -102,9 +104,8 @@ tuning: report: my-model-h100-report.html ``` -A report must be a complete HTML document whose embedded JSON parses. -`validate:schema` fails on a missing, empty or invalid one; the site skips -linking it. +A report must be a complete HTML document whose embedded JSON parses. `validate:schema` fails on a missing, empty or invalid one; the site skips +linking it. The images it records must be the two variants' own (registry, repository and tag), which `validate:images` checks. ## Development @@ -113,17 +114,23 @@ linking it. | `npm run validate:schema` | schemas, tuning references, report HTML (pre-commit and CI) | | `npm run test:lib` | tests for `hack/lib` | | `npm run helm:install` | pinned Helm and kubeconform, once | -| `npm run validate:helm -- ` | render a version's variants against the charts (`--chart-mode declared` for what a deploy uses) | +| `npm run validate:helm` | render every variant against the charts (`--chart-mode declared` for what a deploy uses) | +| `npm run validate:images` | every image is publicly pullable without a login, by tag and by any pinned `digest`, and every perf report measured its variants' builds
a tag that has moved off its pinned digest is a warning; `--match-digest` makes it an error | +| `npm run validate` | `validate:schema`, `validate:helm` and `validate:images` | | `npm run build:site` | the site and `index.json` into `site/` | | `npm test` | `hack/validate.sh`: naming and layout checks (needs `yq`) | | `./hack/serve.sh` | rebuild `index.json` and serve the tree | +`validate:helm` and `validate:images` check every version file; pass files, or +`--base REF` for what a branch changes, to narrow them. + Needs Node.js 24. -- **CI.** `lints` runs on pull requests: `validate:schema`, `test:lib`, and - `validate-helm` on changed version files against the latest stable charts - ([Helm validation](schema/crds/README.md)). `pages` builds the site on pull - requests and deploys it on push to `master`. +- **CI.** `lints` runs on pull requests: `validate:schema`, `test:lib`, + `test:helm`, then `validate:helm` against the latest stable charts + ([Helm validation](schema/crds/README.md)) and `validate:images --match-digest`, + both on every version file. `pages` builds the site on pull requests and + deploys it on push to `master`. - **`index.json` is generated, not committed.** It is a pure function of `models/` and `catalog.yaml`, with no timestamp, so the same tree gives the same catalog ref in swiss. diff --git a/hack/lib/helm-validation.js b/hack/lib/helm-validation.js index 8971117..7594d3c 100644 --- a/hack/lib/helm-validation.js +++ b/hack/lib/helm-validation.js @@ -35,7 +35,7 @@ function git(root, args) { function selectFiles(root, {all = false, base = "origin/HEAD", files = []} = {}) { const models = path.join(root, "models"); const versions = modelNames(models).flatMap((name) => versionFiles(name, models)); - if (all) return {files: versions, added: [], reason: "--all"}; + if (all) return {files: versions, added: [], reason: "every version file"}; if (files.length) { const selected = [...new Set(files.map((file) => path.relative(root, path.resolve(root, file)).split(path.sep).join("/")))].sort(); if (selected.some((file) => !versions.includes(file))) throw new Error("explicit files must be existing model version YAMLs under models/"); diff --git a/hack/lib/images.js b/hack/lib/images.js new file mode 100644 index 0000000..e4be008 --- /dev/null +++ b/hack/lib/images.js @@ -0,0 +1,206 @@ +"use strict"; +// Whether an image the catalog names can be pulled by anyone: no login, from +// the registry it names. A catalog is public; an image only one site can pull +// is a variant only that site can deploy. + +const crypto = require("crypto"); +const dns = require("dns").promises; +const net = require("net"); +const path = require("path"); +const { command } = require("./helm-validation"); +const { modelNames, versionFiles } = require("./catalog"); +const { jsonBlocks } = require("./report"); + +const ACCEPT = [ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +].join(", "); + +// A repository the way containerd reads it: no registry host (or docker.io) +// means Docker Hub, and a bare name there is an official image under library/. +function parseRepository(repository) { + const parts = repository.split("/"); + const first = parts[0]; + if (parts.length > 1 && (first.includes(".") || first.includes(":") || first === "localhost")) { + if (first !== "docker.io" && first !== "index.docker.io") return { registry: first, host: first, name: parts.slice(1).join("/") }; + parts.shift(); + } + return { registry: "docker.io", host: "registry-1.docker.io", name: parts.length === 1 ? `library/${parts[0]}` : parts.join("/") }; +} + +// RFC 1918, loopback, link-local and unique-local: nobody outside reaches them. +function isPrivateAddress(ip) { + if (net.isIPv4(ip)) { + const [a, b] = ip.split(".").map(Number); + return a === 10 || a === 127 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || (a === 169 && b === 254); + } + const v6 = ip.toLowerCase(); + return v6 === "::1" || /^f[cd]/.test(v6) || /^fe[89ab]/.test(v6); +} + +// The private address a registry host resolves to, when it resolves to +// nothing else; null for a host the internet can reach (or that does not +// resolve, which the request itself reports). A proxy or VPN that reaches it +// from here does not make it public. +async function privateNetwork(host, lookup = dns.lookup) { + const name = host.replace(/:\d+$/, "").replace(/^\[|\]$/g, ""); + let addresses; + try { + addresses = net.isIP(name) ? [name] : (await lookup(name, { all: true })).map((a) => a.address); + } catch { + return null; + } + return addresses.length && addresses.every(isPrivateAddress) ? addresses[0] : null; +} + +function bearerChallenge(header) { + if (!/^\s*bearer\s/i.test(header || "")) return null; + const params = {}; + for (const [, k, v] of header.matchAll(/(\w+)="([^"]*)"/g)) params[k.toLowerCase()] = v; + return params.realm ? params : null; +} + +// The manifest digest ref (a tag or digest) names in repository, pulled +// anonymously, as { digest }; or why it cannot be pulled, as { problem }. +// HEAD first: Docker Hub does not count it against pulls. +async function resolveManifest(repository, ref, { fetch = globalThis.fetch, lookup, tokens = new Map(), attempts = 3 } = {}) { + const { host, name } = parseRepository(repository); + const internal = await privateNetwork(host, lookup); + if (internal) return { problem: `${host} is on a private network (${internal})` }; + const url = `https://${host}/v2/${name}/manifests/${ref}`; + const request = async (method) => { + const token = tokens.get(`${host}/${name}`); + return fetch(url, { method, headers: { Accept: ACCEPT, ...(token ? { Authorization: `Bearer ${token}` } : {}) } }); + }; + let lastError; + for (let i = 0; i < attempts; i++) { + try { + let r = await request("HEAD"); + if (r.status === 405) r = await request("GET"); + if (r.status === 401 && !tokens.has(`${host}/${name}`)) { + const challenge = bearerChallenge(r.headers.get("www-authenticate")); + if (!challenge) return { problem: "the registry requires a login" }; + const q = new URLSearchParams({ scope: `repository:${name}:pull` }); + if (challenge.service) q.set("service", challenge.service); + const t = await fetch(`${challenge.realm}?${q}`); + const body = t.ok ? await t.json() : {}; + if (!(body.token || body.access_token)) return { problem: "the registry gives no anonymous pull token" }; + tokens.set(`${host}/${name}`, body.token || body.access_token); + r = await request("HEAD"); + if (r.status === 405) r = await request("GET"); + } + if (r.ok) { + // Registries name the digest in a header; for one that does not, it is + // the digest of the manifest's bytes. + let digest = r.headers.get("docker-content-digest"); + if (!digest) { + const body = await request("GET"); + if (!body.ok) throw new Error(`HTTP ${body.status}`); + digest = `sha256:${crypto.createHash("sha256").update(Buffer.from(await body.arrayBuffer())).digest("hex")}`; + } + return { digest }; + } + if (r.status === 401 || r.status === 403) return { problem: "not pullable without a login (private, or no such repository)" }; + if (r.status === 404) return { problem: "no such manifest" }; + if (r.status === 429 || r.status >= 500) throw new Error(`HTTP ${r.status}`); + return { problem: `HTTP ${r.status}` }; + } catch (err) { + lastError = err; + await new Promise((resolve) => setTimeout(resolve, 500 * 2 ** i)); + } + } + const reason = lastError && lastError.cause ? lastError.cause.code || lastError.cause.message : lastError && lastError.message; + return { problem: `could not reach ${host}: ${reason}` }; +} + +// How an image block fares pulled anonymously, as { level, message }. "error" +// when its tag or pinned digest cannot be pulled; "warning" when the tag has +// moved off a pinned digest that is still pullable; else "ok". With +// matchDigest a moved tag is an error: a runtime pulls the digest and ignores +// the tag, so the two name different builds. Tags move, so only a change +// being made can be held to that. +async function checkImage(image, { matchDigest = false, ...opts } = {}) { + const ref = `${image.repository}:${image.tag}`; + const tag = await resolveManifest(image.repository, image.tag, opts); + if (tag.problem) return { level: "error", message: `${ref}: ${tag.problem}` }; + if (!image.digest) return { level: "ok", message: `${ref}: pullable, resolves to ${tag.digest}` }; + const pinned = `${ref}@${image.digest}`; + if (tag.digest === image.digest) return { level: "ok", message: `${pinned}: pullable, tag matches the pinned digest` }; + const moved = `the tag resolves to ${tag.digest} now, not the pinned digest`; + if (matchDigest) return { level: "error", message: `${pinned}: ${moved}` }; + const digest = await resolveManifest(image.repository, image.digest, opts); + if (digest.problem) return { level: "error", message: `${pinned}: ${digest.problem}` }; + return { level: "warning", message: `${pinned}: ${moved}; both are pullable` }; +} + +// The data block an AutoTune report renders from: its page mounts with +// AutotuneReport.mountStandalone("autotune-report", "autotune-report-data"). +const REPORT_DATA = "autotune-report-data"; + +// The images a perf report's benchmark ran, from the block it renders from: +// the baseline's, and each tuning attempt's, which the page shows as its Image +// row. Empty sets when it records none. +function reportImages(html) { + const out = { baseline: new Set(), attempts: new Set() }; + let data; + try { + data = JSON.parse(jsonBlocks(html).find((block) => block.id === REPORT_DATA).text); + } catch { + return out; + } + const image = (run) => run && run.launch && run.launch.config && run.launch.config.image; + for (const r of data.reports || []) { + const c = r && r.comparison; + if (!c) continue; + if (image(c.baseline)) out.baseline.add(image(c.baseline)); + for (const a of c.attempts || []) if (image(a)) out.attempts.add(image(a)); + } + return out; +} + +// An image reference in full, the way containerd reads it, digest aside: +// registry, repository and tag, so nginx and docker.io/library/nginx:latest are +// one image, and the same tag on another registry or namespace is another. +function imageRef(ref) { + const name = String(ref).split("@")[0]; + const slash = name.lastIndexOf("/"); + const colon = name.lastIndexOf(":"); + const { registry, name: repository } = parseRepository(colon > slash ? name.slice(0, colon) : name); + return `${registry}/${repository}:${colon > slash ? name.slice(colon + 1) : "latest"}`; +} + +// Why a tuned pair's report did not measure the images its two variants run. +// The tuned build need only be among the attempts: a tuning run may try more +// than one, and its best overall can be the baseline. +function reportMismatches(html, { baseline, optimized }) { + const ran = reportImages(html); + const out = []; + for (const [role, want, got] of [["baseline", baseline, ran.baseline], ["tuned", optimized, ran.attempts]]) { + if (!got.size) out.push(`records no ${role} image`); + else if (![...got].some((x) => imageRef(x) === imageRef(want))) { + out.push(`the ${role} variant runs ${want}, but the report measured ${[...got].join(", ")}`); + } + } + return out; +} + +// Version files a branch adds or changes since its merge-base with base, and +// every version of a model whose metadata or perf report changed. Deleted +// files have nothing left to check. +function changedVersionFiles(root, base) { + const git = (args) => { + const result = command("git", args, { cwd: root }); + if (!result.ok) throw new Error(result.log); + return result.stdout; + }; + const mergeBase = git(["merge-base", base, "HEAD"]).trim(); + const changed = git(["diff", "--name-only", "-z", "--diff-filter=AMR", mergeBase, "HEAD"]).split("\0").filter(Boolean); + const touched = new Set(changed.filter((f) => /^models\/[^/]+\/(metadata\.yaml|[^/]+\.html)$/.test(f)).map((f) => f.split("/")[1])); + const models = path.join(root, "models"); + const versions = modelNames(models).flatMap((name) => versionFiles(name, models)); + return versions.filter((file) => changed.includes(file) || touched.has(file.split("/")[1])); +} + +module.exports = { parseRepository, isPrivateAddress, resolveManifest, checkImage, reportImages, imageRef, reportMismatches, changedVersionFiles }; diff --git a/hack/lib/images.test.js b/hack/lib/images.test.js new file mode 100644 index 0000000..1d4d176 --- /dev/null +++ b/hack/lib/images.test.js @@ -0,0 +1,160 @@ +"use strict"; + +const assert = require("node:assert/strict"); +const crypto = require("node:crypto"); +const { test } = require("node:test"); +const { parseRepository, isPrivateAddress, resolveManifest, checkImage, imageRef, reportMismatches } = require("./images"); + +const publicDns = async () => [{ address: "52.45.125.121" }]; +const digestOf = (manifest) => `sha256:${crypto.createHash("sha256").update(manifest).digest("hex")}`; +const [V1, V2] = ['{"schemaVersion":2,"build":1}', '{"schemaVersion":2,"build":2}']; + +// A registry that wants an anonymous token, as Docker Hub and registry do. +// manifests maps a tag or digest to the manifest it serves. +function registry({ manifests = {}, token = "t", failures = 0, digestHeader = true } = {}) { + const calls = []; + const fetch = async (url, init = {}) => { + calls.push(url); + if (failures-- > 0) throw new TypeError("fetch failed", { cause: { code: "ECONNRESET" } }); + if (url.startsWith("https://auth.example/token")) { + return { ok: Boolean(token), status: token ? 200 : 401, json: async () => (token ? { token } : {}) }; + } + const authed = init.headers && init.headers.Authorization === `Bearer ${token}`; + if (!authed) { + return { ok: false, status: 401, headers: new Headers({ "www-authenticate": 'Bearer realm="https://auth.example/token",service="reg"' }) }; + } + const manifest = manifests[url.split("/manifests/")[1]]; + if (!manifest) return { ok: false, status: 404, headers: new Headers() }; + const headers = new Headers(digestHeader ? { "docker-content-digest": digestOf(manifest) } : {}); + return { ok: true, status: 200, headers, arrayBuffer: async () => new TextEncoder().encode(init.method === "HEAD" ? "" : manifest).buffer }; + }; + return { fetch, calls }; +} + +test("repositories resolve the way containerd resolves them", () => { + assert.deepEqual(parseRepository("lmsysorg/sglang"), { registry: "docker.io", host: "registry-1.docker.io", name: "lmsysorg/sglang" }); + assert.deepEqual(parseRepository("nginx"), { registry: "docker.io", host: "registry-1.docker.io", name: "library/nginx" }); + assert.deepEqual(parseRepository("docker.io/nginx"), parseRepository("nginx")); + assert.deepEqual(parseRepository("index.docker.io/lmsysorg/sglang"), parseRepository("lmsysorg/sglang")); + assert.deepEqual(parseRepository("quay.io/ascend/vllm-ascend"), { registry: "quay.io", host: "quay.io", name: "ascend/vllm-ascend" }); + assert.deepEqual(parseRepository("localhost:5000/x"), { registry: "localhost:5000", host: "localhost:5000", name: "x" }); +}); + +test("private, loopback and link-local addresses are not public", () => { + for (const ip of ["10.1.2.3", "172.27.133.35", "192.168.0.1", "127.0.0.1", "169.254.1.1", "::1", "fd00::1", "fe80::1"]) { + assert.ok(isPrivateAddress(ip), ip); + } + for (const ip of ["172.32.0.1", "52.45.125.121", "100.63.226.34", "2600:1f18::1"]) assert.ok(!isPrivateAddress(ip), ip); +}); + +test("an image anyone can pull is ok, after an anonymous token", async () => { + const { fetch } = registry({ manifests: { "v1": V1 } }); + assert.deepEqual(await checkImage({ repository: "org/app", tag: "v1" }, { fetch, lookup: publicDns }), { + level: "ok", + message: `org/app:v1: pullable, resolves to ${digestOf(V1)}`, + }); + assert.deepEqual(await checkImage({ repository: "org/app", tag: "v1", digest: digestOf(V1) }, { fetch, lookup: publicDns }), { + level: "ok", + message: `org/app:v1@${digestOf(V1)}: pullable, tag matches the pinned digest`, + }); +}); + +test("a missing tag or pinned digest is an error", async () => { + const { fetch } = registry({ manifests: { "v1": V1 } }); + assert.deepEqual(await checkImage({ repository: "org/app", tag: "v2" }, { fetch, lookup: publicDns }), { + level: "error", + message: "org/app:v2: no such manifest", + }); + assert.deepEqual(await checkImage({ repository: "org/app", tag: "v1", digest: digestOf(V2) }, { fetch, lookup: publicDns }), { + level: "error", + message: `org/app:v1@${digestOf(V2)}: no such manifest`, + }); +}); + +test("a tag that has moved off its pinned digest warns, or is an error when they must match", async () => { + const { fetch } = registry({ manifests: { "v1": V1, [digestOf(V2)]: V2 } }); + const image = { repository: "org/app", tag: "v1", digest: digestOf(V2) }; + const moved = `org/app:v1@${digestOf(V2)}: the tag resolves to ${digestOf(V1)} now, not the pinned digest`; + assert.deepEqual(await checkImage(image, { fetch, lookup: publicDns }), { level: "warning", message: `${moved}; both are pullable` }); + assert.deepEqual(await checkImage(image, { fetch, lookup: publicDns, matchDigest: true }), { level: "error", message: moved }); + assert.equal((await checkImage({ ...image, digest: digestOf(V1) }, { fetch, lookup: publicDns, matchDigest: true })).level, "ok"); +}); + +test("a registry that names no digest is read from the manifest's bytes", async () => { + const { fetch } = registry({ manifests: { "v1": V1 }, digestHeader: false }); + assert.deepEqual(await resolveManifest("org/app", "v1", { fetch, lookup: publicDns }), { digest: digestOf(V1) }); +}); + +test("a registry that gives no anonymous token fails", async () => { + const { fetch } = registry({ token: null }); + assert.match((await resolveManifest("org/app", "v1", { fetch, lookup: publicDns })).problem, /no anonymous pull token/); +}); + +test("a registry on a private network fails without being asked", async () => { + const { fetch, calls } = registry({ manifests: { "v1": V1 } }); + const lookup = async () => [{ address: "172.27.133.35" }]; + assert.match((await resolveManifest("registry.example.io/team/app", "v1", { fetch, lookup })).problem, /registry\.example\.io is on a private network \(172\.27\.133\.35\)/); + assert.equal(calls.length, 0); +}); + +test("a dropped connection is retried", async () => { + const { fetch } = registry({ manifests: { "v1": V1 }, failures: 1 }); + assert.deepEqual(await resolveManifest("org/app", "v1", { fetch, lookup: publicDns }), { digest: digestOf(V1) }); + const down = registry({ failures: 10 }); + assert.match((await resolveManifest("org/app", "v1", { fetch: down.fetch, lookup: publicDns, attempts: 2 })).problem, /could not reach registry-1\.docker\.io: ECONNRESET/); +}); + +// A perf report as AutoTune writes it: each language carries the comparison. +const report = (baseline, ...attempts) => { + const run = (image) => ({ launch: { config: { image } } }); + const comparison = { baseline: run(baseline), attempts: attempts.map(run) }; + const data = { reports: [{ lang: "en", comparison }, { lang: "zh", comparison }] }; + return ``; +}; + +test("an image is its registry, repository and tag", () => { + assert.equal(imageRef("lmsysorg/sglang:v0.5.19"), "docker.io/lmsysorg/sglang:v0.5.19"); + assert.equal(imageRef("docker.io/library/nginx"), imageRef("nginx:latest")); + assert.equal(imageRef("quay.io/ascend/vllm-ascend:v1@sha256:ff"), "quay.io/ascend/vllm-ascend:v1"); + assert.equal(imageRef("localhost:5000/app"), "localhost:5000/app:latest"); + assert.notEqual(imageRef("registry.example.io/team/sglang:v0.5.19"), imageRef("lmsysorg/sglang:v0.5.19")); +}); + +test("a report that measured its variants' images passes", () => { + const html = report("docker.io/lmsysorg/sglang:v0.5.19", "lmsysorg/sglang:v0.5.19"); + assert.deepEqual(reportMismatches(html, { baseline: "lmsysorg/sglang:v0.5.19", optimized: "lmsysorg/sglang:v0.5.19" }), []); +}); + +test("a report that measured the same tag from another registry fails", () => { + const html = report("registry.example.io/team/sglang:glm", "registry.example.io/team/sglang:glm"); + assert.deepEqual(reportMismatches(html, { baseline: "org/sglang:glm", optimized: "org/sglang:glm" }), [ + "the baseline variant runs org/sglang:glm, but the report measured registry.example.io/team/sglang:glm", + "the tuned variant runs org/sglang:glm, but the report measured registry.example.io/team/sglang:glm", + ]); +}); + +test("a report that measured another build fails, per side", () => { + const html = report("x/sglang:v0.5.18", "x/sglang:v0.5.19"); + const got = reportMismatches(html, { baseline: "lmsysorg/sglang:v0.5.19", optimized: "lmsysorg/sglang:v0.5.20" }); + assert.equal(got.length, 2); + assert.match(got[0], /baseline variant runs lmsysorg\/sglang:v0\.5\.19, but the report measured x\/sglang:v0\.5\.18/); + assert.match(got[1], /tuned variant runs lmsysorg\/sglang:v0\.5\.20/); +}); + +test("the tuned image need only be among the attempts", () => { + const html = report("x/sglang:a", "x/sglang:b", "x/sglang:c"); + assert.deepEqual(reportMismatches(html, { baseline: "x/sglang:a", optimized: "x/sglang:c" }), []); +}); + +test("a report that records no images says so", () => { + const got = reportMismatches("", { baseline: "a:1", optimized: "a:1" }); + assert.deepEqual(got, ["records no baseline image", "records no tuned image"]); +}); + +test("only the block the report renders from is read", () => { + const runs = { baseline: "x/sglang:a", optimized: "x/sglang:b" }; + const decoy = report("y/sglang:a", "y/sglang:b").replace("autotune-report-data", "other-data"); + const decoyBlock = decoy.match(//)[0]; + assert.deepEqual(reportMismatches(report("x/sglang:a", "x/sglang:b").replace("", `${decoyBlock}`), runs), []); + assert.deepEqual(reportMismatches(decoy, runs), ["records no baseline image", "records no tuned image"]); +}); diff --git a/hack/lib/report.js b/hack/lib/report.js index 6e3586b..ca414fa 100644 --- a/hack/lib/report.js +++ b/hack/lib/report.js @@ -3,7 +3,7 @@ // and validate:schema fails on any other, so the two cannot disagree. const fs = require("fs"); -const { HtmlValidate, StaticConfigLoader } = require("html-validate"); +const { HtmlValidate, Parser, StaticConfigLoader } = require("html-validate"); // Conformance to the HTML standard for a whole document, without style rules: // a cut-off file fails as an unclosed element, a non-HTML one on its doctype. @@ -11,9 +11,17 @@ const validator = new HtmlValidate( new StaticConfigLoader({ extends: ["html-validate:standard", "html-validate:document"] }) ); -// A report renders from the JSON it embeds, so a page that is valid HTML but -// carries broken JSON still shows nothing. -const JSON_BLOCK = /]*\btype\s*=\s*["']?application\/(?:ld\+)?json["']?[^>]*>([\s\S]*?)<\/script\s*>/gi; +// The JSON data blocks of an HTML document, '>\n` + + `\n` + + `\n` + + `\n` + + `\n\n`; + assert.deepEqual(jsonBlocks(html), [ + { id: "a", text: '{"a": 1}', line: 4 }, + { id: undefined, text: '{"b": 2}', line: 5 }, + { id: undefined, text: '{"c": 3}', line: 6 }, + ]); +}); diff --git a/hack/validate-helm.js b/hack/validate-helm.js index 6d120c5..a592b3e 100644 --- a/hack/validate-helm.js +++ b/hack/validate-helm.js @@ -14,16 +14,15 @@ function options(args) { const opts = {files: [], chartMode: "latest", output: path.join(root, "artifacts/helm-validation")}; for (let i = 0; i < args.length; i++) { const arg = args[i]; - if (arg === "--all") opts.all = true; - else if (["--base", "--output", "--chart-mode"].includes(arg)) { + if (["--base", "--output", "--chart-mode"].includes(arg)) { if (!args[i + 1] || args[i + 1].startsWith("--")) throw new Error(`${arg}: missing argument`); opts[arg === "--chart-mode" ? "chartMode" : arg.slice(2)] = args[++i]; } else if (arg.startsWith("-")) throw new Error(`unknown option ${arg}`); else opts.files.push(arg); } - if ([Boolean(opts.all), Boolean(opts.base), Boolean(opts.files.length)].filter(Boolean).length > 1) { - throw new Error("choose one of --all, --base REF, or explicit files"); - } + if (opts.base && opts.files.length) throw new Error("choose --base REF or explicit files, not both"); + // Nothing narrows it: every version file. + opts.all = !opts.base && !opts.files.length; opts.output = path.resolve(root, opts.output); if (!["latest", "declared"].includes(opts.chartMode)) throw new Error("--chart-mode must be latest or declared"); return opts; diff --git a/hack/validate-helm.test.js b/hack/validate-helm.test.js index a3cb69e..a0c541f 100644 --- a/hack/validate-helm.test.js +++ b/hack/validate-helm.test.js @@ -147,11 +147,19 @@ for (const failure of ["crds", "resolution", "download", "schema", "lint", "temp } test("CLI rejects ambiguous selectors and incomplete arguments", () => { - assert.throws(() => options(["--all", "--base", "main"])); + assert.throws(() => options(["--base", "main", "models/m/m-1.0.0.yaml"])); + assert.throws(() => options(["--all"])); assert.throws(() => options(["--base"])); assert.throws(() => options(["--unknown"])); assert.throws(() => options(["--chart-mode", "invalid"])); assert.throws(() => options(["--chart-mode"])); assert.equal(options([]).chartMode, "latest"); - assert.equal(options(["--all", "--chart-mode", "declared"]).chartMode, "declared"); + assert.equal(options(["--chart-mode", "declared"]).chartMode, "declared"); +}); + +test("CLI checks every version file unless files or --base narrow it", () => { + assert.equal(options([]).all, true); + assert.equal(options(["--chart-mode", "declared"]).all, true); + assert.equal(options(["--base", "main"]).all, false); + assert.equal(options(["models/m/m-1.0.0.yaml"]).all, false); }); diff --git a/hack/validate-images.js b/hack/validate-images.js new file mode 100644 index 0000000..5e1ec13 --- /dev/null +++ b/hack/validate-images.js @@ -0,0 +1,123 @@ +#!/usr/bin/env node +// Check every image the catalog names can be pulled anonymously, by tag and by +// any digest it pins, and that each tuned pair's perf report measured the +// builds its two variants run. Each image is logged as ok, warning (its tag +// has moved off a pinned digest that is still pullable) or error. +// +// npm run validate:images # the whole catalog +// npm run validate:images -- models/x/x-1.0.0.yaml # only these files +// npm run validate:images -- --base origin/master # only what a branch adds or changes +// npm run validate:images -- --match-digest # and each pinned digest is what its tag resolves to now +"use strict"; + +const fs = require("node:fs"); +const http = require("node:http"); +const path = require("node:path"); +const YAML = require("yaml"); +const { root, modelNames, versionFiles } = require("./lib/catalog"); +const { checkImage, reportMismatches, changedVersionFiles } = require("./lib/images"); + +// Reach registries through HTTP(S)_PROXY / NO_PROXY when they are set, as +// --use-env-proxy would, without the caller having to pass it. +http.setGlobalProxyFromEnv(); + +function parse(args) { + const files = []; + let base = null; + let matchDigest = false; + for (let i = 0; i < args.length; i++) { + if (args[i] === "--base") { + if (!args[i + 1] || args[i + 1].startsWith("--")) throw new Error("--base: missing argument"); + base = args[++i]; + } else if (args[i] === "--match-digest") matchDigest = true; + else if (args[i].startsWith("-")) throw new Error(`unknown option ${args[i]}`); + else files.push(path.relative(root, path.resolve(args[i])).split(path.sep).join("/")); + } + if (base && files.length) throw new Error("choose --base REF or explicit files, not both"); + if (base) return { files: changedVersionFiles(root, base), matchDigest }; + const versions = modelNames().flatMap((name) => versionFiles(name)); + if (!files.length) return { files: versions, matchDigest }; + const unknown = files.filter((f) => !versions.includes(f)); + if (unknown.length) throw new Error(`not a model version file: ${unknown.join(", ")}`); + return { files, matchDigest }; +} + +async function main() { + const { files, matchDigest } = parse(process.argv.slice(2)); + if (files.length === 0) { + console.log("no version files to check"); + return 0; + } + + const docs = new Map(files.map((file) => [file, YAML.parse(fs.readFileSync(path.join(root, file), "utf8"))])); + + // Each distinct image once, however many variants share it. + const uses = new Map(); + for (const [file, doc] of docs) { + for (const v of doc.variants ?? []) { + if (!v?.image?.repository) continue; + const key = JSON.stringify([v.image.repository, String(v.image.tag ?? "latest"), v.image.digest ?? ""]); + if (!uses.has(key)) uses.set(key, []); + uses.get(key).push({ file, id: v.id }); + } + } + + const tokens = new Map(); + const results = new Map(); + const keys = [...uses.keys()]; + // A few at a time: enough to be quick, few enough not to trip rate limits. + for (let i = 0; i < keys.length; i += 4) { + const batch = keys.slice(i, i + 4); + await Promise.all(batch.map(async (key) => { + const [repository, tag, digest] = JSON.parse(key); + results.set(key, await checkImage({ repository, tag, digest }, { tokens, matchDigest })); + })); + for (const key of batch) console.log(`${results.get(key).level.padEnd(7)} ${results.get(key).message}`); + } + + const byFile = new Map(); + const note = (file, line) => { + if (!byFile.has(file)) byFile.set(file, []); + byFile.get(file).push(line); + }; + for (const [key, { level, message }] of results) { + if (level === "error") for (const { file, id } of uses.get(key)) note(file, `${id}: ${message}`); + } + + // Tuned pairs of the selected versions. A missing report or variant is + // validate:schema's to report. + let reports = 0; + for (const name of new Set([...docs.keys()].map((file) => file.split("/")[1]))) { + const meta = `models/${name}/metadata.yaml`; + const tuning = YAML.parse(fs.readFileSync(path.join(root, meta), "utf8"))?.tuning ?? []; + tuning.forEach((t, i) => { + const doc = [...docs].find(([file, d]) => file.split("/")[1] === name && String(d?.version) === String(t?.version))?.[1]; + const variant = (id) => doc?.variants?.find((v) => v?.id === id); + const [b, o] = [variant(t?.baseline), variant(t?.optimized)]; + const html = path.join(root, "models", name, String(t?.report ?? "")); + if (!t?.report || !b?.image || !o?.image || !fs.existsSync(html)) return; + reports++; + const image = (v) => `${v.image.repository}:${v.image.tag ?? "latest"}`; + for (const m of reportMismatches(fs.readFileSync(html, "utf8"), { baseline: image(b), optimized: image(o) })) { + note(meta, `tuning[${i}] (${t.report}): ${m}`); + } + }); + } + [...byFile.keys()].sort().forEach((file, i) => { + if (i) console.error(""); + console.error(file); + for (const line of byFile.get(file)) console.error(` ${line}`); + }); + const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`; + const count = (level) => [...results.values()].filter((r) => r.level === level).length; + const errors = count("error"); + const bad = [...byFile.keys()].filter((f) => f.endsWith("metadata.yaml")).reduce((n, f) => n + byFile.get(f).length, 0); + console.log(`${plural(keys.length, "image")} in ${plural(files.length, "version file")}: ${count("ok")} ok, ${plural(count("warning"), "warning")}, ${plural(errors, "error")}`); + console.log(`${reports} perf report${reports === 1 ? "" : "s"} checked against their variants' images, ${bad} mismatch${bad === 1 ? "" : "es"}`); + return errors || bad ? 1 : 0; +} + +main().then((code) => process.exit(code), (err) => { + console.error(err.message); + process.exit(2); +}); diff --git a/models/deepseek-v4-flash-0731/deepseek-v4-flash-0731-1.0.0.yaml b/models/deepseek-v4-flash-0731/deepseek-v4-flash-0731-1.0.0.yaml index 837ad43..8dda640 100644 --- a/models/deepseek-v4-flash-0731/deepseek-v4-flash-0731-1.0.0.yaml +++ b/models/deepseek-v4-flash-0731/deepseek-v4-flash-0731-1.0.0.yaml @@ -11,7 +11,7 @@ variants: name: sglang version: ">=0.7.1" image: - repository: harbor.4pd.io/hardcore-tech/sglang + repository: lmsysorg/sglang tag: v0.5.19-cu129 requires: gpus: 8 @@ -81,7 +81,7 @@ variants: name: sglang version: ">=0.7.1" image: - repository: harbor.4pd.io/hardcore-tech/sglang + repository: lmsysorg/sglang tag: v0.5.19-cu129 requires: gpus: 8 diff --git a/models/deepseek-v4-flash-0731/deepseek-v4-flash-0731-h100-report.html b/models/deepseek-v4-flash-0731/deepseek-v4-flash-0731-h100-report.html index 9cf6132..7ac994a 100644 --- a/models/deepseek-v4-flash-0731/deepseek-v4-flash-0731-h100-report.html +++ b/models/deepseek-v4-flash-0731/deepseek-v4-flash-0731-h100-report.html @@ -7,7 +7,7 @@
- + + + + + + + + + + + + + + + + + + + + + + +