From e5cb84c164ae1a106228e647201c84c48c9713b7 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 16 Sep 2026 18:22:05 -0400 Subject: [PATCH] fix: require producer-declared native runtime license files The archive allowlist accepted any path under a runtime's licenses/ directory, at any depth, with nothing tying those bytes to what the producer actually shipped. Narrow it to what upstream declares. The tar listing now accepts only a flat, safely named licenses/, since validateArchiveEntries runs before extraction and cannot consult the runtime manifest. After extraction, every runtime file outside lib/, tools/, manifest.json, and README.md must appear in the runtime manifest's runtime.files map with a matching SHA-256. That check runs after the manifest digest is bound to product-manifest.json, so the declarations come from a manifest the product already vouches for. Runtimes that ship no extra files make no declaration and are left alone, so bundles predating license bundling still verify. Traversal, entry-type, checksum, tree-digest, and single-runtime checks are unchanged. --- TODO.md | 8 +++- scripts/upstream-archive.ts | 61 ++++++++++++++++++++++- tests/upstream-archive.test.ts | 88 +++++++++++++++++++++++++++++----- 3 files changed, 144 insertions(+), 13 deletions(-) diff --git a/TODO.md b/TODO.md index efb3ed5..49f65e2 100644 --- a/TODO.md +++ b/TODO.md @@ -5,9 +5,15 @@ formula use the same 30-second bound, focused regression tests pass, the full TypeScript suite remains green, and the v0.76.1 packaging recovery completes. -- [ ] Accept producer-declared native runtime license material in verified +- [x] Accept producer-declared native runtime license material in verified product bundles so v0.76.1 CUDA packaging can complete without weakening the existing checksum, entry-type, tree-digest, or single-runtime checks. + Final result: the archive listing accepts only a flat, safely named + `licenses/`, and extraction then requires every runtime file outside + `lib/`, `tools/`, `manifest.json`, and `README.md` to be declared in the + runtime manifest's `runtime.files` map with a matching SHA-256. Runtimes that + ship no extra files make no declaration and are unaffected, so bundles + predating license bundling still verify. QA: focused archive fixtures accept `licenses/NVIDIA-CUDA-LICENSE.txt`, the full TypeScript suite passes, matrix validation remains green, actionlint passes, and a protected v0.76.1 packaging rerun completes every required job. diff --git a/scripts/upstream-archive.ts b/scripts/upstream-archive.ts index bc474c3..0066c8e 100755 --- a/scripts/upstream-archive.ts +++ b/scripts/upstream-archive.ts @@ -16,6 +16,24 @@ type Inputs = { }; const runtimeIdPattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +const licenseNamePattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +// Libraries and tools are addressed by the product manifest and the runtime +// tree digest. Anything else the producer ships (today: CUDA distribution +// license material) has to earn its place by being named in the runtime +// manifest, so the archive listing can stay a shape check and the digest check +// can happen after extraction. +function isRuntimeLicenseEntry(relative: string): boolean { + const match = /^licenses\/([^/]+)$/.exec(relative); + return match !== null && licenseNamePattern.test(match[1]); +} + +function isStructuralRuntimeFile(relative: string): boolean { + return relative === "manifest.json" + || relative === "README.md" + || relative.startsWith("lib/") + || relative.startsWith("tools/"); +} export function productBackendForFlavor(flavor: string): string { if (flavor === "cuda-12" || flavor === "cuda-13") return "cuda"; @@ -67,7 +85,7 @@ export function validateArchiveEntries(entries: string[]): void { if (!match) throw new Error(`unexpected product archive entry: ${path}`); const [, runtimeId, relative] = match; if (!runtimeIdPattern.test(runtimeId)) throw new Error(`unsafe native runtime id: ${runtimeId}`); - if (relative !== "manifest.json" && relative !== "README.md" && !relative.startsWith("lib/") && !relative.startsWith("licenses/") && !relative.startsWith("tools/")) { + if (!isStructuralRuntimeFile(relative) && !isRuntimeLicenseEntry(relative)) { throw new Error(`unexpected native runtime entry: ${path}`); } runtimeIds.add(runtimeId); @@ -148,6 +166,46 @@ function filesBelow(root: string, current = root): string[] { }); } +function declaredRuntimeFileDigests(runtimeDir: string): Record { + const raw = readFileSync(resolve(runtimeDir, "manifest.json"), "utf8"); + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error("native runtime manifest is not valid JSON"); + } + const files = (parsed as { runtime?: { files?: unknown } })?.runtime?.files; + if (!files || typeof files !== "object" || Array.isArray(files)) return {}; + return files as Record; +} + +/** + * Accepts only the extra runtime files the producer declared, and proves their + * bytes. The archive listing already bounded these to `licenses/`; this + * is where an entry that nothing in the runtime manifest vouches for is + * rejected. Runtimes that ship no extra files make no claim and are left alone, + * which is what keeps releases predating the license bundling verifiable. + */ +export async function verifyDeclaredRuntimeFiles(runtimeDir: string): Promise { + const extra = filesBelow(runtimeDir) + .map((path) => path.slice(runtimeDir.length + 1).replaceAll("\\", "/")) + .filter((relative) => !isStructuralRuntimeFile(relative)) + .sort(); + if (extra.length === 0) return []; + const declared = declaredRuntimeFileDigests(runtimeDir); + for (const relative of extra) { + const expected = declared[relative]; + if (typeof expected !== "string" || !/^[0-9a-f]{64}$/.test(expected)) { + throw new Error(`native runtime manifest does not declare runtime file ${relative}`); + } + const actual = await sha256File(resolve(runtimeDir, relative)); + if (actual !== expected) { + throw new Error(`native runtime file ${relative} does not match its declared digest`); + } + } + return extra; +} + export function sha256Tree(root: string): string { const digest = createHash("sha256"); const buffer = Buffer.allocUnsafe(1024 * 1024); @@ -210,6 +268,7 @@ export async function verifyAndExtract(input: Inputs) { if (runtimeSha256 !== productManifest.runtime.sha256) throw new Error("product runtime digest does not match extracted runtime tree"); const runtimeManifestSha256 = await sha256File(resolve(runtime, "manifest.json")); if (runtimeManifestSha256 !== productManifest.runtime.manifest_sha256) throw new Error("product runtime manifest digest does not match"); + await verifyDeclaredRuntimeFiles(runtime); const provenance = resolve(input.outputDir, "upstream-provenance.json"); writeFileSync(provenance, `${JSON.stringify({ archive: archiveName, diff --git a/tests/upstream-archive.test.ts b/tests/upstream-archive.test.ts index c33578d..d7bbf15 100644 --- a/tests/upstream-archive.test.ts +++ b/tests/upstream-archive.test.ts @@ -15,6 +15,7 @@ import { validateArchiveEntryTypes, validateProductManifest, verifyAndExtract, + verifyDeclaredRuntimeFiles, } from "../scripts/upstream-archive.ts"; test("maps versioned CUDA flavors to the product backend", () => { @@ -44,7 +45,14 @@ function legacySha256Tree(root: string): string { return digest.digest("hex"); } -async function fixture(t: { after(callback: () => void): void }) { +type FixtureOptions = { + // Extra runtime files staged under the runtime directory, and what the + // runtime manifest claims about them. `declare: false` stages the bytes + // without declaring them; `digest` overrides the declared digest. + licenses?: { name: string; contents: string; declare?: boolean; digest?: string }[]; +}; + +async function fixture(t: { after(callback: () => void): void }, options: FixtureOptions = {}) { const directory = mkdtempSync(resolve(tmpdir(), "upstream-test-")); t.after(() => rmSync(directory, { recursive: true, force: true })); const bundle = resolve(directory, "stage/mesh-bundle"); @@ -54,9 +62,26 @@ async function fixture(t: { after(callback: () => void): void }) { writeFileSync(resolve(bundle, "host-imports.json"), "{}\n"); const runtime = resolve(bundle, "native-runtimes/linux-cpu"); mkdirSync(resolve(runtime, "lib"), { recursive: true }); - writeFileSync(resolve(runtime, "manifest.json"), "{}\n"); writeFileSync(resolve(runtime, "README.md"), "runtime\n"); writeFileSync(resolve(runtime, "lib/libllama.so"), "runtime\n"); + if (options.licenses?.length) { + mkdirSync(resolve(runtime, "licenses"), { recursive: true }); + const files: Record = { + "lib/libllama.so": await sha256File(resolve(runtime, "lib/libllama.so")), + }; + for (const license of options.licenses) { + const relative = `licenses/${license.name}`; + writeFileSync(resolve(runtime, relative), license.contents); + if (license.declare === false) continue; + files[relative] = license.digest ?? await sha256File(resolve(runtime, relative)); + } + writeFileSync(resolve(runtime, "manifest.json"), `${JSON.stringify({ + build: { primary_library: "lib/libllama.so" }, + runtime: { files, id: "linux-cpu", libraries: ["lib/libllama.so"] }, + }, null, 2)}\n`); + } else { + writeFileSync(resolve(runtime, "manifest.json"), "{}\n"); + } const hostSha256 = await sha256File(resolve(bundle, "mesh-llm")); const runtimeSha256 = sha256Tree(runtime); const runtimeManifestSha256 = await sha256File(resolve(runtime, "manifest.json")); @@ -123,16 +148,57 @@ test("rejects unsafe runtime IDs before accepting runtime paths", () => { ]), /runtime id/); }); +const cudaEntries = (license: string) => [ + "mesh-bundle/mesh-llm", + "mesh-bundle/product-manifest.json", + "mesh-bundle/host-imports.json", + "mesh-bundle/native-runtimes/linux-cuda/manifest.json", + "mesh-bundle/native-runtimes/linux-cuda/README.md", + "mesh-bundle/native-runtimes/linux-cuda/lib/libllama.so", + `mesh-bundle/native-runtimes/linux-cuda/licenses/${license}`, +]; + test("accepts bundled native runtime license material", () => { - assert.doesNotThrow(() => validateArchiveEntries([ - "mesh-bundle/mesh-llm", - "mesh-bundle/product-manifest.json", - "mesh-bundle/host-imports.json", - "mesh-bundle/native-runtimes/linux-cuda/manifest.json", - "mesh-bundle/native-runtimes/linux-cuda/README.md", - "mesh-bundle/native-runtimes/linux-cuda/lib/libllama.so", - "mesh-bundle/native-runtimes/linux-cuda/licenses/NVIDIA-CUDA-LICENSE.txt", - ])); + assert.doesNotThrow(() => validateArchiveEntries(cudaEntries("NVIDIA-CUDA-LICENSE.txt"))); +}); + +test("keeps the license allowlist flat and safely named", () => { + for (const license of ["nested/EULA.txt", ".hidden", "-leading", "spaced name.txt"]) { + assert.throws(() => validateArchiveEntries(cudaEntries(license)), /unexpected native runtime entry/); + } +}); + +test("accepts producer-declared runtime license files and reports them", async (t) => { + const data = await fixture(t, { licenses: [{ name: "NVIDIA-CUDA-LICENSE.txt", contents: "EULA\n" }] }); + const outputDir = resolve(data.directory, "output"); + await verifyAndExtract({ archive: data.archive, checksum: data.checksum, outputDir, sourceUrl: "https://example.test/archive", version: "0.73.1", flavor: "cpu" }); + assert.deepEqual( + await verifyDeclaredRuntimeFiles(resolve(outputDir, "native-runtimes/linux-cpu")), + ["licenses/NVIDIA-CUDA-LICENSE.txt"], + ); +}); + +test("rejects runtime files the runtime manifest never declared", async (t) => { + const data = await fixture(t, { licenses: [{ name: "SMUGGLED.txt", contents: "payload\n", declare: false }] }); + await assert.rejects( + verifyAndExtract({ archive: data.archive, checksum: data.checksum, outputDir: resolve(data.directory, "output"), sourceUrl: "https://example.test/archive", version: "0.73.1", flavor: "cpu" }), + /does not declare runtime file licenses\/SMUGGLED\.txt/, + ); +}); + +test("rejects runtime license bytes that drifted from the declared digest", async (t) => { + const data = await fixture(t, { licenses: [{ name: "NVIDIA-CUDA-LICENSE.txt", contents: "EULA\n", digest: "b".repeat(64) }] }); + await assert.rejects( + verifyAndExtract({ archive: data.archive, checksum: data.checksum, outputDir: resolve(data.directory, "output"), sourceUrl: "https://example.test/archive", version: "0.73.1", flavor: "cpu" }), + /licenses\/NVIDIA-CUDA-LICENSE\.txt does not match its declared digest/, + ); +}); + +test("makes no declaration demand of runtimes that ship no extra files", async (t) => { + const data = await fixture(t); + const outputDir = resolve(data.directory, "output"); + await verifyAndExtract({ archive: data.archive, checksum: data.checksum, outputDir, sourceUrl: "https://example.test/archive", version: "0.73.1", flavor: "cpu" }); + assert.deepEqual(await verifyDeclaredRuntimeFiles(resolve(outputDir, "native-runtimes/linux-cpu")), []); }); test("keeps sha256Tree digest compatible with the original tree format", async (t) => {