Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion TODO.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,15 @@
formula use the same 30-second bound, focused regression tests pass, the full
TypeScript suite remains green, and the v0.76.1 packaging recovery completes.

- [ ] Accept producer-declared native runtime license material in verified
- [x] Accept producer-declared native runtime license material in verified
product bundles so v0.76.1 CUDA packaging can complete without weakening the
existing checksum, entry-type, tree-digest, or single-runtime checks.
Final result: the archive listing accepts only a flat, safely named
`licenses/<name>`, and extraction then requires every runtime file outside
`lib/`, `tools/`, `manifest.json`, and `README.md` to be declared in the
runtime manifest's `runtime.files` map with a matching SHA-256. Runtimes that
ship no extra files make no declaration and are unaffected, so bundles
predating license bundling still verify.
QA: focused archive fixtures accept `licenses/NVIDIA-CUDA-LICENSE.txt`, the
full TypeScript suite passes, matrix validation remains green, actionlint
passes, and a protected v0.76.1 packaging rerun completes every required job.
Expand Down
61 changes: 60 additions & 1 deletion scripts/upstream-archive.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,24 @@ type Inputs = {
};

const runtimeIdPattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
const licenseNamePattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;

// Libraries and tools are addressed by the product manifest and the runtime
// tree digest. Anything else the producer ships (today: CUDA distribution
// license material) has to earn its place by being named in the runtime
// manifest, so the archive listing can stay a shape check and the digest check
// can happen after extraction.
function isRuntimeLicenseEntry(relative: string): boolean {
const match = /^licenses\/([^/]+)$/.exec(relative);
return match !== null && licenseNamePattern.test(match[1]);
}

function isStructuralRuntimeFile(relative: string): boolean {
return relative === "manifest.json"
|| relative === "README.md"
|| relative.startsWith("lib/")
|| relative.startsWith("tools/");
}

export function productBackendForFlavor(flavor: string): string {
if (flavor === "cuda-12" || flavor === "cuda-13") return "cuda";
Expand Down Expand Up @@ -67,7 +85,7 @@ export function validateArchiveEntries(entries: string[]): void {
if (!match) throw new Error(`unexpected product archive entry: ${path}`);
const [, runtimeId, relative] = match;
if (!runtimeIdPattern.test(runtimeId)) throw new Error(`unsafe native runtime id: ${runtimeId}`);
if (relative !== "manifest.json" && relative !== "README.md" && !relative.startsWith("lib/") && !relative.startsWith("licenses/") && !relative.startsWith("tools/")) {
if (!isStructuralRuntimeFile(relative) && !isRuntimeLicenseEntry(relative)) {
throw new Error(`unexpected native runtime entry: ${path}`);
}
runtimeIds.add(runtimeId);
Expand Down Expand Up @@ -148,6 +166,46 @@ function filesBelow(root: string, current = root): string[] {
});
}

function declaredRuntimeFileDigests(runtimeDir: string): Record<string, unknown> {
const raw = readFileSync(resolve(runtimeDir, "manifest.json"), "utf8");
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch {
throw new Error("native runtime manifest is not valid JSON");
}
const files = (parsed as { runtime?: { files?: unknown } })?.runtime?.files;
if (!files || typeof files !== "object" || Array.isArray(files)) return {};
return files as Record<string, unknown>;
}

/**
* Accepts only the extra runtime files the producer declared, and proves their
* bytes. The archive listing already bounded these to `licenses/<name>`; this
* is where an entry that nothing in the runtime manifest vouches for is
* rejected. Runtimes that ship no extra files make no claim and are left alone,
* which is what keeps releases predating the license bundling verifiable.
*/
export async function verifyDeclaredRuntimeFiles(runtimeDir: string): Promise<string[]> {
const extra = filesBelow(runtimeDir)
.map((path) => path.slice(runtimeDir.length + 1).replaceAll("\\", "/"))
.filter((relative) => !isStructuralRuntimeFile(relative))
.sort();
if (extra.length === 0) return [];
const declared = declaredRuntimeFileDigests(runtimeDir);
for (const relative of extra) {
const expected = declared[relative];
if (typeof expected !== "string" || !/^[0-9a-f]{64}$/.test(expected)) {
throw new Error(`native runtime manifest does not declare runtime file ${relative}`);
}
const actual = await sha256File(resolve(runtimeDir, relative));
if (actual !== expected) {
throw new Error(`native runtime file ${relative} does not match its declared digest`);
}
}
return extra;
}

export function sha256Tree(root: string): string {
const digest = createHash("sha256");
const buffer = Buffer.allocUnsafe(1024 * 1024);
Expand Down Expand Up @@ -210,6 +268,7 @@ export async function verifyAndExtract(input: Inputs) {
if (runtimeSha256 !== productManifest.runtime.sha256) throw new Error("product runtime digest does not match extracted runtime tree");
const runtimeManifestSha256 = await sha256File(resolve(runtime, "manifest.json"));
if (runtimeManifestSha256 !== productManifest.runtime.manifest_sha256) throw new Error("product runtime manifest digest does not match");
await verifyDeclaredRuntimeFiles(runtime);
const provenance = resolve(input.outputDir, "upstream-provenance.json");
writeFileSync(provenance, `${JSON.stringify({
archive: archiveName,
Expand Down
88 changes: 77 additions & 11 deletions tests/upstream-archive.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
validateArchiveEntryTypes,
validateProductManifest,
verifyAndExtract,
verifyDeclaredRuntimeFiles,
} from "../scripts/upstream-archive.ts";

test("maps versioned CUDA flavors to the product backend", () => {
Expand Down Expand Up @@ -44,7 +45,14 @@ function legacySha256Tree(root: string): string {
return digest.digest("hex");
}

async function fixture(t: { after(callback: () => void): void }) {
type FixtureOptions = {
// Extra runtime files staged under the runtime directory, and what the
// runtime manifest claims about them. `declare: false` stages the bytes
// without declaring them; `digest` overrides the declared digest.
licenses?: { name: string; contents: string; declare?: boolean; digest?: string }[];
};

async function fixture(t: { after(callback: () => void): void }, options: FixtureOptions = {}) {
const directory = mkdtempSync(resolve(tmpdir(), "upstream-test-"));
t.after(() => rmSync(directory, { recursive: true, force: true }));
const bundle = resolve(directory, "stage/mesh-bundle");
Expand All @@ -54,9 +62,26 @@ async function fixture(t: { after(callback: () => void): void }) {
writeFileSync(resolve(bundle, "host-imports.json"), "{}\n");
const runtime = resolve(bundle, "native-runtimes/linux-cpu");
mkdirSync(resolve(runtime, "lib"), { recursive: true });
writeFileSync(resolve(runtime, "manifest.json"), "{}\n");
writeFileSync(resolve(runtime, "README.md"), "runtime\n");
writeFileSync(resolve(runtime, "lib/libllama.so"), "runtime\n");
if (options.licenses?.length) {
mkdirSync(resolve(runtime, "licenses"), { recursive: true });
const files: Record<string, string> = {
"lib/libllama.so": await sha256File(resolve(runtime, "lib/libllama.so")),
};
for (const license of options.licenses) {
const relative = `licenses/${license.name}`;
writeFileSync(resolve(runtime, relative), license.contents);
if (license.declare === false) continue;
files[relative] = license.digest ?? await sha256File(resolve(runtime, relative));
}
writeFileSync(resolve(runtime, "manifest.json"), `${JSON.stringify({
build: { primary_library: "lib/libllama.so" },
runtime: { files, id: "linux-cpu", libraries: ["lib/libllama.so"] },
}, null, 2)}\n`);
} else {
writeFileSync(resolve(runtime, "manifest.json"), "{}\n");
}
const hostSha256 = await sha256File(resolve(bundle, "mesh-llm"));
const runtimeSha256 = sha256Tree(runtime);
const runtimeManifestSha256 = await sha256File(resolve(runtime, "manifest.json"));
Expand Down Expand Up @@ -123,16 +148,57 @@ test("rejects unsafe runtime IDs before accepting runtime paths", () => {
]), /runtime id/);
});

const cudaEntries = (license: string) => [
"mesh-bundle/mesh-llm",
"mesh-bundle/product-manifest.json",
"mesh-bundle/host-imports.json",
"mesh-bundle/native-runtimes/linux-cuda/manifest.json",
"mesh-bundle/native-runtimes/linux-cuda/README.md",
"mesh-bundle/native-runtimes/linux-cuda/lib/libllama.so",
`mesh-bundle/native-runtimes/linux-cuda/licenses/${license}`,
];

test("accepts bundled native runtime license material", () => {
assert.doesNotThrow(() => validateArchiveEntries([
"mesh-bundle/mesh-llm",
"mesh-bundle/product-manifest.json",
"mesh-bundle/host-imports.json",
"mesh-bundle/native-runtimes/linux-cuda/manifest.json",
"mesh-bundle/native-runtimes/linux-cuda/README.md",
"mesh-bundle/native-runtimes/linux-cuda/lib/libllama.so",
"mesh-bundle/native-runtimes/linux-cuda/licenses/NVIDIA-CUDA-LICENSE.txt",
]));
assert.doesNotThrow(() => validateArchiveEntries(cudaEntries("NVIDIA-CUDA-LICENSE.txt")));
});

test("keeps the license allowlist flat and safely named", () => {
for (const license of ["nested/EULA.txt", ".hidden", "-leading", "spaced name.txt"]) {
assert.throws(() => validateArchiveEntries(cudaEntries(license)), /unexpected native runtime entry/);
}
});

test("accepts producer-declared runtime license files and reports them", async (t) => {
const data = await fixture(t, { licenses: [{ name: "NVIDIA-CUDA-LICENSE.txt", contents: "EULA\n" }] });
const outputDir = resolve(data.directory, "output");
await verifyAndExtract({ archive: data.archive, checksum: data.checksum, outputDir, sourceUrl: "https://example.test/archive", version: "0.73.1", flavor: "cpu" });
assert.deepEqual(
await verifyDeclaredRuntimeFiles(resolve(outputDir, "native-runtimes/linux-cpu")),
["licenses/NVIDIA-CUDA-LICENSE.txt"],
);
});

test("rejects runtime files the runtime manifest never declared", async (t) => {
const data = await fixture(t, { licenses: [{ name: "SMUGGLED.txt", contents: "payload\n", declare: false }] });
await assert.rejects(
verifyAndExtract({ archive: data.archive, checksum: data.checksum, outputDir: resolve(data.directory, "output"), sourceUrl: "https://example.test/archive", version: "0.73.1", flavor: "cpu" }),
/does not declare runtime file licenses\/SMUGGLED\.txt/,
);
});

test("rejects runtime license bytes that drifted from the declared digest", async (t) => {
const data = await fixture(t, { licenses: [{ name: "NVIDIA-CUDA-LICENSE.txt", contents: "EULA\n", digest: "b".repeat(64) }] });
await assert.rejects(
verifyAndExtract({ archive: data.archive, checksum: data.checksum, outputDir: resolve(data.directory, "output"), sourceUrl: "https://example.test/archive", version: "0.73.1", flavor: "cpu" }),
/licenses\/NVIDIA-CUDA-LICENSE\.txt does not match its declared digest/,
);
});

test("makes no declaration demand of runtimes that ship no extra files", async (t) => {
const data = await fixture(t);
const outputDir = resolve(data.directory, "output");
await verifyAndExtract({ archive: data.archive, checksum: data.checksum, outputDir, sourceUrl: "https://example.test/archive", version: "0.73.1", flavor: "cpu" });
assert.deepEqual(await verifyDeclaredRuntimeFiles(resolve(outputDir, "native-runtimes/linux-cpu")), []);
});

test("keeps sha256Tree digest compatible with the original tree format", async (t) => {
Expand Down