From a863d57aeb7cfda25daf4bd2062592c30298a308 Mon Sep 17 00:00:00 2001 From: Ashok Gelal Date: Thu, 19 Nov 2026 16:30:44 -0500 Subject: [PATCH] feat: list the fine-tuning trainer in the v3 dev runtime feed The trainer (msty-nexus-trainer) is read from its dev test publish at runtime/trainer/latest/dev/ and pinned by SHA-256. It is v3 only, because Runtime 0.9.1 and earlier reject a feed with a runtime they do not know, and dev only until it has a production publish. --- scripts/runtime-feed-lib.mjs | 67 ++++++++++++++++++++++++++++++++++- scripts/runtime-feed.test.mjs | 49 ++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/scripts/runtime-feed-lib.mjs b/scripts/runtime-feed-lib.mjs index 715b03c..45c15af 100644 --- a/scripts/runtime-feed-lib.mjs +++ b/scripts/runtime-feed-lib.mjs @@ -22,6 +22,11 @@ export const DEFAULT_FOUNDATIONMODELS_ASSET_URL = export const FOUNDATIONMODELS_CHANNELS = new Set(["dev"]); export const DEFAULT_IMAGES_ASSET_URL = "https://nexus-assets.msty.ai/msty-nexus/runtime/images/latest/msty-nexus-images.tgz"; +// The fine-tuning worker has a dev test lane only (latest/dev/). Add "stable" +// once it has a production publish. +export const TRAINER_CHANNELS = new Set(["dev"]); +export const DEFAULT_TRAINER_ASSET_URL = + "https://nexus-assets.msty.ai/msty-nexus/runtime/trainer/latest/msty-nexus-trainer.tgz"; // The feed is published at two paths with the same schema and signing key. // v2 is read by Msty Nexus Runtime 0.9.1 and earlier, which reject the whole @@ -31,7 +36,7 @@ export const DEFAULT_IMAGES_ASSET_URL = // New runtimes, fields and artifact kinds go to v3 only. export const FEED_VERSIONS = [2, 3]; const V2_RUNTIME_IDS = ["ollama", "llamacpp", "mlx", "foundationmodels"]; -const V3_ONLY_RUNTIME_IDS = ["images"]; +const V3_ONLY_RUNTIME_IDS = ["images", "trainer"]; export function parseFeedVersion(value) { const version = Number(value); @@ -489,6 +494,55 @@ async function buildImagesRelease(fetchImpl, channel, publishedAt, assetURL) { }; } +// The fine-tuning worker (msty-nexus-trainer) is read from its dev test +// publish (latest/dev/), which has no versioned copy and is pinned by +// SHA-256. Its archive has no top folder: the Runtime launches +// /bin/msty-nexus-trainer. Returns null only when the manifest +// is not published yet (HTTP 404). Any other problem fails the refresh. +async function buildTrainerRelease(fetchImpl, channel, publishedAt, assetURL) { + const parsed = new URL(assetURL); + invariant(parsed.protocol === "https:" && parsed.search === "" && parsed.hash === "", "Trainer runtime asset URL must be immutable HTTPS metadata input"); + invariant(parsed.pathname.endsWith("/trainer/latest/msty-nexus-trainer.tgz"), "Trainer runtime asset URL must be the latest msty-nexus-trainer.tgz alias"); + invariant(channel === "dev", "The trainer runtime has a dev lane only"); + const devURL = assetURL.replace("/trainer/latest/", "/trainer/latest/dev/"); + const prefix = devURL.slice(0, -"msty-nexus-trainer.tgz".length); + const probe = await fetchImpl(`${prefix}runtime-manifest.json`, { method: "HEAD", redirect: "follow" }); + if (probe.status === 404) return null; + const manifestResponse = await fetchWithRetry(`${prefix}runtime-manifest.json`, { fetchImpl }); + const manifest = object(await manifestResponse.json(), "Trainer runtime manifest"); + invariant( + manifest.object === "msty.nexus.trainer_runtime" && manifest.runtimeId === "trainer", + "Trainer runtime manifest identity is invalid", + ); + const version = string(manifest.runtimeVersion, "Trainer runtime version"); + invariant(SEMVER.test(version), "Trainer runtime version must be semantic"); + const checksumResponse = await fetchWithRetry(`${prefix}checksums.txt`, { fetchImpl }); + const checksumText = await checksumResponse.text(); + const checksumLine = checksumText.split(/\r?\n/).find((line) => line.trim().endsWith("msty-nexus-trainer.tgz")); + const sha256 = String(checksumLine ?? "").trim().split(/\s+/)[0]?.toLowerCase(); + invariant(SHA256.test(sha256), "Trainer runtime checksum file did not contain a valid archive SHA-256"); + const head = await fetchWithRetry(devURL, { fetchImpl, method: "HEAD" }); + const sizeBytes = Number(head.headers.get("content-length")); + integer(sizeBytes, "Trainer runtime artifact content-length"); + return { + runtimeId: "trainer", + channel, + runtimeVersion: version, + publishedAt, + source: { type: "msty_asset", url: devURL }, + artifacts: [{ + os: "darwin", + arch: "arm64", + backend: "metal", + url: devURL, + sha256, + sizeBytes, + archiveFormat: "tar.gz", + executablePath: "bin/msty-nexus-trainer", + }], + }; +} + async function readMLXMetadata(fetchImpl, prefix) { const manifestResponse = await fetchWithRetry(`${prefix}runtime-manifest.json`, { fetchImpl }); const manifest = object(await manifestResponse.json(), "MLX runtime manifest"); @@ -569,6 +623,8 @@ export async function buildRuntimeFeed({ foundationModelsChannels = FOUNDATIONMODELS_CHANNELS, feedVersion = 2, imagesAssetURL = DEFAULT_IMAGES_ASSET_URL, + trainerAssetURL = DEFAULT_TRAINER_ASSET_URL, + trainerChannels = TRAINER_CHANNELS, }) { feedVersion = parseFeedVersion(feedVersion); invariant(channel === "stable" || channel === "dev", "runtime feed channel must be stable or dev"); @@ -588,6 +644,11 @@ export async function buildRuntimeFeed({ const images = await buildImagesRelease(fetchImpl, channel, publishedAt, imagesAssetURL); if (images) releases.push(images); } + // Like images, the trainer is v3 only; it also has a dev lane only. + if (feedVersion >= 3 && trainerChannels.has(channel)) { + const trainer = await buildTrainerRelease(fetchImpl, channel, publishedAt, trainerAssetURL); + if (trainer) releases.push(trainer); + } const feed = { schemaVersion: FEED_SCHEMA_VERSION, generatedAt: publishedAt, releases }; validateRuntimeFeed(feed, channel, { feedVersion }); return feed; @@ -610,6 +671,10 @@ export function validateRuntimeFeed(feed, expectedChannel, { feedVersion = 2 } = runtimeId !== "foundationmodels" || FOUNDATIONMODELS_CHANNELS.has(expectedChannel), `foundationmodels is not enabled for the ${expectedChannel} channel`, ); + invariant( + runtimeId !== "trainer" || TRAINER_CHANNELS.has(expectedChannel), + `trainer is not enabled for the ${expectedChannel} channel`, + ); invariant(!byID.has(runtimeId), `duplicate runtime ${runtimeId}`); byID.set(runtimeId, release); invariant(release.channel === expectedChannel, `${runtimeId}.channel must be ${expectedChannel}`); diff --git a/scripts/runtime-feed.test.mjs b/scripts/runtime-feed.test.mjs index b73f87d..8c4f95f 100644 --- a/scripts/runtime-feed.test.mjs +++ b/scripts/runtime-feed.test.mjs @@ -72,8 +72,21 @@ function fixtureGitHub() { }; } -function fixtureFetch({ foundationModelsPublished = false, images = "missing", imagesStatus = 500, mlxDev } = {} = {}) { +function fixtureFetch({ foundationModelsPublished = false, images = "missing", imagesStatus = 500, mlxDev, trainer = "missing" } = {} = {}) { return async (url, options = {}) => { + if (url.includes("/trainer/")) { + if (trainer === "missing") return new Response(null, { status: 404 }); + if (url.endsWith("runtime-manifest.json")) { + const manifest = trainer === "malformed" + ? { object: "msty.nexus.mlx_runtime", runtimeId: "trainer", runtimeVersion: "0.2.0-dev.1" } + : { object: "msty.nexus.trainer_runtime", runtimeId: "trainer", runtimeVersion: "0.2.0-dev.1" }; + return new Response(options.method === "HEAD" ? null : JSON.stringify(manifest)); + } + if (url.endsWith("checksums.txt")) return new Response(`${sha("c")} msty-nexus-trainer.tgz\n`); + if (options.method === "HEAD" && url.endsWith("msty-nexus-trainer.tgz")) { + return new Response(null, { headers: { "content-length": "126000000" } }); + } + } // mlxDev: undefined serves the production manifest on latest/dev/ too, // "missing" serves 404, and a version string serves a dev build. if (url.includes("/mlx/latest/dev/") && mlxDev !== undefined) { @@ -344,3 +357,37 @@ test("v3 leaves the image runtime out only when its manifest is missing", async await assert.rejects(build({ fetchImpl: fixtureFetch({ images: "no-checksum" }), feedVersion: 3 }), /Image runtime checksum file/); await assert.rejects(build({ fetchImpl: fixtureFetch({ images: "error", imagesStatus: 403 }), feedVersion: 3 }), /HTTP 403/); }); + +test("v3 dev lists the trainer from its dev publish; stable and v2 never ask", async () => { + const published = fixtureFetch({ trainer: "published" }); + const dev = await build({ fetchImpl: published, channel: "dev", feedVersion: 3 }); + const trainer = dev.releases.find((release) => release.runtimeId === "trainer"); + const url = "https://nexus-assets.msty.ai/msty-nexus/runtime/trainer/latest/dev/msty-nexus-trainer.tgz"; + assert.deepEqual(trainer.artifacts, [{ + os: "darwin", + arch: "arm64", + backend: "metal", + url, + sha256: sha("c"), + sizeBytes: 126000000, + archiveFormat: "tar.gz", + executablePath: "bin/msty-nexus-trainer", + }]); + assert.equal(trainer.runtimeVersion, "0.2.0-dev.1"); + + for (const [channel, feedVersion] of [["stable", 3], ["dev", 2], ["stable", 2]]) { + const fetchImpl = async (target, options) => { + assert.equal(target.includes("/trainer/"), false, `${channel} v${feedVersion} must not ask for the trainer`); + return published(target, options); + }; + assert.equal(runtimeIds(await build({ fetchImpl, channel, feedVersion })).includes("trainer"), false); + } + + assert.equal(runtimeIds(await build({ fetchImpl: fixtureFetch(), channel: "dev", feedVersion: 3 })).includes("trainer"), false); + await assert.rejects(build({ fetchImpl: fixtureFetch({ trainer: "malformed" }), channel: "dev", feedVersion: 3 }), /Trainer runtime manifest identity is invalid/); + assert.throws(() => validateRuntimeFeed(dev, "dev"), /unsupported runtime trainer in a v2 feed/); + const stable = structuredClone(dev); + for (const release of stable.releases) release.channel = "stable"; + stable.releases = stable.releases.filter((release) => release.runtimeId !== "foundationmodels"); + assert.throws(() => validateRuntimeFeed(stable, "stable", { feedVersion: 3 }), /trainer is not enabled for the stable channel/); +});