diff --git a/README.md b/README.md index 10b852d..094fbda 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,29 @@ It loads published graph JSON directly from Vercel Blob, verifies its digest, and keeps selection/filter state in linkable URL fragments. It does not embed generated HTML or accept population datasets. +The provider's `/browser` package owns bounded document downloads, metadata and +digest validation, and Orrery parsing. Its `/server` package reads completed +publication records and resolves only listed evidence through this app's storage +adapter. The app retains Next routes, the React component, URL state, collector +authorization, immutable writes and Vercel Blob credentials. It does not start a +Python service or move the calibration dashboard's telemetry. + +## Provider package + +The app depends on one exact npm release of +`@policyengine/microcosm-provider-orrery`, which PolicyEngine/microcosm-emitter +publishes automatically on every merge to its `main`. The package contains +compiled JavaScript and declarations, so the app needs no transpilation or +resolver override for it. To adopt a new release, change the exact version, +regenerate `bun.lock` and rerun every check below. + +No new environment variables, secrets, stores or endpoints are introduced by this +migration. Existing provisioning remains unchanged. Provider contract and browser +reader tests live in the provider repository; this app retains its own HTTP, +write-service, authorization, URL-state and provisioning tests using synthetic +data and in-memory storage. Neither repository adds a test of a Microcosm build +against the runs app. + ## Publication API 1. POST a version-1 file inventory to `/api/uploads/` with a short-lived collector diff --git a/app/runs/[id]/page.tsx b/app/runs/[id]/page.tsx index ba7c096..30297d1 100644 --- a/app/runs/[id]/page.tsx +++ b/app/runs/[id]/page.tsx @@ -1,4 +1,4 @@ -import { publicationId } from "@/lib/contract"; +import { publicationId } from "@policyengine/microcosm-provider-orrery/contract"; import RunViewer from "@/components/RunViewer"; import { notFound } from "next/navigation"; export default async function Page({ diff --git a/bun.lock b/bun.lock index bb8437c..5504521 100644 --- a/bun.lock +++ b/bun.lock @@ -6,6 +6,7 @@ "name": "microcosm-runs", "dependencies": { "@axiom-foundation/orrery": "0.6.1", + "@policyengine/microcosm-provider-orrery": "0.2.1", "@policyengine/ui-kit": "^0.4.0", "@vercel/blob": "^2.8.1", "next": "^16.2.6", @@ -204,6 +205,8 @@ "@nolyfill/is-core-module": ["@nolyfill/is-core-module@1.0.39", "", {}, "sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA=="], + "@policyengine/microcosm-provider-orrery": ["@policyengine/microcosm-provider-orrery@0.2.1", "", { "peerDependencies": { "@axiom-foundation/orrery": "0.6.1" } }, "sha512-kJuxUV/qoVN24Ac8g7g7sSfrZi8FwbOVMBNgSGbTWFlt5VLZlRLHVypJLJofLyeMGi464OkloX5JqRu87sEjOA=="], + "@policyengine/ui-kit": ["@policyengine/ui-kit@0.4.0", "", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", "d3-geo": "^3.1.0", "dayjs": "^1.11.13", "lucide-react": "^0.577.0", "radix-ui": "^1.4.3", "tailwind-merge": "^3.5.0", "tw-animate-css": "^1.4.0" }, "peerDependencies": { "react": ">=19", "react-dom": ">=19", "recharts": ">=2.12" } }, "sha512-Hjb1WzJNbboSeIn9/4GRFapmTUnDOU6RHzjnBU7fDfGPNT6XJ01LQAJ5k+MEMFoH1NX6iZtQeu/P3Km4oJ3/hw=="], "@radix-ui/number": ["@radix-ui/number@1.1.3", "", {}, "sha512-Road2bidD0uu/1BGDOWNdPI06g0lIRy6IF9GZcIrDK2KGItfor8IQwQa+yM2ERgHM1MmHxaxpTzk0/Jp42lNfA=="], diff --git a/changelog.d/provider-graph-readers.changed.md b/changelog.d/provider-graph-readers.changed.md new file mode 100644 index 0000000..219d446 --- /dev/null +++ b/changelog.d/provider-graph-readers.changed.md @@ -0,0 +1 @@ +Read and validate graph publications with the `@policyengine/microcosm-provider-orrery` package (0.2.1) instead of a copy of its contract and readers in this app. The viewer, routes and publication behaviour are unchanged. diff --git a/components/RunViewer.tsx b/components/RunViewer.tsx index 0a54a8f..e52ade5 100644 --- a/components/RunViewer.tsx +++ b/components/RunViewer.tsx @@ -6,7 +6,7 @@ import { type GraphDocument, } from "@axiom-foundation/orrery"; import { Orrery } from "@axiom-foundation/orrery/react"; -import { loadDocument } from "@/lib/viewer"; +import { loadDocument } from "@policyengine/microcosm-provider-orrery/browser"; function subscribe(callback: () => void) { window.addEventListener("hashchange", callback); diff --git a/lib/auth.ts b/lib/auth.ts index c50f3a0..7382466 100644 --- a/lib/auth.ts +++ b/lib/auth.ts @@ -1,4 +1,8 @@ -import { ApiError, inventoryDigest, type Inventory } from "./contract"; +import { + ApiError, + type Inventory, +} from "@policyengine/microcosm-provider-orrery/contract"; +import { inventoryDigest } from "@policyengine/microcosm-provider-orrery/server"; import type { Fetcher } from "./net"; export const PRODUCTION_COLLECTOR_ORIGIN = diff --git a/lib/contract.ts b/lib/contract.ts deleted file mode 100644 index 0fb953b..0000000 --- a/lib/contract.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { createHash } from "node:crypto"; - -export const MAX_FILE_BYTES = 64 * 1024 * 1024; -export const MAX_TOTAL_BYTES = 512 * 1024 * 1024; -export const MAX_FILES = 256; -export type FileRole = - | "graph" - | "schema" - | "index" - | "declaration" - | "manifest" - | "binding" - | "summaries" - | "upstream"; -export interface RunFile { - name: string; - role: FileRole; - bytes: number; - sha256: string; -} -export interface Inventory { - version: 1; - publication_id: string; - files: RunFile[]; -} -export interface Publication extends Inventory { - inventory_sha256: string; - published_at: string; - files: (RunFile & { url: string })[]; -} -export class ApiError extends Error { - constructor( - public status: number, - public code: string, - ) { - super(code); - } -} -export function publicationId(value: unknown): string { - if ( - typeof value !== "string" || - !/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(value) - ) - throw new ApiError(400, "invalid_publication_id"); - return value; -} -const roleNames: Record = { - graph: /^graph\.orrery\.json$/, - schema: /^graph\.schema\.json$/, - index: /^execution\.evidence\.json$/, - declaration: /^evidence-[a-zA-Z0-9_-]+-graph\.json$/, - manifest: /^evidence-[a-zA-Z0-9_-]+-manifest\.json$/, - binding: /^evidence-[a-zA-Z0-9_-]+-binding\.json$/, - summaries: /^evidence-[a-zA-Z0-9_-]+-summaries\.json$/, - upstream: /^upstream-[a-f0-9]{64}\.json$/, -}; -export function inventory(raw: unknown): Inventory { - if (!raw || typeof raw !== "object") - throw new ApiError(400, "invalid_inventory"); - const data = raw as Record; - const id = publicationId(data.publication_id); - if ( - data.version !== 1 || - !Array.isArray(data.files) || - data.files.length === 0 || - data.files.length > MAX_FILES - ) - throw new ApiError(400, "invalid_inventory"); - const names = new Set(); - let total = 0; - const files: RunFile[] = data.files.map((raw) => { - if (!raw || typeof raw !== "object") - throw new ApiError(400, "invalid_file"); - const f = raw as RunFile; - if ( - !Object.hasOwn(roleNames, f.role) || - typeof f.name !== "string" || - !roleNames[f.role].test(f.name) || - names.has(f.name) || - !Number.isSafeInteger(f.bytes) || - f.bytes <= 0 || - f.bytes > MAX_FILE_BYTES || - typeof f.sha256 !== "string" || - !/^[a-f0-9]{64}$/.test(f.sha256) - ) - throw new ApiError(400, "invalid_file"); - names.add(f.name); - total += f.bytes; - return { name: f.name, role: f.role, bytes: f.bytes, sha256: f.sha256 }; - }); - if ( - total > MAX_TOTAL_BYTES || - files.filter((f) => f.role === "graph").length !== 1 - ) - throw new ApiError(400, "invalid_inventory"); - return { - version: 1, - publication_id: id, - files: files.sort((a, b) => - a.name < b.name ? -1 : a.name > b.name ? 1 : 0, - ), - }; -} -export function sha256(bytes: string | Uint8Array): string { - return createHash("sha256").update(bytes).digest("hex"); -} -export function inventoryDigest(value: Inventory): string { - return sha256(JSON.stringify(inventory(value))); -} -export function objectPath(value: Inventory, name: string): string { - return `objects/${value.publication_id}/${inventoryDigest(value)}/${name}`; -} -export function publicationPath(id: string): string { - return `publications/${publicationId(id)}.json`; -} diff --git a/lib/http.ts b/lib/http.ts index d41a830..299ac49 100644 --- a/lib/http.ts +++ b/lib/http.ts @@ -1,4 +1,7 @@ -import { ApiError, inventory } from "./contract"; +import { + ApiError, + inventory, +} from "@policyengine/microcosm-provider-orrery/contract"; import { authorize } from "./auth"; import { createService } from "./service"; import { blobStorage, type Storage } from "./storage"; diff --git a/lib/service.ts b/lib/service.ts index d727ae4..03c957c 100644 --- a/lib/service.ts +++ b/lib/service.ts @@ -2,15 +2,17 @@ import { parseGraphDocument } from "@axiom-foundation/orrery"; import { ApiError, inventory, + type Inventory, + type Publication, +} from "@policyengine/microcosm-provider-orrery/contract"; +import { inventoryDigest, objectPath, publicationPath, - publicationId, sha256, - type Inventory, - type Publication, -} from "./contract"; +} from "@policyengine/microcosm-provider-orrery/server"; import type { Storage } from "./storage"; +import { createPublicationReader } from "@policyengine/microcosm-provider-orrery/server"; const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value)); @@ -35,10 +37,7 @@ export function checkSources(value: unknown, names: Set): void { } } export function createService(storage: Storage, subject: string) { - async function read(id: string): Promise { - const object = await storage.read(publicationPath(id), 256 * 1024); - return object ? (decode(object.bytes) as Publication) : null; - } + const { read, evidence } = createPublicationReader(storage); async function reserve(value: Inventory) { if (!subject) throw new ApiError(401, "unauthorized"); const claim = { subject, inventory_sha256: inventoryDigest(value) }; @@ -132,9 +131,6 @@ export function createService(storage: Storage, subject: string) { throw new ApiError(409, "publication_conflict"); return winner; }, - async evidence(id: string, name: string) { - const published = await read(publicationId(id)); - return published?.files.find((f) => f.name === name)?.url ?? null; - }, + evidence, }; } diff --git a/lib/storage.ts b/lib/storage.ts index 031ee0b..9a088fa 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -1,5 +1,5 @@ import { get, put, issueSignedToken, presignUrl } from "@vercel/blob"; -import { ApiError } from "./contract"; +import { ApiError } from "@policyengine/microcosm-provider-orrery/contract"; export interface Storage { read( diff --git a/lib/viewer.ts b/lib/viewer.ts deleted file mode 100644 index 9b78d79..0000000 --- a/lib/viewer.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { parseGraphDocument } from "@axiom-foundation/orrery"; -import type { Publication } from "./contract"; -import type { Fetcher } from "./net"; - -export async function loadDocument( - id: string, - signal: AbortSignal, - fetcher: Fetcher = fetch, -) { - const response = await fetcher(`/api/runs/${encodeURIComponent(id)}/`, { - signal, - }); - if (response.status === 404) - throw new Error("This graph publication was not found."); - if (!response.ok) - throw new Error("Graph publication metadata is unavailable."); - const publication: Publication = await response.json(); - const file = publication.files?.find( - (f) => f.role === "graph" && f.name === "graph.orrery.json", - ); - if ( - !file || - publication.publication_id !== id || - file.bytes > 64 * 1024 * 1024 - ) - throw new Error("Invalid graph publication metadata."); - const url = new URL(file.url); - if ( - url.protocol !== "https:" || - !url.hostname.endsWith(".public.blob.vercel-storage.com") || - url.username || - url.password - ) - throw new Error("Invalid graph storage URL."); - const graph = await fetcher(url, { signal }); - if (!graph.ok || !graph.body) - throw new Error("The stored graph is unavailable."); - const reader = graph.body.getReader(); - const chunks: Uint8Array[] = []; - let length = 0; - try { - while (true) { - const { value, done } = await reader.read(); - if (done) break; - length += value.length; - if (length > file.bytes) - throw new Error("Graph size does not match its publication."); - chunks.push(value); - } - } finally { - await reader.cancel(); - } - if (length !== file.bytes) - throw new Error("Graph size does not match its publication."); - const bytes = new Uint8Array(length); - let offset = 0; - for (const chunk of chunks) { - bytes.set(chunk, offset); - offset += chunk.length; - } - const digest = Array.from( - new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)), - (v) => v.toString(16).padStart(2, "0"), - ).join(""); - if (digest !== file.sha256) - throw new Error("Graph digest does not match its publication."); - return { - document: parseGraphDocument( - JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)), - ), - digest, - }; -} diff --git a/package.json b/package.json index 6bb3460..08a7a4d 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "@axiom-foundation/orrery": "0.6.1", + "@policyengine/microcosm-provider-orrery": "0.2.1", "@policyengine/ui-kit": "^0.4.0", "@vercel/blob": "^2.8.1", "next": "^16.2.6", diff --git a/scripts/qualify.ts b/scripts/qualify.ts index 10c1f87..ef0a662 100644 --- a/scripts/qualify.ts +++ b/scripts/qualify.ts @@ -1,7 +1,11 @@ // Explicit, non-visual deployment qualification. Never imports Microcosm code. import { randomUUID } from "node:crypto"; import { parseGraphDocument } from "@axiom-foundation/orrery"; -import { inventory, inventoryDigest, sha256 } from "../lib/contract"; +import { inventory } from "@policyengine/microcosm-provider-orrery/contract"; +import { + inventoryDigest, + sha256, +} from "@policyengine/microcosm-provider-orrery/server"; import { createService } from "../lib/service"; import { blobStorage } from "../lib/storage"; diff --git a/tests/auth.test.ts b/tests/auth.test.ts index 936930f..a4e88eb 100644 --- a/tests/auth.test.ts +++ b/tests/auth.test.ts @@ -1,7 +1,8 @@ import { expect, test } from "bun:test"; import { authorize, collectorOrigin } from "../lib/auth"; import type { Fetcher } from "../lib/net"; -import { inventory, inventoryDigest } from "../lib/contract"; +import { inventory } from "@policyengine/microcosm-provider-orrery/contract"; +import { inventoryDigest } from "@policyengine/microcosm-provider-orrery/server"; const value = inventory({ version: 1, publication_id: "standalone", diff --git a/tests/contract.test.ts b/tests/contract.test.ts deleted file mode 100644 index 8f5ce37..0000000 --- a/tests/contract.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { - inventory, - inventoryDigest, - objectPath, - MAX_FILE_BYTES, -} from "../lib/contract"; - -const graph = { - name: "graph.orrery.json", - role: "graph", - bytes: 100, - sha256: "a".repeat(64), -}; -const valid = { version: 1, publication_id: "uk-run-123", files: [graph] }; -describe("publication inventory", () => { - test("canonical order and content-addressed paths", () => { - const evidence = { - ...graph, - name: "execution.evidence.json", - role: "index", - }; - const a = inventory({ ...valid, files: [graph, evidence] }); - const b = inventory({ ...valid, files: [evidence, graph] }); - expect(inventoryDigest(a)).toBe(inventoryDigest(b)); - expect(objectPath(a, graph.name)).toBe( - `objects/uk-run-123/${inventoryDigest(a)}/graph.orrery.json`, - ); - }); - for (const value of [ - { ...valid, version: 2 }, - { ...valid, publication_id: "../bad" }, - { ...valid, files: [graph, graph] }, - { ...valid, files: [{ ...graph, name: "../graph.orrery.json" }] }, - { ...valid, files: [{ ...graph, bytes: MAX_FILE_BYTES + 1 }] }, - { ...valid, files: [{ ...graph, bytes: -1 }] }, - { ...valid, files: [{ ...graph, sha256: "wrong" }] }, - { ...valid, files: [{ ...graph, name: "private.h5", role: "manifest" }] }, - { - ...valid, - files: [{ ...graph, name: "unreviewed.json", role: "manifest" }], - }, - { ...valid, files: [] }, - ]) - test(`rejects ${JSON.stringify(value).slice(0, 100)}`, () => - expect(() => inventory(value)).toThrow()); -}); diff --git a/tests/http.test.ts b/tests/http.test.ts index 8c9cf57..ea2fd8a 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -1,6 +1,7 @@ import { expect, test } from "bun:test"; import { write } from "../lib/http"; -import { ApiError, objectPath } from "../lib/contract"; +import { ApiError } from "@policyengine/microcosm-provider-orrery/contract"; +import { objectPath } from "@policyengine/microcosm-provider-orrery/server"; import { syntheticPublication } from "../scripts/qualify"; import type { Storage } from "../lib/storage"; diff --git a/tests/service.test.ts b/tests/service.test.ts index dec56a8..5f300ae 100644 --- a/tests/service.test.ts +++ b/tests/service.test.ts @@ -3,11 +3,13 @@ import { createService } from "../lib/service"; import { ApiError, inventory, + type Inventory, +} from "@policyengine/microcosm-provider-orrery/contract"; +import { objectPath, publicationPath, sha256, - type Inventory, -} from "../lib/contract"; +} from "@policyengine/microcosm-provider-orrery/server"; import type { Storage } from "../lib/storage"; const encoder = new TextEncoder(); @@ -141,3 +143,26 @@ test("publications require no run id and reserved ids cannot change content", as ); expect(Object.hasOwn(f.value, "run_id")).toBe(false); }); + +test("read and evidence routes use validated completed-publication metadata", async () => { + const f = fixture(); + f.upload(); + const published = await f.service.publish(f.value); + expect(await f.service.read(f.value.publication_id)).toEqual(published); + expect( + await f.service.evidence(f.value.publication_id, "graph.orrery.json"), + ).toBe(published.files[0].url); + expect( + await f.service.evidence(f.value.publication_id, "private.h5"), + ).toBeNull(); + expect(await f.service.read("unpublished")).toBeNull(); + f.data.set( + publicationPath(f.value.publication_id), + encoder.encode( + JSON.stringify({ ...published, inventory_sha256: "a".repeat(64) }), + ), + ); + await expect(f.service.read(f.value.publication_id)).rejects.toThrow( + "inventory_digest_mismatch", + ); +}); diff --git a/tests/viewer.test.ts b/tests/viewer.test.ts index cc9983b..f1591fe 100644 --- a/tests/viewer.test.ts +++ b/tests/viewer.test.ts @@ -1,70 +1,5 @@ import { expect, test } from "bun:test"; import { encodeLocation, decodeLocation } from "@axiom-foundation/orrery"; -import { loadDocument } from "../lib/viewer"; -import type { Fetcher } from "../lib/net"; -import { sha256 } from "../lib/contract"; -const document = { - schemaVersion: "graph-explorer/v1", - id: "fixture", - title: "Fixture", - nodes: [], - edges: [], -}; -test("viewer downloads exact graph bytes directly from Blob", async () => { - const bytes = JSON.stringify(document); - const calls: string[] = []; - const fetcher = (async (url) => { - calls.push(String(url)); - return calls.length === 1 - ? Response.json({ - publication_id: "graph", - files: [ - { - name: "graph.orrery.json", - role: "graph", - bytes: new TextEncoder().encode(bytes).length, - sha256: sha256(bytes), - url: "https://example.public.blob.vercel-storage.com/graph.json", - }, - ], - }) - : new Response(bytes); - }) as Fetcher; - expect( - (await loadDocument("graph", new AbortController().signal, fetcher)) - .document.id, - ).toBe("fixture"); - expect(calls[1]).toBe( - "https://example.public.blob.vercel-storage.com/graph.json", - ); -}); -test("viewer handles missing publications and rejects mismatched byte digests", async () => { - await expect( - loadDocument( - "missing", - new AbortController().signal, - (async () => new Response("", { status: 404 })) as Fetcher, - ), - ).rejects.toThrow("not found"); - let call = 0; - await expect( - loadDocument("graph", new AbortController().signal, (async () => - ++call === 1 - ? Response.json({ - publication_id: "graph", - files: [ - { - name: "graph.orrery.json", - role: "graph", - bytes: 2, - sha256: "a".repeat(64), - url: "https://example.public.blob.vercel-storage.com/graph.json", - }, - ], - }) - : new Response("{}")) as Fetcher), - ).rejects.toThrow("digest"); -}); test("Orrery deep links preserve selections and filters", () => { const location = { selectedId: "node/1",