From e35d57d7b49270bad5d2a0287535138a273322db Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Tue, 15 Sep 2026 11:29:15 -0400 Subject: [PATCH 1/8] =?UTF-8?q?=E2=9C=A8=20Add=20source-bundle=20workflow?= =?UTF-8?q?=20identity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A workflow definition can now be the source itself. The v2 descriptor addresses exact bytes by logical path, so a file outside a repository, an untracked file and a modified tracked file all have one immutable identity before anything retains them. Identity is two domain-separated, length-framed SHA-256 computations over the entrypoint, the canonical source manifest and the component mapping, through the platform's own `crypto`. The exact target stays outside the bundle hash and inside the definition: selecting a section does not change the bytes in the bundle. Host paths, props and retrieval metadata are not members of the shape at all. Parsing is closed and canonical. Logical paths are NFC scalar values without NUL, C0, DEL, backslash or `#`; manifests arrive in UTF-8 byte order with no duplicates, and a parser refuses a malformed order rather than sorting it. Snapshot verification copies the caller's buffers before checking them, so a later mutation of a caller-owned array cannot change what a run would retain. `WorkflowDefinition` stays v1-only here, so existing storage and runtime code remains coherent; storage, lifecycle, journals and artifacts become version-aware in the next commit. Refs #443 --- packages/workflow/mod.ts | 18 + .../workflow/src/storage/source-bundle.ts | 670 ++++++++++++++++++ .../tests/workflow-definition.test.ts | 496 +++++++++++++ 3 files changed, 1184 insertions(+) create mode 100644 packages/workflow/src/storage/source-bundle.ts diff --git a/packages/workflow/mod.ts b/packages/workflow/mod.ts index 37976145a..9f4112870 100644 --- a/packages/workflow/mod.ts +++ b/packages/workflow/mod.ts @@ -356,6 +356,24 @@ export type { WorkflowDefinition, } from "./src/storage/definition.ts"; +export { + decodeSourceText, + parseSourceBundleDefinition, + sourceBundleComponents, + sourceBundleDefinitionToJson, + sourceBundleHash, + sourceContentHash, + verifySourceBundleDefinition, + verifySourceBundleSnapshot, +} from "./src/storage/source-bundle.ts"; +export type { + SourceBundleComponentV2, + SourceBundleEntryV2, + SourceBundleIdentityV2, + SourceBundleSnapshotEntryV2, + SourceBundleWorkflowDefinitionV2, +} from "./src/storage/source-bundle.ts"; + export { conflictingFields } from "./src/storage/compatibility.ts"; export { diff --git a/packages/workflow/src/storage/source-bundle.ts b/packages/workflow/src/storage/source-bundle.ts new file mode 100644 index 000000000..a8daabf13 --- /dev/null +++ b/packages/workflow/src/storage/source-bundle.ts @@ -0,0 +1,670 @@ +/** + * A workflow definition that is the source itself. + * + * The Git descriptor beside this one names a document that lives somewhere + * else: an object id, a path inside it, and a host able to read both. A source + * bundle instead identifies the exact bytes, so a run of a file outside a + * repository, of an untracked file, or of a file edited since its last commit + * is one immutable definition the moment it is retained. + * + * Identity is content addressing over logical paths. A logical path is a + * portable name inside the bundle, never a host filesystem path — the + * containing directory, the absolute path, the invocation working directory + * and the platform separator are all retrieval facts, so two hosts holding the + * same bytes under the same logical entrypoint hold the same definition. + * + * The target is deliberately outside the bundle hash: choosing a section does + * not change the bytes in the bundle. It stays part of the complete definition + * identity, which is what keeps a run of one section distinct from a run of the + * whole document. + * + * Hashing is domain separated and length framed, through the platform's own + * `crypto`. This module names no host: it is the same identity whichever + * provider retains it. + */ + +import { Err, Ok, type Operation, type Result, until } from "effection"; +import { isCanonicalDocumentTarget, isComponentName } from "@executablemd/core"; +import type { Json } from "@executablemd/durable-streams"; +import { WorkflowDefinitionError, WorkflowRequestError } from "./errors.ts"; +import { + describe, + type Members, + parseMembers, + parseStringMember, + requireMemberNames, +} from "./members.ts"; + +/** + * The exact bytes a workflow runs, named by logical path. + * + * `sources` is the complete closure the run executes, in canonical UTF-8 byte + * order of `path`, with `entrypoint` naming exactly one of them. `components` + * maps authored component names onto members of that closure, so adding, + * removing or changing a retained dependency changes `bundleHash`. + */ +export interface SourceBundleWorkflowDefinitionV2 { + readonly version: 2; + readonly kind: "source-bundle"; + readonly hashAlgorithm: "sha256"; + readonly bundleHash: string; + readonly entrypoint: string; + readonly sources: readonly SourceBundleEntryV2[]; + /** One exact canonical document target, without a leading `#`. */ + readonly targetPath?: string; + readonly components?: readonly SourceBundleComponentV2[]; +} + +/** One retained source: its logical path, and what its bytes hash and weigh. */ +export interface SourceBundleEntryV2 { + readonly path: string; + readonly sourceHash: string; + readonly byteLength: number; +} + +/** One authored component name, resolved onto a retained source. */ +export interface SourceBundleComponentV2 { + readonly name: string; + readonly path: string; +} + +/** The exact bytes offered for one logical path when a run is created. */ +export interface SourceBundleSnapshotEntryV2 { + readonly path: string; + readonly bytes: Uint8Array; +} + +/** What a bundle hash is computed over: the closure, and nothing else. */ +export interface SourceBundleIdentityV2 { + readonly entrypoint: string; + readonly sources: readonly SourceBundleEntryV2[]; + readonly components?: readonly SourceBundleComponentV2[]; +} + +const SOURCE_DOMAIN = "executablemd.workflow.source.v2"; +const BUNDLE_DOMAIN = "executablemd.workflow.bundle.v2"; + +/** Hexadecimal digits in a SHA-256 hash, and the bytes they decode to. */ +const HASH_DIGITS = 64; +const HASH_BYTES = 32; + +const MEMBER_NAMES = [ + "version", + "kind", + "hashAlgorithm", + "bundleHash", + "entrypoint", + "sources", + "targetPath", + "components", +]; + +const SOURCE_MEMBER_NAMES = ["path", "sourceHash", "byteLength"]; +const COMPONENT_MEMBER_NAMES = ["name", "path"]; +const SNAPSHOT_MEMBER_NAMES = ["path", "bytes"]; + +const encoder = new TextEncoder(); + +/** + * One decoder, refusing rather than replacing. + * + * `ignoreBOM` keeps a leading U+FEFF as a character instead of consuming it: + * the bytes are the identity, and a decoder that silently dropped three of them + * would hand the Markdown parser something the bundle hash does not describe. + */ +const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + +function fail(reason: string, path: string): Error { + return new WorkflowDefinitionError(reason, path); +} + +/** + * The source-bundle definition a value describes. + * + * Parsed rather than asserted, and parsed closed: a descriptor reaches storage + * from a host and comes back out of a database column, and neither is trusted + * to hold only the members this shape declares. Arrays must arrive canonical — + * a parser that sorted them would turn two spellings of one malformed value + * into one accepted identity. + */ +export function parseSourceBundleDefinition( + value: unknown, +): Result { + try { + return Ok(parseDefinition(value)); + } catch (error) { + if (error instanceof WorkflowDefinitionError) { + return Err(error); + } + throw error; + } +} + +function parseDefinition(value: unknown): SourceBundleWorkflowDefinitionV2 { + const members = parseMembers(value, "$", fail); + requireMemberNames(members, MEMBER_NAMES, "$", fail); + + if (members.get("version") !== 2) { + throw fail("expected version 2", "$.version"); + } + if (parseStringMember(members, "kind", "$", fail) !== "source-bundle") { + throw fail('expected the kind "source-bundle"', "$.kind"); + } + if (parseStringMember(members, "hashAlgorithm", "$", fail) !== "sha256") { + throw fail('expected the hash algorithm "sha256"', "$.hashAlgorithm"); + } + + const bundleHash = parseHash(parseStringMember(members, "bundleHash", "$", fail), "$.bundleHash"); + const sources = parseSources(members); + const entrypoint = parseEntrypoint(parseStringMember(members, "entrypoint", "$", fail), sources); + const targetPath = parseTargetPath(members); + const components = parseComponents(members, sources); + + return { + version: 2, + kind: "source-bundle", + hashAlgorithm: "sha256", + bundleHash, + entrypoint, + sources, + ...(targetPath === undefined ? {} : { targetPath }), + ...(components === undefined ? {} : { components }), + }; +} + +/** + * The complete closure, canonical. + * + * Exactly one entry per logical path, in the UTF-8 byte order of those paths. + * The bundle hash commits to this array, so a descriptor that listed one path + * twice over — or listed them in some other order — would be a second identity + * for bytes that already have one. + */ +function parseSources(members: Members): readonly SourceBundleEntryV2[] { + const path = "$.sources"; + const value = members.get("sources"); + if (!Array.isArray(value)) { + throw fail(`expected an array, found ${describe(value)}`, path); + } + if (value.length === 0) { + throw fail("expected at least one source", path); + } + + const sources: SourceBundleEntryV2[] = []; + let previous: Uint8Array | undefined; + for (let index = 0; index < value.length; index++) { + const entry = parseSourceEntry(value[index], `${path}[${index}]`); + const bytes = encoder.encode(entry.path); + if (previous !== undefined) { + const order = compareBytes(previous, bytes); + if (order === 0) { + throw fail("expected each source path once", path); + } + if (order > 0) { + throw fail("expected sources sorted by the UTF-8 bytes of their paths", path); + } + } + previous = bytes; + sources.push(entry); + } + return Object.freeze(sources); +} + +function parseSourceEntry(value: unknown, path: string): SourceBundleEntryV2 { + const members = parseMembers(value, path, fail); + requireMemberNames(members, SOURCE_MEMBER_NAMES, path, fail); + return { + path: parseLogicalPath(parseStringMember(members, "path", path, fail), `${path}.path`), + sourceHash: parseHash( + parseStringMember(members, "sourceHash", path, fail), + `${path}.sourceHash`, + ), + byteLength: parseByteLength(members.get("byteLength"), `${path}.byteLength`), + }; +} + +/** + * The bundle member the root resolves its component names through. + * + * Absence identifies a definition that declares no workflow components; an + * empty array is not a second spelling of it and is refused. Every path names a + * retained source, so the mapping is closed over the same bytes the run + * executes rather than over something a later resolution would have to find. + */ +function parseComponents( + members: Members, + sources: readonly SourceBundleEntryV2[], +): readonly SourceBundleComponentV2[] | undefined { + if (!members.has("components")) { + return undefined; + } + const path = "$.components"; + const value = members.get("components"); + if (!Array.isArray(value)) { + throw fail(`expected an array, found ${describe(value)}`, path); + } + if (value.length === 0) { + throw fail("expected at least one component", path); + } + + const paths = new Set(sources.map((source) => source.path)); + const components: SourceBundleComponentV2[] = []; + let previous: Uint8Array | undefined; + for (let index = 0; index < value.length; index++) { + const entry = parseComponent(value[index], `${path}[${index}]`, paths); + const bytes = encoder.encode(entry.name); + if (previous !== undefined) { + const order = compareBytes(previous, bytes); + if (order === 0) { + throw fail("expected each component name once", path); + } + if (order > 0) { + throw fail("expected components sorted by the UTF-8 bytes of their names", path); + } + } + previous = bytes; + components.push(entry); + } + return Object.freeze(components); +} + +function parseComponent( + value: unknown, + path: string, + sourcePaths: ReadonlySet, +): SourceBundleComponentV2 { + const members = parseMembers(value, path, fail); + requireMemberNames(members, COMPONENT_MEMBER_NAMES, path, fail); + const name = parseStringMember(members, "name", path, fail); + // Deliberately says nothing about the name it read. A declaration key is + // authored text, and one that fails the grammar has not earned being printed. + if (!isComponentName(name)) { + throw fail("expected a component name", `${path}.name`); + } + const mapped = parseLogicalPath(parseStringMember(members, "path", path, fail), `${path}.path`); + if (!sourcePaths.has(mapped)) { + throw fail("expected a path this definition retains as a source", `${path}.path`); + } + return { name, path: mapped }; +} + +/** The one source the run begins at, which is Markdown by the name it has. */ +function parseEntrypoint(value: string, sources: readonly SourceBundleEntryV2[]): string { + const entrypoint = parseLogicalPath(value, "$.entrypoint"); + if (!entrypoint.endsWith(".md")) { + throw fail('expected a ".md" path', "$.entrypoint"); + } + if (!sources.some((source) => source.path === entrypoint)) { + throw fail("expected a path this definition retains as a source", "$.entrypoint"); + } + return entrypoint; +} + +/** + * The exact target this descriptor names, if it names one. + * + * Presence is the member being written at all, not its value: a descriptor that + * wrote `targetPath` and gave it `undefined` or `null` asked for a target and + * failed to say which, which is not the same as asking for the whole document. + */ +function parseTargetPath(members: Members): string | undefined { + if (!members.has("targetPath")) { + return undefined; + } + const path = "$.targetPath"; + const value = members.get("targetPath"); + if (typeof value !== "string") { + throw fail(`expected a string, found ${describe(value)}`, path); + } + // Deliberately says nothing about the target it read: a canonical target + // encodes heading text, and heading text is document content. + if (!isCanonicalDocumentTarget(value)) { + throw fail("expected one exact canonical document target", path); + } + return value; +} + +/** + * A portable name inside the bundle. + * + * Normalized rather than merely normalizable: two spellings of one path would + * be two identities for one source, and NFC is the one form the descriptor + * admits. The excluded characters are the ones that stop a logical path being + * read back as itself — a separator the host would reinterpret, the fragment + * delimiter a target uses, and the control characters a terminal acts on. + */ +function parseLogicalPath(value: string, path: string): string { + if (value === "") { + throw fail("expected a path", path); + } + // Asked before normalization: an unpaired surrogate is not a scalar value, + // and encoding one substitutes U+FFFD, which would hash bytes nobody + // supplied under a path nobody wrote. + if (/\p{Surrogate}/u.test(value)) { + throw fail("expected Unicode scalar values, found an unpaired surrogate", path); + } + if (value.normalize("NFC") !== value) { + throw fail("expected an NFC-normalized path", path); + } + if (value.includes("\u0000")) { + throw fail("expected a path without a NUL", path); + } + for (const character of value) { + const code = character.codePointAt(0) ?? 0; + if (code < 0x20 || code === 0x7f) { + throw fail("expected a path without control characters", path); + } + } + if (value.includes("\\")) { + throw fail("expected POSIX separators, found a backslash", path); + } + if (value.includes("#")) { + throw fail('expected a path without a "#"', path); + } + if (value.startsWith("/")) { + throw fail("expected a bundle-relative path, found an absolute one", path); + } + if (value.endsWith("/")) { + throw fail("expected a path, found a trailing separator", path); + } + for (const segment of value.split("/")) { + if (segment === "") { + throw fail("expected a normalized path, found an empty segment", path); + } + if (segment === "." || segment === "..") { + throw fail(`expected a normalized path, found a ${JSON.stringify(segment)} segment`, path); + } + } + return value; +} + +/** + * A hash is compared, never re-derived from its spelling, so the spelling is + * the identity. One case is admitted so two hosts that agree about the bytes + * also agree about the run. + */ +function parseHash(value: string, path: string): string { + if (value.length !== HASH_DIGITS) { + throw fail(`expected ${HASH_DIGITS} hexadecimal digits`, path); + } + if (!/^[0-9a-f]+$/.test(value)) { + throw fail("expected lowercase hexadecimal digits", path); + } + return value; +} + +/** How many bytes a source weighs: countable, and countable by this runtime. */ +function parseByteLength(value: unknown, path: string): number { + if (typeof value !== "number") { + throw fail(`expected a number, found ${describe(value)}`, path); + } + if (!Number.isSafeInteger(value) || value < 0) { + throw fail("expected a non-negative safe integer", path); + } + return value; +} + +/** + * The descriptor as a plain JSON value. + * + * An interface has no index signature, so a descriptor is not a `Json` until it + * is written out member by member. Doing that here is also what keeps the + * stored shape and the parsed shape one decision. + */ +export function sourceBundleDefinitionToJson(definition: SourceBundleWorkflowDefinitionV2): Json { + return { + version: definition.version, + kind: definition.kind, + hashAlgorithm: definition.hashAlgorithm, + bundleHash: definition.bundleHash, + entrypoint: definition.entrypoint, + sources: definition.sources.map((source) => ({ + path: source.path, + sourceHash: source.sourceHash, + byteLength: source.byteLength, + })), + // Written only when there is one. A descriptor that stored an explicit + // absence would parse back as one that asked for a target, or for a bundle, + // and failed to name it. + ...(definition.targetPath === undefined ? {} : { targetPath: definition.targetPath }), + ...(definition.components === undefined + ? {} + : { + components: definition.components.map((component) => ({ + name: component.name, + path: component.path, + })), + }), + }; +} + +/** The component mapping this definition declares, empty when it declares none. */ +export function sourceBundleComponents( + definition: SourceBundleWorkflowDefinitionV2, +): readonly SourceBundleComponentV2[] { + return definition.components ?? []; +} + +/** + * The hash of one source's exact bytes. + * + * Domain separated and length framed, so bytes that hash as a source cannot be + * presented as any other structure this repository hashes, and so a source's + * own length is committed to rather than inferred from where it ended. + */ +export function* sourceContentHash(bytes: Uint8Array): Operation { + return yield* digest([field(SOURCE_DOMAIN), u64(bytes.byteLength), bytes]); +} + +/** + * The hash of the closure a definition executes. + * + * Each source contributes its path, its decoded 32-byte hash and its length — + * the decoded bytes rather than their hexadecimal text, so the hash commits to + * the value and not to one spelling of it. The target is absent on purpose: + * selecting a section does not change the bytes in the bundle. + */ +export function* sourceBundleHash(identity: SourceBundleIdentityV2): Operation { + const components = identity.components ?? []; + const chunks: Uint8Array[] = [ + field(BUNDLE_DOMAIN), + field(identity.entrypoint), + u32(identity.sources.length), + ]; + for (const source of identity.sources) { + chunks.push(field(source.path), decodeHash(source.sourceHash), u64(source.byteLength)); + } + chunks.push(u32(components.length)); + for (const component of components) { + chunks.push(field(component.name), field(component.path)); + } + return yield* digest(chunks); +} + +/** + * The definition, once its own hash is recomputed from what it retains. + * + * A descriptor that parses is well formed; this is the separate question of + * whether it describes itself. Storage asks it before exposing a record, + * because a bundle hash its own manifest does not produce is a retained + * identity nobody can reproduce. + */ +export function* verifySourceBundleDefinition( + definition: SourceBundleWorkflowDefinitionV2, +): Operation> { + const recomputed = yield* sourceBundleHash(definition); + if (recomputed !== definition.bundleHash) { + return Err(fail("expected the bundle hash this definition's sources produce", "$.bundleHash")); + } + return Ok(definition); +} + +/** + * The exact bytes offered for a definition, owned and checked against it. + * + * Copied before anything is checked, so what is verified is what is retained: a + * caller holding the original `Uint8Array` can mutate it afterwards without + * changing the run. The snapshot must name exactly the descriptor's paths in + * exactly its order — a missing, extra or reordered entry describes a different + * closure, not one to be repaired by sorting. + */ +export function* verifySourceBundleSnapshot( + definition: SourceBundleWorkflowDefinitionV2, + snapshot: unknown, +): Operation> { + const owned = copySnapshot(snapshot); + if (!owned.ok) { + return owned; + } + + const entries = owned.value; + if (entries.length !== definition.sources.length) { + return Err(new WorkflowRequestError(refusal("one entry per retained source"))); + } + for (let index = 0; index < entries.length; index++) { + const source = definition.sources[index]; + const entry = entries[index]; + if (entry.path !== source.path) { + return Err( + new WorkflowRequestError(refusal(`the retained source path at position ${index}`)), + ); + } + if (entry.bytes.byteLength !== source.byteLength) { + return Err( + new WorkflowRequestError(refusal(`the declared byte length at position ${index}`)), + ); + } + const hash = yield* sourceContentHash(entry.bytes); + if (hash !== source.sourceHash) { + return Err( + new WorkflowRequestError(refusal(`the declared source hash at position ${index}`)), + ); + } + } + + // The bundle hash commits to the manifest the entries were just checked + // against, and is recomputed anyway: a hash is not a reason to retain a + // descriptor whose own structure disagrees with itself. + const verified = yield* verifySourceBundleDefinition(definition); + if (!verified.ok) { + return verified; + } + return Ok(entries); +} + +/** + * The Markdown a retained source holds. + * + * Strict: a byte sequence that is not well-formed UTF-8 is refused rather than + * decoded to replacement characters, because the replacement would parse as a + * document the bundle hash does not describe. No normalization of any kind + * happens here — the bytes stay authoritative, and this is only their reading. + */ +export function decodeSourceText(bytes: Uint8Array): Result { + try { + return Ok(decoder.decode(bytes)); + } catch { + return Err(new WorkflowRequestError("A workflow source must be well-formed UTF-8.")); + } +} + +/** Names what disagreed, never the bytes, the path or the props involved. */ +function refusal(subject: string): string { + return ( + `The source snapshot offered for this workflow run does not match ${subject} its ` + + "definition declares. Nothing was retained." + ); +} + +function copySnapshot(value: unknown): Result { + if (!Array.isArray(value)) { + return Err(new WorkflowRequestError(refusal("the array of sources"))); + } + if (value.length === 0) { + return Err(new WorkflowRequestError(refusal("the non-empty set of sources"))); + } + + const entries: SourceBundleSnapshotEntryV2[] = []; + for (const candidate of value) { + if (candidate === null || typeof candidate !== "object" || Array.isArray(candidate)) { + return Err(new WorkflowRequestError(refusal("the shape of the entries"))); + } + const members = new Map(Object.entries(candidate)); + for (const key of members.keys()) { + if (!SNAPSHOT_MEMBER_NAMES.includes(key)) { + return Err(new WorkflowRequestError(refusal("the members of the entries"))); + } + } + const path = members.get("path"); + const bytes = members.get("bytes"); + if (typeof path !== "string") { + return Err(new WorkflowRequestError(refusal("the shape of the entries"))); + } + if (!(bytes instanceof Uint8Array)) { + return Err(new WorkflowRequestError(refusal("the shape of the entries"))); + } + entries.push(Object.freeze({ path, bytes: Uint8Array.from(bytes) })); + } + return Ok(Object.freeze(entries)); +} + +function* digest(chunks: readonly Uint8Array[]): Operation { + const computed = yield* until(crypto.subtle.digest("SHA-256", concat(chunks))); + return Array.from(new Uint8Array(computed), (byte) => byte.toString(16).padStart(2, "0")).join( + "", + ); +} + +function concat(chunks: readonly Uint8Array[]): Uint8Array { + let length = 0; + for (const chunk of chunks) { + length += chunk.byteLength; + } + const joined = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + joined.set(chunk, offset); + offset += chunk.byteLength; + } + return joined; +} + +/** `u32(utf8(value).length)` followed by those bytes. */ +function field(value: string): Uint8Array { + const bytes = encoder.encode(value); + return concat([u32(bytes.byteLength), bytes]); +} + +function u32(value: number): Uint8Array { + const bytes = new Uint8Array(4); + new DataView(bytes.buffer).setUint32(0, value); + return bytes; +} + +function u64(value: number): Uint8Array { + const bytes = new Uint8Array(8); + new DataView(bytes.buffer).setBigUint64(0, BigInt(value)); + return bytes; +} + +function decodeHash(value: string): Uint8Array { + const bytes = new Uint8Array(HASH_BYTES); + for (let index = 0; index < HASH_BYTES; index++) { + bytes[index] = Number.parseInt(value.slice(index * 2, index * 2 + 2), 16); + } + return bytes; +} + +/** Two byte strings in UTF-8 order, which `<` on their strings is not. */ +function compareBytes(left: Uint8Array, right: Uint8Array): number { + const shared = Math.min(left.byteLength, right.byteLength); + for (let index = 0; index < shared; index++) { + if (left[index] !== right[index]) { + return left[index] < right[index] ? -1 : 1; + } + } + if (left.byteLength === right.byteLength) { + return 0; + } + return left.byteLength < right.byteLength ? -1 : 1; +} diff --git a/packages/workflow/tests/workflow-definition.test.ts b/packages/workflow/tests/workflow-definition.test.ts index bc2d56a0a..a8322a391 100644 --- a/packages/workflow/tests/workflow-definition.test.ts +++ b/packages/workflow/tests/workflow-definition.test.ts @@ -14,14 +14,24 @@ import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { isCanonicalDocumentTarget } from "@executablemd/core"; +import type { Json } from "@executablemd/durable-streams"; import { canonicalJson, conflictingFields, + decodeSourceText, definitionComponents, definitionToJson, type GitWorkflowDefinitionV1, + parseSourceBundleDefinition, parseStopReasonInput, parseWorkflowDefinition, + sourceBundleComponents, + sourceBundleDefinitionToJson, + sourceBundleHash, + type SourceBundleWorkflowDefinitionV2, + sourceContentHash, + verifySourceBundleDefinition, + verifySourceBundleSnapshot, WORKFLOW_RUN_STATUSES, WorkflowDefinitionError, WorkflowRequestError, @@ -679,3 +689,489 @@ describe("Tier WD — a bundle decides compatible reuse", () => { } }); }); + +/** + * Tier WD — the source bundle a definition retains. + * + * Version 2 is not a looser version 1. Its identity is the bytes themselves, + * addressed by logical paths that are portable names inside the bundle rather + * than anything a filesystem hands out — so the questions here are what a path + * may be, what order a manifest may arrive in, and what exactly the two + * domain-separated hashes are computed over. + * + * The hash vectors below were produced by a separate implementation of the + * specified framing rather than by the code under test. A vector derived from + * the implementation would agree with whatever framing it happened to have. + */ + +const encoder = new TextEncoder(); + +/** Three logical paths, in the canonical UTF-8 byte order of the manifest. */ +const ROOT_PATH = "release.md"; +/** U+FB01, whose UTF-8 sorts before the emoji and whose UTF-16 sorts after it. */ +const LIGATURE_PATH = "\uFB01.md"; +const EMOJI_PATH = "\u{1F600}.md"; + +const ROOT_TEXT = "# Release\n"; +const EMOJI_TEXT = "# \u00DCn\u00EFc\u00F8d\u00E9\n"; + +const ROOT_BYTES = encoder.encode(ROOT_TEXT); +const LIGATURE_BYTES = new Uint8Array(0); +const EMOJI_BYTES = encoder.encode(EMOJI_TEXT); + +const ROOT_HASH = "b78cd463c5885c1b595de07f665ce82b61df6636eb8c5f00cf11985cbfeb986d"; +const LIGATURE_HASH = "32b9c0cb4d326ff21913998350eccb9cd8c437576eafcbe0bc79833e90a7cd3c"; +const EMOJI_HASH = "3c81db896c70d1bdd74b0318f509d475248ae03b111e7f7c0b96fc10b26b6fbf"; + +/** The whole three-source bundle, with its component mapping. */ +const BUNDLE_HASH = "078f7cf1b61cad754ad9d03333df0027a477d5a2cee967903c54b147c29c37e3"; +/** The same three sources, declaring no components. */ +const UNMAPPED_HASH = "4ac6d339215f887347b0226f388b86e010e8d60e41c4eac7eed5b61f791f5e29"; +/** The entrypoint alone, which is what a root declaring no components retains. */ +const SOLO_HASH = "adc2c8e139c6956377090d3f2b26471f461164020c0e28812cda91dcdbceb419"; + +const SOURCES = [ + { path: ROOT_PATH, sourceHash: ROOT_HASH, byteLength: 10 }, + { path: LIGATURE_PATH, sourceHash: LIGATURE_HASH, byteLength: 0 }, + { path: EMOJI_PATH, sourceHash: EMOJI_HASH, byteLength: 14 }, +]; + +const COMPONENTS = [ + { name: "Discovery", path: LIGATURE_PATH }, + { name: "Planning", path: EMOJI_PATH }, +]; + +const SNAPSHOT = [ + { path: ROOT_PATH, bytes: ROOT_BYTES }, + { path: LIGATURE_PATH, bytes: LIGATURE_BYTES }, + { path: EMOJI_PATH, bytes: EMOJI_BYTES }, +]; + +/** A v2 descriptor, loosely typed: half of these tests build ones that are wrong. */ +function bundleV2(overrides: Record = {}): Record { + return { + version: 2, + kind: "source-bundle", + hashAlgorithm: "sha256", + bundleHash: BUNDLE_HASH, + entrypoint: ROOT_PATH, + sources: SOURCES, + components: COMPONENTS, + ...overrides, + }; +} + +/** The same three sources with no `components` member written at all. */ +function unmappedV2(): Record { + return { + version: 2, + kind: "source-bundle", + hashAlgorithm: "sha256", + bundleHash: UNMAPPED_HASH, + entrypoint: ROOT_PATH, + sources: SOURCES, + }; +} + +/** The one-source descriptor a root declaring no components produces. */ +function soloV2(overrides: Record = {}): Record { + return { + version: 2, + kind: "source-bundle", + hashAlgorithm: "sha256", + bundleHash: SOLO_HASH, + entrypoint: ROOT_PATH, + sources: [SOURCES[0]], + ...overrides, + }; +} + +function parsedV2(value: Record): SourceBundleWorkflowDefinitionV2 { + const result = parseSourceBundleDefinition(value); + if (!result.ok) { + throw result.error; + } + return result.value; +} + +function v2Refusal(value: unknown): WorkflowDefinitionError { + const result = parseSourceBundleDefinition(value); + if (result.ok) { + throw new Error("expected the descriptor to be refused"); + } + if (!(result.error instanceof WorkflowDefinitionError)) { + throw result.error; + } + return result.error; +} + +/** A one-entry manifest carrying one deliberately wrong path. */ +function sourceAt(path: unknown): Record[] { + return [{ path, sourceHash: ROOT_HASH, byteLength: 10 }]; +} + +/** + * The members one serialized descriptor wrote, in the order it wrote them. + * + * Narrowed rather than asserted: presentation order is what these cases are + * about, and a cast would make the claim hold for a serializer that answered + * with an array or a scalar. + */ +function jsonMembers(value: Json): string[] { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new Error("expected the serialized descriptor to be a JSON object"); + } + return Object.keys(value); +} + +/** Every logical path this grammar refuses, whichever member carries it. */ +const REFUSED_PATHS = [ + "", + "/release.md", + "release.md/", + "workflows//release.md", + "./release.md", + "../release.md", + "workflows/../release.md", + "workflows\\release.md", + "rele#ase.md", + "rele\u0000ase.md", + "rele\u0001ase.md", + "rele\u001Fase.md", + "rele\u007Fase.md", + // Decomposed: `e` followed by a combining acute is a second spelling of a + // path that already has one, and two spellings would be two identities. + "cafe\u0301.md", + // Not a Unicode scalar value: encoding it would substitute U+FFFD and hash + // bytes nobody supplied. + "\uD800.md", +]; + +describe("Tier WD — a source-bundle descriptor", () => { + it("WD36: reads a complete descriptor and round-trips it in presentation order", function* () { + const first = parsedV2(bundleV2({ targetPath: "Release/Publish" })); + + expect(first).toEqual({ + version: 2, + kind: "source-bundle", + hashAlgorithm: "sha256", + bundleHash: BUNDLE_HASH, + entrypoint: ROOT_PATH, + sources: SOURCES, + targetPath: "Release/Publish", + components: COMPONENTS, + }); + + const json = sourceBundleDefinitionToJson(first); + expect(jsonMembers(json)).toEqual([ + "version", + "kind", + "hashAlgorithm", + "bundleHash", + "entrypoint", + "sources", + "targetPath", + "components", + ]); + + const again = parseSourceBundleDefinition(json); + expect(again.ok).toBe(true); + expect(again.ok && again.value).toEqual(first); + }); + + it("WD37: admits exactly its own members, and only version 2 source bundles", function* () { + // Neither the host path the bytes were read from nor the props a run was + // started with is a member of this shape, so neither reaches the identity. + expect(v2Refusal({ ...bundleV2(), sourcePath: "/home/ada/release.md" }).path).toBe("$"); + expect(v2Refusal({ ...bundleV2(), props: { channel: "stable" } }).path).toBe("$"); + expect(v2Refusal({ ...bundleV2(), base: "main" }).path).toBe("$"); + + expect(v2Refusal(null).message).toContain("found null"); + expect(v2Refusal([]).message).toContain("found an array"); + expect(v2Refusal(bundleV2({ version: 1 })).path).toBe("$.version"); + expect(v2Refusal(bundleV2({ kind: "git" })).path).toBe("$.kind"); + expect(v2Refusal(bundleV2({ hashAlgorithm: "sha1" })).path).toBe("$.hashAlgorithm"); + }); + + it("WD38: a logical path is a portable name, not something a host handed out", function* () { + for (const path of REFUSED_PATHS) { + expect({ path, at: v2Refusal(soloV2({ sources: sourceAt(path) })).path }).toEqual({ + path, + at: "$.sources[0].path", + }); + } + + expect(v2Refusal(soloV2({ sources: sourceAt(42) })).message).toContain("expected a string"); + // The two non-ASCII paths are ordinary logical paths and survive exactly. + expect(parsedV2(bundleV2()).sources.map((source) => source.path)).toEqual([ + ROOT_PATH, + LIGATURE_PATH, + EMOJI_PATH, + ]); + }); + + it("WD39: the entrypoint is Markdown the bundle actually retains", function* () { + expect(v2Refusal(soloV2({ entrypoint: "release.txt" })).message).toContain('expected a ".md"'); + expect(v2Refusal(soloV2({ entrypoint: "other.md" })).message).toContain( + "expected a path this definition retains as a source", + ); + expect(v2Refusal(soloV2({ entrypoint: "/release.md" })).path).toBe("$.entrypoint"); + expect(v2Refusal(soloV2({ entrypoint: 7 })).path).toBe("$.entrypoint"); + }); + + it("WD40: the manifest is ordered by UTF-8 bytes, which is not string order", function* () { + // The discriminating pair. `<` on strings compares UTF-16 code units, so it + // sorts the supplementary character before the ligature while UTF-8 sorts + // the ligature first: a parser that used `<` would admit the wrong array. + const byCodeUnit = [...SOURCES].sort((left, right) => (left.path < right.path ? -1 : 1)); + expect(byCodeUnit.map((source) => source.path)).toEqual([ROOT_PATH, EMOJI_PATH, LIGATURE_PATH]); + expect(v2Refusal(bundleV2({ sources: byCodeUnit })).message).toContain("UTF-8 bytes"); + + expect(v2Refusal(bundleV2({ sources: [...SOURCES].reverse() })).message).toContain( + "UTF-8 bytes", + ); + expect(v2Refusal(bundleV2({ sources: [SOURCES[0], SOURCES[0]] })).message).toContain( + "each source path once", + ); + expect(v2Refusal(bundleV2({ sources: [] })).message).toContain("at least one source"); + expect(v2Refusal(bundleV2({ sources: {} })).path).toBe("$.sources"); + }); + + it("WD41: a component mapping is closed over the sources declared beside it", function* () { + const solo = parsedV2(soloV2()); + expect("components" in solo).toBe(false); + expect(sourceBundleComponents(solo)).toEqual([]); + expect(jsonMembers(sourceBundleDefinitionToJson(solo))).toEqual([ + "version", + "kind", + "hashAlgorithm", + "bundleHash", + "entrypoint", + "sources", + ]); + + // Declaring none and declaring an empty set are not two spellings of one + // thing: the second asked for a bundle and named nothing. + expect(v2Refusal(bundleV2({ components: [] })).message).toContain("at least one component"); + expect(v2Refusal(bundleV2({ components: undefined })).path).toBe("$.components"); + expect(v2Refusal(bundleV2({ components: null })).path).toBe("$.components"); + + expect( + v2Refusal(bundleV2({ components: [{ name: "Discovery", path: "absent.md" }] })).message, + ).toContain("expected a path this definition retains as a source"); + expect( + v2Refusal(bundleV2({ components: [{ ...COMPONENTS[0], sourceHash: ROOT_HASH }] })).path, + ).toBe("$.components[0]"); + expect( + v2Refusal(bundleV2({ components: [{ ...COMPONENTS[0], name: "discovery" }] })).path, + ).toBe("$.components[0].name"); + expect(v2Refusal(bundleV2({ components: [...COMPONENTS].reverse() })).message).toContain( + "UTF-8 bytes", + ); + expect(v2Refusal(bundleV2({ components: [COMPONENTS[0], COMPONENTS[0]] })).message).toContain( + "each component name once", + ); + }); + + it("WD42: a hash has one spelling, and a length is a count of bytes", function* () { + expect(v2Refusal(bundleV2({ bundleHash: BUNDLE_HASH.toUpperCase() })).message).toContain( + "lowercase", + ); + expect(v2Refusal(bundleV2({ bundleHash: BUNDLE_HASH.slice(1) })).message).toContain( + "64 hexadecimal digits", + ); + expect(v2Refusal(bundleV2({ bundleHash: `${BUNDLE_HASH}0` })).path).toBe("$.bundleHash"); + expect(v2Refusal(bundleV2({ bundleHash: `z${BUNDLE_HASH.slice(1)}` })).path).toBe( + "$.bundleHash", + ); + expect(v2Refusal(soloV2({ sources: [{ ...SOURCES[0], sourceHash: "abc" }] })).path).toBe( + "$.sources[0].sourceHash", + ); + + for (const byteLength of [-1, 1.5, Number.NaN, Number.MAX_SAFE_INTEGER + 1, "10", null]) { + expect({ + byteLength, + at: v2Refusal(soloV2({ sources: [{ ...SOURCES[0], byteLength }] })).path, + }).toEqual({ byteLength, at: "$.sources[0].byteLength" }); + } + // Zero is a length a source may have: an empty file is exact bytes too. + expect(parsedV2(bundleV2()).sources[1].byteLength).toBe(0); + }); + + it("WD43: the exact target is present or absent, never synthesized", function* () { + expect("targetPath" in parsedV2(bundleV2())).toBe(false); + expect(parsedV2(bundleV2({ targetPath: "Release/Publish" })).targetPath).toBe( + "Release/Publish", + ); + + for (const targetPath of ["#Release", "Release/*", "Release/", undefined, null, 1]) { + expect({ targetPath, at: v2Refusal(bundleV2({ targetPath })).path }).toEqual({ + targetPath, + at: "$.targetPath", + }); + } + }); + + it("WD44: a refusal never repeats the bundle it refused", function* () { + const canary = "never-printed-canary-4c1f8a"; + + for (const error of [ + v2Refusal(bundleV2({ entrypoint: `/${canary}.md` })), + v2Refusal(soloV2({ sources: sourceAt(`/${canary}.md`) })), + v2Refusal(soloV2({ sources: [{ ...SOURCES[0], sourceHash: canary }] })), + v2Refusal(bundleV2({ components: [{ ...COMPONENTS[0], name: canary }] })), + v2Refusal({ ...bundleV2(), [canary]: 1 }), + ]) { + expect(error.message).not.toContain(canary); + expect(error.path).not.toContain(canary); + } + }); +}); + +describe("Tier WD — what a source bundle hashes", () => { + it("WD45: a source hash is its domain, its length and its exact bytes", function* () { + expect(yield* sourceContentHash(ROOT_BYTES)).toBe(ROOT_HASH); + // Zero-length content still hashes its domain and its declared length, so + // an empty source is a source rather than an absent one. + expect(yield* sourceContentHash(LIGATURE_BYTES)).toBe(LIGATURE_HASH); + // Fourteen bytes behind ten characters: the framing commits to the bytes. + expect(EMOJI_BYTES.byteLength).toBe(14); + expect(EMOJI_TEXT.length).toBe(10); + expect(yield* sourceContentHash(EMOJI_BYTES)).toBe(EMOJI_HASH); + + // One byte more is a different source. + expect(yield* sourceContentHash(encoder.encode(`${ROOT_TEXT}\n`))).not.toBe(ROOT_HASH); + }); + + it("WD46: a bundle hash is the entrypoint, the manifest and the mapping", function* () { + expect(yield* sourceBundleHash(parsedV2(bundleV2()))).toBe(BUNDLE_HASH); + + // Dropping the mapping is a different bundle over the same three sources. + expect(yield* sourceBundleHash(parsedV2(unmappedV2()))).toBe(UNMAPPED_HASH); + expect(UNMAPPED_HASH).not.toBe(BUNDLE_HASH); + + expect(yield* sourceBundleHash(parsedV2(soloV2()))).toBe(SOLO_HASH); + }); + + it("WD47: the target is outside the bundle hash and inside the descriptor", function* () { + const whole = parsedV2(bundleV2()); + const section = parsedV2(bundleV2({ targetPath: "Release/Publish" })); + const other = parsedV2(bundleV2({ targetPath: "Release/Announce" })); + + // Selecting a section does not change the bytes in the bundle, so all three + // carry the one hash this manifest produces. + for (const descriptor of [whole, section, other]) { + expect(yield* sourceBundleHash(descriptor)).toBe(BUNDLE_HASH); + expect(descriptor.bundleHash).toBe(BUNDLE_HASH); + } + // And the three descriptors remain three identities. + expect(section.targetPath).not.toBe(other.targetPath); + expect("targetPath" in whole).toBe(false); + }); + + it("WD48: a descriptor whose own manifest disagrees with its hash is refused", function* () { + const honest = yield* verifySourceBundleDefinition(parsedV2(bundleV2())); + expect(honest.ok).toBe(true); + + // Another bundle's hash, worn by this one. It parses — the grammar is + // satisfied — and it does not describe itself. + const lying = yield* verifySourceBundleDefinition( + parsedV2(bundleV2({ bundleHash: SOLO_HASH })), + ); + expect(lying.ok).toBe(false); + expect(!lying.ok && lying.error).toBeInstanceOf(WorkflowDefinitionError); + expect(!lying.ok && lying.error.message).toContain("$.bundleHash"); + }); +}); + +describe("Tier WD — the snapshot a source bundle is created from", () => { + it("WD49: the accepted snapshot is a copy, so a later mutation is inert", function* () { + const mine = encoder.encode(ROOT_TEXT); + const offered = [ + { path: ROOT_PATH, bytes: mine }, + { path: LIGATURE_PATH, bytes: LIGATURE_BYTES }, + { path: EMOJI_PATH, bytes: EMOJI_BYTES }, + ]; + + const accepted = yield* verifySourceBundleSnapshot(parsedV2(bundleV2()), offered); + expect(accepted.ok).toBe(true); + if (!accepted.ok) { + throw accepted.error; + } + + mine[0] = 0x21; + expect(Array.from(mine.slice(0, 1))).toEqual([0x21]); + expect(Array.from(accepted.value[0].bytes)).toEqual(Array.from(ROOT_BYTES)); + expect(yield* sourceContentHash(accepted.value[0].bytes)).toBe(ROOT_HASH); + expect(accepted.value.map((entry) => entry.path)).toEqual([ + ROOT_PATH, + LIGATURE_PATH, + EMOJI_PATH, + ]); + }); + + it("WD50: a snapshot that is not exactly the manifest retains nothing", function* () { + const descriptor = parsedV2(bundleV2()); + + for (const snapshot of [ + SNAPSHOT.slice(1), + [...SNAPSHOT, { path: "extra.md", bytes: ROOT_BYTES }], + [SNAPSHOT[0], SNAPSHOT[2], SNAPSHOT[1]], + [{ path: LIGATURE_PATH, bytes: ROOT_BYTES }, ...SNAPSHOT.slice(1)], + [{ path: ROOT_PATH, bytes: encoder.encode("# Release") }, ...SNAPSHOT.slice(1)], + [{ path: ROOT_PATH, bytes: encoder.encode("# release\n") }, ...SNAPSHOT.slice(1)], + [], + {}, + ]) { + const result = yield* verifySourceBundleSnapshot(descriptor, snapshot); + expect(result.ok).toBe(false); + expect(!result.ok && result.error).toBeInstanceOf(WorkflowRequestError); + // The bytes a caller offered are the document, so a refusal names the + // position that disagreed and nothing about what it held. + expect(!result.ok && result.error.message).not.toContain("# Release"); + } + }); + + it("WD51: a snapshot entry admits exactly a path and its bytes", function* () { + const descriptor = parsedV2(soloV2()); + + for (const snapshot of [ + [{ path: ROOT_PATH, bytes: ROOT_BYTES, origin: "/home/ada/release.md" }], + [{ path: ROOT_PATH }], + [{ path: ROOT_PATH, bytes: ROOT_TEXT }], + [{ path: ROOT_PATH, bytes: Array.from(ROOT_BYTES) }], + [{ bytes: ROOT_BYTES }], + [ROOT_PATH], + [null], + ]) { + const result = yield* verifySourceBundleSnapshot(descriptor, snapshot); + expect(result.ok).toBe(false); + expect(!result.ok && result.error).toBeInstanceOf(WorkflowRequestError); + } + + // The one snapshot that does describe this definition is accepted. + const accepted = yield* verifySourceBundleSnapshot(descriptor, [SNAPSHOT[0]]); + expect(accepted.ok).toBe(true); + }); + + it("WD52: a retained source reads as UTF-8 strictly, and without normalization", function* () { + const text = decodeSourceText(EMOJI_BYTES); + expect(text.ok && text.value).toBe(EMOJI_TEXT); + expect(decodeSourceText(LIGATURE_BYTES).ok).toBe(true); + + // A byte-order mark is content, not punctuation to be swallowed: the bytes + // are the identity, and dropping three of them changes what parses. + const marked = decodeSourceText(encoder.encode(`\uFEFF${ROOT_TEXT}`)); + expect(marked.ok && marked.value).toBe(`\uFEFF${ROOT_TEXT}`); + + // Decomposed content stays decomposed. Normalizing it here would hand the + // document parser text the bundle hash does not describe. + const decomposed = decodeSourceText(encoder.encode("cafe\u0301\n")); + expect(decomposed.ok && decomposed.value).toBe("cafe\u0301\n"); + expect(decomposed.ok && decomposed.value).not.toBe("caf\u00E9\n"); + + const invalid = decodeSourceText(new Uint8Array([0x23, 0x20, 0xff, 0xfe])); + expect(invalid.ok).toBe(false); + expect(!invalid.ok && invalid.error).toBeInstanceOf(WorkflowRequestError); + }); +}); From 64639f20e179d3a9f7087fa057e9ed5afb411da1 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Tue, 15 Sep 2026 12:54:39 -0400 Subject: [PATCH 2/8] =?UTF-8?q?=E2=9C=A8=20Retain=20source-bundle=20workfl?= =?UTF-8?q?ow=20runs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A workflow run can now be a run of the bytes it retains. Storage, lifecycle, journals and artifacts each became version-aware by their own identity rather than by widening version 1's, so a Git run keeps exactly the schema, record, descriptor, journal value and artifact bytes it always had. Live storage gains a second immutable inventory. Version 2 keeps every version-1 object byte for byte, replaces `workflow_run` with one that has no base column, and adds the two definition-source tables. Recognition reads the application id, then dispatches on version 1 or 2 and holds the whole inventory to that version's declaration: a hybrid, an extra object or version 0 is the file disagreeing with itself, never a migration candidate. Creating a version-2 run crosses only the trusted lifecycle transition. Public `WorkflowRunStorage.create()` stays version-1 and refuses a source-bundle descriptor, because that request carries a descriptor and no bytes. The transition copies the caller's buffers and holds them to the descriptor before the transaction opens, then writes schema, run, manifest, de-duplicated content, Workspace, lifecycle and the first execution together or not at all. Source is proved before anything moves. A version-2 run's content is re-derived from its own store — every length, every source hash and the bundle hash — and a missing entry, damaged content or a disagreeing hash refuses with the run's lifecycle and journal untouched. A version-1 run's content crosses the host-captured `LegacyWorkflowSourceReader`, which is a direct closure reachable through no Context, Api, component or Plugin; Workflow recomputes the returned root and component blob identities from the bytes themselves before any of it counts as this run's. A creation is held to its own descriptor before persistence, so a host that cannot obtain a Git definition's Markdown never reaches the transaction that would make the run exist. The public and durable `WorkflowRun` is one closed union. Version 1 keeps its exact three members; version 2 records `definitionVersion`, `bundleHash` and its exact target and invents no base or pinned commit. Readers accept either exact member set in any key order and nothing between them. Artifacts keep the physical container at version 1 and dispatch the semantic format from the header. Format 1 keeps its manifest, identity domain and Git closure byte for byte; format 2 has manifest version 2, the `xmd-artifact\0v2\0` domain, its own closed inventory, and one entry/content pair per logical source. The writer selects the format from the run's version and the reader verifies only the format its header declares. Refs #443 --- packages/workflow/deno.ts | 15 +- packages/workflow/mod.ts | 55 +- .../workflow/src/deno/artifact-frontier.ts | 38 +- .../workflow/src/deno/artifact/manifest.ts | 24 +- packages/workflow/src/deno/artifact/read.ts | 117 ++++- .../workflow/src/deno/artifact/records.ts | 277 +++++++++- packages/workflow/src/deno/artifact/schema.ts | 24 +- packages/workflow/src/deno/artifact/types.ts | 120 +++-- packages/workflow/src/deno/artifact/write.ts | 28 +- packages/workflow/src/deno/database.ts | 4 +- .../workflow/src/deno/definition-source.ts | 311 ++++++++++++ packages/workflow/src/deno/lifecycle.ts | 150 ++++-- packages/workflow/src/deno/provider.ts | 21 +- packages/workflow/src/deno/rows.ts | 52 +- packages/workflow/src/deno/run-host.ts | 25 +- packages/workflow/src/deno/schema.ts | 230 ++++++++- packages/workflow/src/deno/transitions.ts | 474 +++++++++++++++--- packages/workflow/src/fork.ts | 14 +- packages/workflow/src/journal.ts | 119 ++++- packages/workflow/src/lifecycle/execution.ts | 60 ++- packages/workflow/src/lifecycle/source.ts | 105 ++++ packages/workflow/src/run.ts | 84 +++- packages/workflow/src/storage/api.ts | 16 +- .../workflow/src/storage/compatibility.ts | 142 +++++- packages/workflow/src/storage/definition.ts | 70 ++- packages/workflow/src/storage/errors.ts | 91 ++++ packages/workflow/src/storage/record.ts | 47 +- .../workflow/tests/public-entrypoint.test.ts | 172 +++++++ .../tests/support/artifact-fixture.ts | 107 +++- .../workflow/tests/support/composition.ts | 31 +- .../workflow/tests/support/executor-holder.ts | 3 +- .../workflow/tests/support/legacy-source.ts | 73 +++ .../workflow/tests/support/restart-child.ts | 3 +- packages/workflow/tests/support/storage.ts | 107 +++- .../tests/workflow-definition.test.ts | 35 +- .../workflow/tests/workflow-export.test.ts | 173 ++++++- packages/workflow/tests/workflow-fork.test.ts | 304 +++++++++++ .../workflow-lifecycle-authority.test.ts | 208 +++++++- .../tests/workflow-lifecycle-control.test.ts | 3 +- .../tests/workflow-run-journal.test.ts | 84 ++++ .../tests/workflow-run-storage.test.ts | 208 +++++++- packages/workflow/tests/workflow-run.test.ts | 33 +- packages/workflow/tests/xmd-artifact.test.ts | 160 +++++- 43 files changed, 3972 insertions(+), 445 deletions(-) create mode 100644 packages/workflow/src/deno/definition-source.ts create mode 100644 packages/workflow/src/lifecycle/source.ts create mode 100644 packages/workflow/tests/support/legacy-source.ts diff --git a/packages/workflow/deno.ts b/packages/workflow/deno.ts index 65b3948f7..a00857af5 100644 --- a/packages/workflow/deno.ts +++ b/packages/workflow/deno.ts @@ -28,7 +28,11 @@ export { useWorkflowRunStorage } from "./src/deno/provider.ts"; export type { WorkflowRunStorageOptions } from "./src/deno/provider.ts"; export { useWorkflowLifecycle } from "./src/deno/lifecycle.ts"; export { useWorkflowRunHost } from "./src/deno/run-host.ts"; +export type { WorkflowRunHostOptions } from "./src/deno/run-host.ts"; +export { isGitWorkflowRunCreation } from "./src/lifecycle/execution.ts"; export type { + GitWorkflowRunCreationV1, + SourceBundleWorkflowRunCreationV2, WorkflowBeginRequest, WorkflowExecutionTransitions, WorkflowExecutionBegun, @@ -47,7 +51,16 @@ export type { WorkflowLifecycleOptions } from "./src/deno/lifecycle.ts"; * nothing that reads or writes a container is exported from any entrypoint. */ export { gitBlobIdentity } from "./src/deno/artifact/source.ts"; -export type { WorkflowDefinitionSourceReader } from "./src/deno/artifact/source.ts"; +export type { + GitDefinitionSourceClosureV1, + GitDefinitionSourceComponentV1, + GitDefinitionSourceRootV1, + GitRetainedDefinitionSourcesV1, + LegacyWorkflowSourceReader, + RetainedDefinitionSources, + SourceBundleRetainedDefinitionSourcesV2, + SourceBundleRetainedSourceV2, +} from "./src/lifecycle/source.ts"; export type { DetachedXmdArtifact, VerifiedXmdArtifact, diff --git a/packages/workflow/mod.ts b/packages/workflow/mod.ts index 9f4112870..7f189a6cb 100644 --- a/packages/workflow/mod.ts +++ b/packages/workflow/mod.ts @@ -3,12 +3,20 @@ * * Workflow runs for Executable.md. * - * A workflow run is a run of one immutable definition — a Git object and the - * path of the root document inside it — from one resolved base, recorded - * durably before the root document is imported so later document executions - * and durable effects share one explicit identity. The run itself is retained, - * so another process can find it by its public id and continue from durable - * data rather than from whoever happened to be holding the journal. + * A workflow run is a run of one immutable definition, recorded durably before + * the root document is imported so later document executions and durable + * effects share one explicit identity. The run itself is retained, so another + * process can find it by its public id and continue from durable data rather + * than from whoever happened to be holding the journal. + * + * A definition is one of two things. Version 1 is a Git object and the path of + * the root document inside it, run from one resolved base; its Markdown lives + * in a repository, and a trusted host supplies the reader that fetches it. + * Version 2 is a **source bundle**: the exact bytes themselves, addressed by + * portable logical paths and retained with the run. A source-bundle run needs + * no repository to start, resume, replay or export — which is what lets a file + * outside Git, an untracked file, and a file edited since its last commit each + * be one immutable definition the moment it is retained. * * ```ts * import { workflowInstallation } from "@executablemd/workflow"; @@ -63,6 +71,8 @@ export type { GitApi, GitObjectFormat } from "./src/git.ts"; export { getWorkflowRun, retainedWorkflowInstallation, workflowInstallation } from "./src/run.ts"; export { workflowBundleInstallation, WorkflowBundleHistoryError } from "./src/bundle.ts"; export type { WorkflowRun } from "./src/run.ts"; +export { isGitWorkflowRun, workflowRunValue } from "./src/journal.ts"; +export type { GitWorkflowRunV1, SourceBundleWorkflowRunV2 } from "./src/journal.ts"; export { useWorkflowServiceDenial, WorkflowServiceDeniedError } from "./src/service-denial.ts"; export { RepositoryComposition } from "./src/composition/api.ts"; @@ -303,6 +313,23 @@ export type { WorkflowRunTransaction, } from "./src/storage/api.ts"; +export { isGitWorkflowRunCreation } from "./src/lifecycle/execution.ts"; +export type { + GitWorkflowRunCreationV1, + SourceBundleWorkflowRunCreationV2, + WorkflowRunCreation, +} from "./src/lifecycle/execution.ts"; +export type { + GitDefinitionSourceClosureV1, + GitDefinitionSourceComponentV1, + GitDefinitionSourceRootV1, + GitRetainedDefinitionSourcesV1, + LegacyWorkflowSourceReader, + RetainedDefinitionSources, + SourceBundleRetainedDefinitionSourcesV2, + SourceBundleRetainedSourceV2, +} from "./src/lifecycle/source.ts"; + export { WorkflowLifecycle, WorkflowLifecycleProviderError } from "./src/lifecycle/api.ts"; export type { ExecutorAcquisition, @@ -347,7 +374,10 @@ export type { export { definitionComponents, + definitionTargetPath, definitionToJson, + isGitWorkflowDefinition, + isSourceBundleWorkflowDefinition, parseWorkflowDefinition, } from "./src/storage/definition.ts"; export type { @@ -375,9 +405,15 @@ export type { } from "./src/storage/source-bundle.ts"; export { conflictingFields } from "./src/storage/compatibility.ts"; +export type { + GitWorkflowRunComparisonV1, + SourceBundleWorkflowRunComparisonV2, + WorkflowRunComparison, +} from "./src/storage/compatibility.ts"; export { canonicalJson, + isGitWorkflowRunRecord, parseStopReasonInput, parseWorkflowRunStatus, parseWorkflowStopReason, @@ -385,6 +421,8 @@ export { } from "./src/storage/record.ts"; export type { DefinitionRetrieval, + GitWorkflowRunRecordV1, + SourceBundleWorkflowRunRecordV2, DocumentExecutionCompletion, DocumentExecutionRecord, StoredRunState, @@ -394,10 +432,15 @@ export type { } from "./src/storage/record.ts"; export { + LegacyWorkflowSourceMismatchError, + LegacyWorkflowSourceReaderUnavailableError, + LegacyWorkflowSourceUnavailableError, WorkflowDatabaseClosedError, WorkflowDatabaseCorruptError, WorkflowDatabaseFormatError, + WorkflowDefinitionCorruptError, WorkflowDefinitionError, + WorkflowDefinitionSourceMissingError, WorkflowDocumentExecutionError, WorkflowIncompleteVersionOneError, WorkflowInspectionRecoveryError, diff --git a/packages/workflow/src/deno/artifact-frontier.ts b/packages/workflow/src/deno/artifact-frontier.ts index c8957363d..2a601669a 100644 --- a/packages/workflow/src/deno/artifact-frontier.ts +++ b/packages/workflow/src/deno/artifact-frontier.ts @@ -28,14 +28,8 @@ */ import type { DatabaseSync } from "node:sqlite"; -import { Err, Ok, type Result } from "effection"; import type { Json } from "@executablemd/durable-streams"; -import type { WorkflowDefinition } from "../storage/definition.ts"; -import type { - DetachedXmdArtifact, - XmdArtifactDefinitionClosure, - XmdArtifactJournalRow, -} from "./artifact/types.ts"; +import type { DetachedXmdArtifact, XmdArtifactJournalRow } from "./artifact/types.ts"; import type { InheritedEventProvenance } from "../lifecycle/history.ts"; import { WorkflowRequestError } from "../storage/errors.ts"; import type { WorkflowRunRecord } from "../storage/record.ts"; @@ -120,36 +114,6 @@ export function readRetrievalMetadata(database: DatabaseSync): Json | undefined return readRetrieval(row).metadata; } -/** - * Whether a fetched closure is this run's, as a refusal or nothing. - * - * The identities are compared, not the bytes: whether the Markdown hashes to - * what the definition names is the artifact writer's question, and asking it - * twice in two places would be two answers to keep in agreement. What this - * catches is the closure belonging to a different definition entirely, which - * no amount of hashing downstream would notice. - */ -export function matchesRetainedDefinition( - definition: WorkflowDefinition, - closure: XmdArtifactDefinitionClosure, -): Result { - const root = closure.root; - if ( - root.objectFormat !== definition.objectFormat || - root.pinnedCommit !== definition.objectId || - root.rootDocumentPath !== definition.rootDocumentPath || - root.targetPath !== definition.targetPath - ) { - return Err( - new WorkflowRequestError( - "the definition source this host read back does not describe the definition the run " + - "retains, so it is not this run's source.", - ), - ); - } - return Ok(); -} - function readLineageCreatedAt(database: DatabaseSync, path: string): string { const row = reading(database, "SELECT created_at FROM workflow_fork_lineage WHERE id = 1").get(); const createdAt = row?.["created_at"]; diff --git a/packages/workflow/src/deno/artifact/manifest.ts b/packages/workflow/src/deno/artifact/manifest.ts index 7a0893a27..3d4216acd 100644 --- a/packages/workflow/src/deno/artifact/manifest.ts +++ b/packages/workflow/src/deno/artifact/manifest.ts @@ -34,17 +34,25 @@ import type { XmdArtifactManifestV1, } from "./types.ts"; -/** The artifact manifest version this build produces and reads. */ +/** The artifact manifest version a format-1 artifact produces and reads. */ export const XMD_ARTIFACT_MANIFEST_VERSION = 1; +/** The manifest version a format-2 artifact produces and reads. */ +export const XMD_ARTIFACT_SOURCE_BUNDLE_MANIFEST_VERSION = 2; + /** * What the artifact identity is a hash *of*, beyond the manifest bytes. * * Domain separation, so the same bytes appearing as some other structure's - * canonical encoding cannot be presented as an artifact identity. + * canonical encoding cannot be presented as an artifact identity — and so one + * format's manifest can never derive the other's identity, however similar the + * two inventories happen to look. */ export const XMD_ARTIFACT_IDENTITY_DOMAIN = "xmd-artifact\0v1\0"; +/** The same separation for format 2, which is a different set of records. */ +export const XMD_ARTIFACT_SOURCE_BUNDLE_IDENTITY_DOMAIN = "xmd-artifact\0v2\0"; + const encoder = new TextEncoder(); /** The compact canonical JSON encoding of a value, as UTF-8 bytes. */ @@ -125,6 +133,7 @@ export interface XmdArtifactManifestBuild { export function buildXmdArtifactManifest( entries: readonly XmdArtifactContentEntry[], duplicate: (kind: string) => never, + version: 1 | 2 = XMD_ARTIFACT_MANIFEST_VERSION, ): XmdArtifactManifestBuild { const seen = new Set(); const rows: Array<{ row: XmdArtifactManifestEntryV1; entry: XmdArtifactContentEntry }> = []; @@ -139,14 +148,14 @@ export function buildXmdArtifactManifest( rows.sort((left, right) => compareEntries(left.row, right.row)); const manifest: XmdArtifactManifestV1 = Object.freeze({ - version: XMD_ARTIFACT_MANIFEST_VERSION, + version, entries: Object.freeze(rows.map((each) => each.row)), }); const bytes = canonicalJsonBytes(manifestToJson(manifest)); return Object.freeze({ manifest, bytes, - identity: deriveXmdArtifactIdentity(bytes), + identity: deriveXmdArtifactIdentity(bytes, version), ordered: Object.freeze(rows.map((each) => each.entry)), }); } @@ -172,9 +181,12 @@ export function manifestToJson(manifest: XmdArtifactManifestV1): Json { } /** The lowercase SHA-256 of the domain prefix followed by the manifest bytes. */ -export function deriveXmdArtifactIdentity(manifestBytes: Uint8Array): string { +export function deriveXmdArtifactIdentity(manifestBytes: Uint8Array, version: 1 | 2 = 1): string { return createHash("sha256") - .update(XMD_ARTIFACT_IDENTITY_DOMAIN, "utf8") + .update( + version === 2 ? XMD_ARTIFACT_SOURCE_BUNDLE_IDENTITY_DOMAIN : XMD_ARTIFACT_IDENTITY_DOMAIN, + "utf8", + ) .update(manifestBytes) .digest("hex"); } diff --git a/packages/workflow/src/deno/artifact/read.ts b/packages/workflow/src/deno/artifact/read.ts index 76e0c49d9..e48f545cd 100644 --- a/packages/workflow/src/deno/artifact/read.ts +++ b/packages/workflow/src/deno/artifact/read.ts @@ -28,7 +28,6 @@ import { DatabaseSync } from "node:sqlite"; import { lstat } from "@effectionx/fs"; import { Err, Ok, type Operation, type Result } from "effection"; import type { Json } from "@executablemd/durable-streams"; -import type { WorkflowDefinition } from "../../storage/definition.ts"; import { type JsonObject, parseJsonValue } from "../../storage/members.ts"; import type { DocumentExecutionRecord, WorkflowRunRecord } from "../../storage/record.ts"; import type { RetainedAnswer } from "../answers.ts"; @@ -60,20 +59,25 @@ import { translateArtifactSqliteError, recognizeXmdArtifactContainer, verifyXmdArtifactFormatVersion, + type XmdArtifactFormatVersion, verifyXmdArtifactStructure, XMD_ARTIFACT_EXTENSION, } from "./schema.ts"; import { XMD_ARTIFACT_CONTENT_KINDS, + XMD_ARTIFACT_CONTENT_KINDS_V2, type VerifiedXmdArtifact, type XmdArtifactAgentEvidence, type XmdArtifactAgentPortability, type XmdArtifactContentEntry, type XmdArtifactContents, - type XmdArtifactDefinitionClosure, type XmdArtifactEncoding, type XmdArtifactJournalRow, } from "./types.ts"; +import type { RetainedDefinitionSources } from "../../lifecycle/source.ts"; +import type { SourceBundleWorkflowDefinitionV2 } from "../../storage/source-bundle.ts"; +import { isGitWorkflowRunRecord } from "../../storage/record.ts"; +import type { GitWorkflowDefinitionV1 } from "../../storage/definition.ts"; const SELECT_HEADER_VERSION = "SELECT artifact_version FROM xmd_artifact_header WHERE id = 1"; const SELECT_HEADER = @@ -81,7 +85,13 @@ const SELECT_HEADER = const SELECT_CONTENT = "SELECT kind, identity, encoding, length, sha256, content FROM xmd_artifact_content"; -const KINDS: ReadonlySet = new Set(XMD_ARTIFACT_CONTENT_KINDS); +const KINDS_V1: ReadonlySet = new Set(XMD_ARTIFACT_CONTENT_KINDS); +const KINDS_V2: ReadonlySet = new Set(XMD_ARTIFACT_CONTENT_KINDS_V2); + +/** The closed inventory one semantic format declares, and no other. */ +function kindsFor(format: XmdArtifactFormatVersion): ReadonlySet { + return format === 2 ? KINDS_V2 : KINDS_V1; +} /** Whether a stored column names one of the three ways bytes may be read. */ function isXmdArtifactEncoding(value: unknown): value is XmdArtifactEncoding { @@ -114,6 +124,8 @@ interface SealedContent { readonly entries: readonly XmdArtifactContentEntry[]; readonly manifest: Uint8Array; readonly identity: string; + /** The semantic format its header declared, which chose every gate after it. */ + readonly format: XmdArtifactFormatVersion; } /** @@ -189,7 +201,7 @@ function sealedContent(database: DatabaseSync, path: string): SealedContent { // does not implement is an unsupported version whatever its schema looks // like. Read through a targeted statement: a header that is missing or is // not shaped to answer this is a schema failure, and says so. - verifyXmdArtifactFormatVersion(declaredFormatVersion(database, path), path); + const format = verifyXmdArtifactFormatVersion(declaredFormatVersion(database, path), path); // Gate 5: the complete declared schema, the declared references, and the // singleton the header is. @@ -200,9 +212,10 @@ function sealedContent(database: DatabaseSync, path: string): SealedContent { // Gates 6 and 7: every entry held to what its own row declares, and all of it // into detached memory. return Object.freeze({ - entries: readContent(database, path), + entries: readContent(database, path, format), manifest: header.manifest, identity: header.identity, + format, }); } @@ -225,12 +238,16 @@ function* recognized(sealed: SealedContent, path: string): Operation { - throw new XmdArtifactInventoryError( - path, - `it holds more than one ${kind} record under one identity`, - ); - }); + const rebuilt = buildXmdArtifactManifest( + sealed.entries, + (kind) => { + throw new XmdArtifactInventoryError( + path, + `it holds more than one ${kind} record under one identity`, + ); + }, + sealed.format, + ); if (Buffer.compare(Buffer.from(rebuilt.bytes), Buffer.from(sealed.manifest)) !== 0) { throw new XmdArtifactManifestMismatchError(path); } @@ -310,11 +327,16 @@ function readHeader( * rather than skipped: version 1's inventory is closed, so a record nobody * declared is an undeclared semantic record and therefore corruption. */ -function readContent(database: DatabaseSync, path: string): XmdArtifactContentEntry[] { +function readContent( + database: DatabaseSync, + path: string, + format: XmdArtifactFormatVersion, +): XmdArtifactContentEntry[] { + const kinds = kindsFor(format); const entries: XmdArtifactContentEntry[] = []; for (const row of reading(database, SELECT_CONTENT).all()) { const kind = row["kind"]; - if (typeof kind !== "string" || !KINDS.has(kind)) { + if (typeof kind !== "string" || !kinds.has(kind)) { throw new XmdArtifactInventoryError( path, "it holds a record of a kind this artifact version does not declare", @@ -445,19 +467,25 @@ function sealedBytes(bytes: Uint8Array): () => Uint8Array { } function frozenRun(run: WorkflowRunRecord): WorkflowRunRecord { - return Object.freeze({ + const shared = { runId: run.runId, - definition: frozenDefinition(run.definition), - base: run.base, props: frozenJsonObject(run.props), status: run.status, ...(run.stopReason === undefined ? {} : { stopReason: Object.freeze({ ...run.stopReason }) }), createdAt: run.createdAt, updatedAt: run.updatedAt, - }); + }; + if (isGitWorkflowRunRecord(run)) { + return Object.freeze({ + ...shared, + definition: frozenGitDefinition(run.definition), + base: run.base, + }); + } + return Object.freeze({ ...shared, definition: frozenSourceBundle(run.definition) }); } -function frozenDefinition(definition: WorkflowDefinition): WorkflowDefinition { +function frozenGitDefinition(definition: GitWorkflowDefinitionV1): GitWorkflowDefinitionV1 { return Object.freeze({ version: definition.version, kind: definition.kind, @@ -475,6 +503,27 @@ function frozenDefinition(definition: WorkflowDefinition): WorkflowDefinition { }); } +function frozenSourceBundle( + definition: SourceBundleWorkflowDefinitionV2, +): SourceBundleWorkflowDefinitionV2 { + return Object.freeze({ + version: definition.version, + kind: definition.kind, + hashAlgorithm: definition.hashAlgorithm, + bundleHash: definition.bundleHash, + entrypoint: definition.entrypoint, + sources: Object.freeze(definition.sources.map((source) => Object.freeze({ ...source }))), + ...(definition.targetPath === undefined ? {} : { targetPath: definition.targetPath }), + ...(definition.components === undefined + ? {} + : { + components: Object.freeze( + definition.components.map((component) => Object.freeze({ ...component })), + ), + }), + }); +} + function frozenExecution(execution: DocumentExecutionRecord): DocumentExecutionRecord { return Object.freeze({ ...execution, @@ -599,10 +648,36 @@ function frozenPortability(record: XmdArtifactAgentPortability): XmdArtifactAgen }); } -function frozenClosure(closure: XmdArtifactDefinitionClosure): XmdArtifactDefinitionClosure { +function frozenClosure(sources: RetainedDefinitionSources): RetainedDefinitionSources { + if (sources.definitionVersion === 1) { + return Object.freeze({ + definitionVersion: 1, + definition: frozenGitDefinition(sources.definition), + closure: Object.freeze({ + root: Object.freeze({ ...sources.closure.root }), + components: Object.freeze( + sources.closure.components.map((each) => Object.freeze({ ...each })), + ), + }), + }); + } + // Byte leaves behind an accessor, on the same terms as every other byte + // member here: the sealed array stays in the closure and each read answers + // with a copy, so evidence a caller received is evidence it cannot edit. return Object.freeze({ - root: Object.freeze({ ...closure.root }), - components: Object.freeze(closure.components.map((each) => Object.freeze({ ...each }))), + definitionVersion: 2, + definition: frozenSourceBundle(sources.definition), + sources: Object.freeze( + sources.sources.map((source) => { + const bytes = sealedBytes(source.bytes); + return Object.freeze({ + path: source.path, + get bytes(): Uint8Array { + return bytes(); + }, + }); + }), + ), }); } diff --git a/packages/workflow/src/deno/artifact/records.ts b/packages/workflow/src/deno/artifact/records.ts index 08cf5b18c..9f9b140c6 100644 --- a/packages/workflow/src/deno/artifact/records.ts +++ b/packages/workflow/src/deno/artifact/records.ts @@ -25,7 +25,6 @@ * kind of file. */ -import { createHash } from "node:crypto"; import { Buffer } from "node:buffer"; import type { Operation } from "effection"; import { prepareElicitation, validateParsed } from "@executablemd/core"; @@ -104,6 +103,17 @@ import type { XmdArtifactJournalRow, XmdArtifactProviderSessionIdentity, } from "./types.ts"; +import type { RetainedDefinitionSources } from "../../lifecycle/source.ts"; +import { + sourceBundleHash, + type SourceBundleWorkflowDefinitionV2, + sourceContentHash, +} from "../../storage/source-bundle.ts"; +import { + type GitWorkflowRunRecordV1, + isGitWorkflowRunRecord, + type SourceBundleWorkflowRunRecordV2, +} from "../../storage/record.ts"; const encoder = new TextEncoder(); const decoder = new TextDecoder("utf-8", { fatal: true }); @@ -215,11 +225,14 @@ export function encodeXmdArtifactInventory( }), ); + // The run record carries what its own version retains and nothing more: a + // version-2 entry admits no base and no pinned commit, because a source + // bundle never had a repository state to name. entries.push( json("workflow-run", null, { runId: contents.run.runId, definition: definitionToJson(contents.run.definition), - base: contents.run.base, + ...(isGitWorkflowRunRecord(contents.run) ? { base: contents.run.base } : {}), props: contents.run.props, status: contents.run.status, ...(contents.run.stopReason === undefined @@ -366,27 +379,49 @@ export function encodeXmdArtifactInventory( } } - const root = contents.definition.root; - entries.push( - json("definition-source-root", null, { - objectFormat: root.objectFormat, - pinnedCommit: root.pinnedCommit, - rootDocumentPath: root.rootDocumentPath, - ...(root.targetPath === undefined ? {} : { targetPath: root.targetPath }), - blobId: root.blobId, - }), - ); - entries.push(utf8("definition-source-root-content", null, root.content)); + if (contents.definition.definitionVersion === 1) { + const root = contents.definition.closure.root; + entries.push( + json("definition-source-root", null, { + objectFormat: root.objectFormat, + pinnedCommit: root.pinnedCommit, + rootDocumentPath: root.rootDocumentPath, + ...(root.targetPath === undefined ? {} : { targetPath: root.targetPath }), + blobId: root.blobId, + }), + ); + entries.push(utf8("definition-source-root-content", null, root.content)); + + for (const component of contents.definition.closure.components) { + entries.push( + json("definition-source-component", component.name, { + name: component.name, + path: component.path, + blobId: component.blobId, + }), + ); + entries.push(utf8("definition-source-component-content", component.name, component.content)); + } + return entries; + } - for (const component of contents.definition.components) { + // One entry and one content value per logical source path, both keyed by the + // canonical JSON string of that path. Bytes rather than text: the retained + // BLOB is what the run executes, and re-encoding it through a decoder would + // seal something the bundle hash does not describe. + const declared = new Map( + contents.definition.definition.sources.map((source) => [source.path, source]), + ); + for (const source of contents.definition.sources) { + const entry = declared.get(source.path); entries.push( - json("definition-source-component", component.name, { - name: component.name, - path: component.path, - blobId: component.blobId, + json("definition-source-entry", source.path, { + path: source.path, + sourceHash: entry?.sourceHash ?? "", + byteLength: entry?.byteLength ?? source.bytes.byteLength, }), ); - entries.push(utf8("definition-source-component-content", component.name, component.content)); + entries.push(raw("definition-source-content", source.path, source.bytes)); } return entries; @@ -666,7 +701,7 @@ export function decodeXmdArtifactInventory( const worktrees = decodeWorktrees(inventory, path); const answers = decodeAnswers(inventory, path); const agentSessions = decodeAgentSessions(inventory, path); - const closure = decodeDefinitionClosure(inventory, path); + const closure = decodeDefinitionSources(inventory, path, run); inventory.requireNothingLeftOver(); @@ -712,15 +747,41 @@ function decodeRun(inventory: Inventory, path: string): WorkflowRunRecord { kind, ); const reason = stopReason(parsed, path, kind); - const record: WorkflowRunRecord = { + const retained = definition(parsed, path, kind); + const shared = { runId: parseRunId(parsed.get("runId"), "$.runId", failing(path, kind)), - definition: definition(parsed, path, kind), - base: required(parsed, "base", path, kind), props: parseJsonObject(parsed.get("props"), "$.props", failing(path, kind)), status: status(parsed, "status", path, kind), createdAt: instant(parsed, "createdAt", path, kind), updatedAt: instant(parsed, "updatedAt", path, kind), }; + + if (retained.kind === "source-bundle") { + // A base here would be a repository state this run never had, so a + // version-2 entry that carries one does not describe a version-2 run. + if (parsed.get("base") !== undefined) { + throw new XmdArtifactRecordError(path, kind, "a source-bundle run retains no base"); + } + const record: SourceBundleWorkflowRunRecordV2 = { + runId: shared.runId, + definition: retained, + props: shared.props, + status: shared.status, + createdAt: shared.createdAt, + updatedAt: shared.updatedAt, + }; + return Object.freeze(reason === undefined ? record : { ...record, stopReason: reason }); + } + + const record: GitWorkflowRunRecordV1 = { + runId: shared.runId, + definition: retained, + base: required(parsed, "base", path, kind), + props: shared.props, + status: shared.status, + createdAt: shared.createdAt, + updatedAt: shared.updatedAt, + }; return Object.freeze(reason === undefined ? record : { ...record, stopReason: reason }); } @@ -1194,6 +1255,95 @@ function decodeAgentSessions(inventory: Inventory, path: string): readonly Agent ); } +/** + * The source closure this artifact carries, in the form its run's version has. + * + * Driven by the definition already decoded rather than by which entries happen + * to be present: a format-2 artifact admits no Git closure and a format-1 one + * admits no source bundle, so the version decides which kinds are claimed and + * `requireNothingLeftOver` refuses whatever the other version would have used. + */ +function decodeDefinitionSources( + inventory: Inventory, + path: string, + run: WorkflowRunRecord, +): RetainedDefinitionSources { + if (run.definition.kind === "source-bundle") { + return decodeSourceBundle(inventory, path, run.definition); + } + return Object.freeze({ + definitionVersion: 1, + definition: run.definition, + closure: decodeDefinitionClosure(inventory, path), + }); +} + +/** + * The retained bytes a format-2 artifact carries, one pair per logical path. + * + * Each pair's natural identity is the canonical JSON string of its path, and + * both halves are claimed under it: an entry whose own `path` is not the + * identity it was stored under would let one source be read as another's. + * + * The declared hash and length are read out with the path rather than skipped + * over. They are the entry's own statement about its content, and a statement + * nobody reads is a field a re-sealed artifact could move freely — so they are + * carried to semantic verification and compared with the descriptor there. + */ +function decodeSourceBundle( + inventory: Inventory, + path: string, + definition: SourceBundleWorkflowDefinitionV2, +): RetainedDefinitionSources { + const kind = "definition-source-entry"; + const retained = new Map(definition.sources.map((source) => [source.path, source])); + const sources = inventory.identities(kind).map((identity) => { + const parsed = members( + structured(inventory.take(kind, identity), path), + ["path", "sourceHash", "byteLength"], + path, + kind, + ); + const logical = required(parsed, "path", path, kind); + if (canonicalJsonText(logical) !== canonicalJsonText(identity)) { + throw new XmdArtifactInventoryError( + path, + "a definition source is stored under an identity it does not carry", + ); + } + // Held to the descriptor here, where they are read. The bytes beside them + // are checked against the descriptor too, so agreeing with it is what makes + // the entry, the content and the definition one statement rather than three + // a re-sealed artifact could move independently. + const entry = retained.get(logical); + if (entry === undefined) { + throw new XmdArtifactInventoryError( + path, + "it carries a definition source the run's descriptor does not retain", + ); + } + if (required(parsed, "sourceHash", path, kind) !== entry.sourceHash) { + throw new XmdArtifactRecordError( + path, + kind, + "its declared source hash is not the one the definition names", + ); + } + if (whole(parsed, "byteLength", path, kind) !== entry.byteLength) { + throw new XmdArtifactRecordError( + path, + kind, + "its declared byte length is not the one the definition names", + ); + } + return Object.freeze({ + path: logical, + bytes: bytesOf(inventory.take("definition-source-content", identity), path), + }); + }); + return Object.freeze({ definitionVersion: 2, definition, sources: Object.freeze(sources) }); +} + function decodeDefinitionClosure(inventory: Inventory, path: string): XmdArtifactDefinitionClosure { const kind = "definition-source-root"; const parsed = members( @@ -1285,7 +1435,7 @@ export function* verifyXmdArtifactSemantics( verifyCheckouts(contents, reject); yield* verifySuspensions(contents, path, reject); verifyAgentSessions(contents, reject); - verifyDefinitionClosure(contents, reject); + yield* verifyDefinitionSources(contents, reject); } function verifyLifecycle( @@ -1741,9 +1891,84 @@ function verifyAgentSessions(contents: XmdArtifactContents, reject: Reject): voi * commit; and each blob identity must be the Git object id of the bytes stored * beside it, or the closure is not the source that definition names. */ +/** + * The source closure, held to the definition the run retains, by its version. + * + * Version 1 checks Git identities; version 2 recomputes every source hash from + * the retained bytes and then the bundle hash from the manifest those hashes + * make. Neither version's verifier is asked about the other's closure: the two + * describe different things, and a shared check would be one that could + * complete without having proved either. + */ +function* verifyDefinitionSources(contents: XmdArtifactContents, reject: Reject): Operation { + if (contents.definition.definitionVersion === 2) { + yield* verifySourceBundleClosure(contents, reject); + return; + } + verifyDefinitionClosure(contents, reject); +} + +/** + * Exactly one entry and content pair per descriptor path, and nothing else. + * + * The lengths the manifest and the descriptor declare must both be the bytes' + * own, every source hash must be what those bytes produce, and the bundle hash + * must be what the resulting manifest and mapping produce — so an artifact + * whose embedded content drifted from its descriptor is refused before any + * status or history is returned. + */ +function* verifySourceBundleClosure( + contents: XmdArtifactContents, + reject: Reject, +): Operation { + if (contents.definition.definitionVersion !== 2) { + reject("its definition source closure is not the source bundle the run retains"); + return; + } + const { definition, sources } = contents.definition; + if (contents.run.definition.kind !== "source-bundle") { + reject("its definition source closure is not the source bundle the run retains"); + return; + } + if (contents.run.definition.bundleHash !== definition.bundleHash) { + reject("its definition source closure describes a bundle other than the run's"); + return; + } + if (sources.length !== definition.sources.length) { + reject("it does not carry exactly one source pair per retained path"); + return; + } + + const carried = new Map(sources.map((source) => [source.path, source.bytes])); + if (carried.size !== sources.length) { + reject("it carries one logical source path more than once"); + return; + } + for (const entry of definition.sources) { + const bytes = carried.get(entry.path); + if (bytes === undefined) { + reject("it does not carry every source the definition retains"); + return; + } + if (bytes.byteLength !== entry.byteLength) { + reject("a carried source is not the length the definition declares"); + } + if ((yield* sourceContentHash(bytes)) !== entry.sourceHash) { + reject("a carried source is not the content the definition names"); + } + } + if ((yield* sourceBundleHash(definition)) !== definition.bundleHash) { + reject("its bundle hash is not the one its own manifest produces"); + } +} + function verifyDefinitionClosure(contents: XmdArtifactContents, reject: Reject): void { const definition = contents.run.definition; - const root = contents.definition.root; + if (definition.kind === "source-bundle" || contents.definition.definitionVersion === 2) { + reject("its definition source closure is not the Git closure the run retains"); + return; + } + const root = contents.definition.closure.root; if ( root.objectFormat !== definition.objectFormat || root.pinnedCommit !== definition.objectId || @@ -1757,7 +1982,7 @@ function verifyDefinitionClosure(contents: XmdArtifactContents, reject: Reject): } const declared = definitionComponents(definition); - const carried = contents.definition.components; + const carried = contents.definition.closure.components; if (declared.length !== carried.length) { reject("its definition source closure does not carry every declared component"); } diff --git a/packages/workflow/src/deno/artifact/schema.ts b/packages/workflow/src/deno/artifact/schema.ts index 6a8fdc8e1..e81d59117 100644 --- a/packages/workflow/src/deno/artifact/schema.ts +++ b/packages/workflow/src/deno/artifact/schema.ts @@ -48,9 +48,15 @@ export const XMD_ARTIFACT_APPLICATION_ID = 0x584d4441; /** The only container schema version this build reads or writes. */ export const XMD_ARTIFACT_CONTAINER_VERSION = 1; -/** The only artifact format version this build reads or writes. */ +/** The artifact format a Git-definition run is sealed as. */ export const XMD_ARTIFACT_FORMAT_VERSION = 1; +/** The artifact format a source-bundle run is sealed as. */ +export const XMD_ARTIFACT_SOURCE_BUNDLE_FORMAT_VERSION = 2; + +/** Every semantic artifact format this build reads or writes. */ +export type XmdArtifactFormatVersion = 1 | 2; + /** The extension the public format is named by. */ export const XMD_ARTIFACT_EXTENSION = ".xmd"; @@ -197,10 +203,20 @@ export function recognizeXmdArtifactContainer( * them mean. A future artifact version inside a version-1 container is still a * file this build must not guess at, and never one it rewrites. */ -export function verifyXmdArtifactFormatVersion(stored: number, path: string): void { - if (stored !== XMD_ARTIFACT_FORMAT_VERSION) { - throw new XmdArtifactFormatVersionError(path, stored, XMD_ARTIFACT_FORMAT_VERSION); +export function verifyXmdArtifactFormatVersion( + stored: number, + path: string, +): XmdArtifactFormatVersion { + // The physical container stays at version 1 for both. Format 2 is a different + // set of records inside the same layout, so a build that reads one layout may + // still meet a semantic version it cannot verify. + if (stored === XMD_ARTIFACT_FORMAT_VERSION) { + return 1; + } + if (stored === XMD_ARTIFACT_SOURCE_BUNDLE_FORMAT_VERSION) { + return 2; } + throw new XmdArtifactFormatVersionError(path, stored, XMD_ARTIFACT_SOURCE_BUNDLE_FORMAT_VERSION); } /** diff --git a/packages/workflow/src/deno/artifact/types.ts b/packages/workflow/src/deno/artifact/types.ts index fce00f740..f88bf262f 100644 --- a/packages/workflow/src/deno/artifact/types.ts +++ b/packages/workflow/src/deno/artifact/types.ts @@ -37,6 +37,12 @@ import type { RetainedWorktree, } from "../fork-source.ts"; import type { StoredWorkspaceRoot } from "../workspace/manifest.ts"; +import type { + GitDefinitionSourceClosureV1, + GitDefinitionSourceComponentV1, + GitDefinitionSourceRootV1, + RetainedDefinitionSources, +} from "../../lifecycle/source.ts"; /** The boundary the lifecycle chose, as the shape every record here is about. */ export type { XmdArtifactFrontier }; @@ -67,40 +73,16 @@ export interface XmdArtifactJournalRow { } /** - * The root document this run is a run of, and the bytes behind it. + * What one definition is closed over, spelled where the lifecycle spells it. * - * The descriptor members repeat what the workflow definition already pins so - * that the closure can be checked against it: an artifact whose embedded - * Markdown belongs to a different commit than the definition names is not a - * closure of that definition, and a fork made from it would continue a document - * the run never ran. + * The source contract belongs to the lifecycle rather than to this encoder: a + * closure is what a run executes, and an artifact is one place it is written + * down. These aliases keep the names this directory already used without + * declaring a second copy of the shapes they name. */ -export interface XmdArtifactDefinitionRoot { - readonly objectFormat: "sha1" | "sha256"; - /** The commit the definition pins, as the definition's own object id. */ - readonly pinnedCommit: string; - readonly rootDocumentPath: string; - /** One exact canonical document target, when the definition selects one. */ - readonly targetPath?: string; - /** The Git blob identity of `content`, under `objectFormat`. */ - readonly blobId: string; - readonly content: string; -} - -/** One declared component, including one the run never expanded. */ -export interface XmdArtifactDefinitionComponent { - readonly name: string; - readonly path: string; - /** The Git blob identity of `content`, under the root's `objectFormat`. */ - readonly blobId: string; - readonly content: string; -} - -/** Everything a fork needs to continue this definition without its repository. */ -export interface XmdArtifactDefinitionClosure { - readonly root: XmdArtifactDefinitionRoot; - readonly components: readonly XmdArtifactDefinitionComponent[]; -} +export type XmdArtifactDefinitionRoot = GitDefinitionSourceRootV1; +export type XmdArtifactDefinitionComponent = GitDefinitionSourceComponentV1; +export type XmdArtifactDefinitionClosure = GitDefinitionSourceClosureV1; /** * One retained Prompt event, and the provider checkpoint token taken at it. @@ -217,7 +199,15 @@ export interface XmdArtifactContents { readonly agentSessions: readonly AgentSessionRecord[]; /** Absent unless the artifact classifies its Prompt-contributing sessions. */ readonly agentEvidence?: XmdArtifactAgentEvidence; - readonly definition: XmdArtifactDefinitionClosure; + /** + * The source this run executes, in the form its own version retains. + * + * A format-1 artifact carries the Git closure and only that; a format-2 + * artifact carries the source bundle and only that. The union is closed here + * so the writer selects a format from what it was handed rather than from a + * superset both versions could be read out of. + */ + readonly definition: RetainedDefinitionSources; } /** @@ -288,6 +278,36 @@ export type XmdArtifactContentKind = | "definition-source-component" | "definition-source-component-content"; +/** + * The closed set of content a format-2 artifact may hold. + * + * The same records as format 1 up to the definition, and then a source bundle + * instead of a Git closure. It admits no format-1 definition kind: one version's + * closure is never read as the other's, and a shared superset would be an + * inventory neither version's verifier could complete. + */ +export type XmdArtifactContentKindV2 = + | "artifact-frontier" + | "workflow-run" + | "document-execution" + | "fork-lineage" + | "journal-event" + | "journal-record" + | "workspace-root" + | "workspace-root-manifest" + | "dofs-manifest" + | "dofs-manifest-bytes" + | "dofs-blob" + | "dofs-blob-bytes" + | "workspace-repository" + | "workspace-worktree" + | "suspension-answer" + | "agent-session" + | "agent-session-portability" + | "agent-session-bundle-bytes" + | "definition-source-entry" + | "definition-source-content"; + /** Every declared kind, for recognition and for exhaustiveness. */ export const XMD_ARTIFACT_CONTENT_KINDS: readonly XmdArtifactContentKind[] = Object.freeze([ "agent-session", @@ -335,9 +355,15 @@ export interface XmdArtifactManifestEntryV1 { readonly sha256: string; } -/** The canonical versioned inventory of one artifact. */ +/** + * The canonical versioned inventory of one artifact. + * + * The version is the semantic format's, and the entry rows are the same shape + * in both: a manifest states what the artifact holds, and what those records + * mean is the format's question rather than the row's. + */ export interface XmdArtifactManifestV1 { - readonly version: 1; + readonly version: 1 | 2; readonly entries: readonly XmdArtifactManifestEntryV1[]; } @@ -348,3 +374,27 @@ export interface XmdArtifactContentEntry { readonly encoding: XmdArtifactEncoding; readonly content: Uint8Array; } + +/** Every kind format 2 declares, for recognition and for exhaustiveness. */ +export const XMD_ARTIFACT_CONTENT_KINDS_V2: readonly XmdArtifactContentKindV2[] = Object.freeze([ + "agent-session", + "agent-session-bundle-bytes", + "agent-session-portability", + "artifact-frontier", + "definition-source-content", + "definition-source-entry", + "document-execution", + "dofs-blob", + "dofs-blob-bytes", + "dofs-manifest", + "dofs-manifest-bytes", + "fork-lineage", + "journal-event", + "journal-record", + "suspension-answer", + "workflow-run", + "workspace-repository", + "workspace-root", + "workspace-root-manifest", + "workspace-worktree", +]); diff --git a/packages/workflow/src/deno/artifact/write.ts b/packages/workflow/src/deno/artifact/write.ts index c21e61d7d..5ceca7917 100644 --- a/packages/workflow/src/deno/artifact/write.ts +++ b/packages/workflow/src/deno/artifact/write.ts @@ -63,10 +63,9 @@ import { initializeXmdArtifactSchema, translateArtifactSqliteError, XMD_ARTIFACT_EXTENSION, - XMD_ARTIFACT_FORMAT_VERSION, XMD_ARTIFACT_CONTAINER_VERSION, } from "./schema.ts"; -import type { DetachedXmdArtifact, VerifiedXmdArtifact, XmdArtifactWriteResult } from "./types.ts"; +import type { DetachedXmdArtifact, XmdArtifactWriteResult } from "./types.ts"; const INSERT_CONTENT = `INSERT INTO xmd_artifact_content (kind, identity, encoding, length, sha256, content) VALUES (?, ?, ?, ?, ?, ?)`; @@ -125,6 +124,10 @@ function* seal(path: string, contents: DetachedXmdArtifact): Operation { - throw new XmdArtifactInventoryError( - path, - `the snapshot offers more than one ${kind} record under one identity`, - ); - }); + const built = buildXmdArtifactManifest( + entries, + (kind) => { + throw new XmdArtifactInventoryError( + path, + `the snapshot offers more than one ${kind} record under one identity`, + ); + }, + format, + ); let published = false; yield* ensure(function* () { @@ -148,7 +155,7 @@ function* seal(path: string, contents: DetachedXmdArtifact): Operation ({ + path: rowText(row, "workflow_definition_source.path"), + sourceHash: rowText(row, "workflow_definition_source.source_hash", "source_hash"), + })), + blobs: reading(database, SELECT_BLOBS) + .all() + .map((row) => ({ + sourceHash: rowText(row, "workflow_definition_blob.source_hash", "source_hash"), + byteLength: rowInteger(row, "workflow_definition_blob.byte_length", "byte_length"), + content: rowBytes(row, "workflow_definition_blob.content", "content"), + })), + }; +} + +/** + * The retained source this descriptor names, re-derived from what is stored. + * + * Every check the issue settles, in the order an operator can act on: the + * manifest must be exactly the descriptor's entries, every referenced blob must + * exist, both retained lengths must equal the content's own, recomputing each + * blob's hash must produce its key, no blob may be unreferenced, and the bundle + * hash must be what the retained manifest and mapping produce. + * + * Nothing partial is ever returned. A closure with one source missing is not a + * smaller closure, it is a different definition. + */ +export function* verifyRetainedSources( + definition: SourceBundleWorkflowDefinitionV2, + rows: RetainedSourceRows, +): Operation> { + const stored = new Map(rows.manifest.map((row) => [row.path, row.sourceHash])); + if (stored.size !== rows.manifest.length) { + return Err(new WorkflowDefinitionCorruptError("it retains one logical path more than once")); + } + if (rows.manifest.length > definition.sources.length) { + return Err(new WorkflowDefinitionCorruptError("it retains a source the descriptor does not")); + } + + const blobs = new Map(rows.blobs.map((row) => [row.sourceHash, row])); + if (blobs.size !== rows.blobs.length) { + return Err(new WorkflowDefinitionCorruptError("it retains one content hash more than once")); + } + + const referenced = new Set(); + const sources: SourceBundleRetainedSourceV2[] = []; + for (const entry of definition.sources) { + const retainedHash = stored.get(entry.path); + if (retainedHash === undefined) { + return Err(new WorkflowDefinitionSourceMissingError()); + } + if (retainedHash !== entry.sourceHash) { + return Err( + new WorkflowDefinitionCorruptError( + "a retained source names content the descriptor does not", + ), + ); + } + const blob = blobs.get(retainedHash); + if (blob === undefined) { + return Err(new WorkflowDefinitionSourceMissingError()); + } + referenced.add(retainedHash); + + if (blob.byteLength !== entry.byteLength || blob.content.byteLength !== entry.byteLength) { + return Err( + new WorkflowDefinitionCorruptError("a retained source is not the length it declares"), + ); + } + const recomputed = yield* sourceContentHash(blob.content); + if (recomputed !== entry.sourceHash) { + return Err( + new WorkflowDefinitionCorruptError("a retained source is not the content it names"), + ); + } + sources.push(Object.freeze({ path: entry.path, bytes: Uint8Array.from(blob.content) })); + } + + // Unreferenced content is corruption rather than tolerated garbage: a blob no + // manifest row names is bytes this run retains and cannot account for. + if (referenced.size !== blobs.size) { + return Err(new WorkflowDefinitionCorruptError("it retains content no source references")); + } + + const bundleHash = yield* sourceBundleHash(definition); + if (bundleHash !== definition.bundleHash) { + return Err( + new WorkflowDefinitionCorruptError("its bundle hash is not the one its manifest produces"), + ); + } + + return Ok( + Object.freeze({ + definitionVersion: 2, + definition, + sources: Object.freeze(sources), + }), + ); +} + +/** + * Hold a legacy reader's answer to the version-1 definition it was asked about. + * + * The adapter fetched it; this decides whether what came back is this run's. + * Every term the descriptor pins is compared — the object format, the pinned + * commit, the repository-relative path, the exact target, and the declared + * component set with its paths and blob identities — because a reader that + * returned another commit's bytes has not obtained this source, and none of + * what it returned may execute. + */ +export function validateLegacySources( + definition: GitWorkflowDefinitionV1, + answered: RetainedDefinitionSources, +): Result { + if (answered.definitionVersion !== 1) { + return Err( + new LegacyWorkflowSourceMismatchError("it describes a source bundle, not a Git object"), + ); + } + const { root, components } = answered.closure; + if ( + root.objectFormat !== definition.objectFormat || + root.pinnedCommit !== definition.objectId || + root.rootDocumentPath !== definition.rootDocumentPath || + root.targetPath !== definition.targetPath + ) { + return Err(new LegacyWorkflowSourceMismatchError("its root is not the object this run pins")); + } + // The declared identity is not taken on the reader's word: the blob id is + // recomputed from the bytes that came back, so a closure carrying one + // document's identity beside another's content is refused rather than + // executed. + if (gitBlobIdentity(root.content, root.objectFormat) !== root.blobId) { + return Err( + new LegacyWorkflowSourceMismatchError("its root content is not the object it declares"), + ); + } + + const declared = definitionComponents(definition); + if (components.length !== declared.length) { + return Err( + new LegacyWorkflowSourceMismatchError( + "it carries a different component set than this run declares", + ), + ); + } + for (let index = 0; index < declared.length; index++) { + const expected = declared[index]; + const supplied = components[index]; + if ( + expected === undefined || + supplied === undefined || + supplied.name !== expected.name || + supplied.path !== expected.path || + supplied.blobId !== expected.sourceHash + ) { + return Err( + new LegacyWorkflowSourceMismatchError( + "a component it carries is not one this run declares", + ), + ); + } + if (gitBlobIdentity(supplied.content, definition.objectFormat) !== supplied.blobId) { + return Err( + new LegacyWorkflowSourceMismatchError( + "a component's content is not the object it declares", + ), + ); + } + } + + return Ok(Object.freeze({ definitionVersion: 1, definition, closure: answered.closure })); +} + +function rowText(row: Record, location: string, column = "path"): string { + const value = row[column]; + if (typeof value !== "string") { + throw new WorkflowRecordMalformedError(location, "expected text"); + } + return value; +} + +function rowInteger(row: Record, location: string, column: string): number { + const value = row[column]; + if (typeof value === "bigint") { + if (value < 0n || value > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new WorkflowRecordMalformedError(location, "expected a whole number of bytes"); + } + return Number(value); + } + if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0) { + return value; + } + throw new WorkflowRecordMalformedError(location, "expected a whole number of bytes"); +} + +function rowBytes(row: Record, location: string, column: string): Uint8Array { + const value = row[column]; + if (value instanceof Uint8Array) { + return value; + } + throw new WorkflowRecordMalformedError(location, "expected bytes"); +} diff --git a/packages/workflow/src/deno/lifecycle.ts b/packages/workflow/src/deno/lifecycle.ts index 22eced242..f9ed7e1c0 100644 --- a/packages/workflow/src/deno/lifecycle.ts +++ b/packages/workflow/src/deno/lifecycle.ts @@ -50,13 +50,16 @@ import { WorkflowLifecycle, type WorkflowLifecycleSnapshot, } from "../lifecycle/api.ts"; -import type { - WorkflowBeginRequest, - WorkflowExecutionTransitions, - WorkflowExecutionBegun, - WorkflowForkRequest, +import { + isGitWorkflowRunCreation, + type WorkflowBeginRequest, + type WorkflowExecutionTransitions, + type WorkflowExecutionBegun, + type WorkflowForkRequest, + type WorkflowRunCreation, } from "../lifecycle/execution.ts"; import { forkRunRecordEvent } from "../fork.ts"; +import type { WorkflowRun } from "../journal.ts"; import { readForkSource, type ForkSourceSnapshot } from "./fork-source.ts"; import { removeProviderSessions } from "./provider-sessions.ts"; import { readForkLineage, type ForkHeadEvents } from "./fork-write.ts"; @@ -66,6 +69,8 @@ import { type WorkflowHistoryEntry, } from "../lifecycle/history.ts"; import { + LegacyWorkflowSourceReaderUnavailableError, + WorkflowDefinitionSourceMissingError, WorkflowInspectionRecoveryError, WorkflowRequestError, WorkflowRunIdMismatchError, @@ -93,19 +98,26 @@ import { workflowForkStaging, workflowRunPath } from "./path.ts"; import { authorizedRoot, checkRunId } from "./provider.ts"; import { reading, readTransaction } from "./reading.ts"; import type { DetachedXmdArtifact } from "./artifact/types.ts"; -import type { WorkflowDefinitionSourceReader } from "./artifact/source.ts"; -import type { WorkflowDefinition } from "../storage/definition.ts"; +import type { LegacyWorkflowSourceReader, RetainedDefinitionSources } from "../lifecycle/source.ts"; +import type { GitWorkflowDefinitionV1, WorkflowDefinition } from "../storage/definition.ts"; import type { Json } from "@executablemd/durable-streams"; import { writeXmdArtifact } from "./artifact/mod.ts"; import { historyArtifact, inspectArtifact } from "./artifact-inspection.ts"; +import { readExportFrontier, readRetrievalMetadata } from "./artifact-frontier.ts"; import { - matchesRetainedDefinition, - readExportFrontier, - readRetrievalMetadata, -} from "./artifact-frontier.ts"; + readDefinitionSourceRows, + type RetainedSourceRows, + validateLegacySources, + verifyRetainedSources, +} from "./definition-source.ts"; import type { WorkflowExportRequest, WorkflowExportResult } from "../lifecycle/export.ts"; import { readDocumentExecution, readRetrieval, readRunRecord } from "./rows.ts"; -import { translateSqliteError, verifySchema, WorkflowReadonlyRollbackError } from "./schema.ts"; +import { + liveSchemaVersion, + translateSqliteError, + verifySchema, + WorkflowReadonlyRollbackError, +} from "./schema.ts"; import { holdRecoveryCoordination } from "./recovery-coordination.ts"; const SELECT_RUN = "SELECT * FROM workflow_run WHERE id = 1"; @@ -157,14 +169,18 @@ export interface WorkflowLifecycleOptions { /** The directory this host keeps runs in. Absolute, as storage requires. */ readonly root: string; /** - * How this host reads a retained definition's Markdown back, for export. + * How this host turns a retained version-1 definition back into Markdown. * - * Captured in the provider's closure at installation and never reachable - * afterwards. A host that installs none can inspect and control runs and - * cannot export one — which is the honest answer, because an export it could - * perform without this would be sealing source nobody fetched. + * Captured in the provider's closure at installation and reachable through no + * Context, contextual Api, component, Plugin installation result or authored + * value. A host that installs none can inspect and control runs and cannot + * begin, fork or export a Git one — which is the honest answer, because it + * has no way to obtain what such a run executes. + * + * Version 2 never consults it: a source bundle's content is in the run's own + * store, and reaching a repository for it would be a second source of truth. */ - readonly definitionSource?: WorkflowDefinitionSourceReader; + readonly legacySource?: LegacyWorkflowSourceReader; } /** @@ -229,7 +245,7 @@ export function* installWorkflowLifecycle( }, *export([request]) { - return yield* exportRun(root, executors, options.definitionSource, request, observe); + return yield* exportRun(root, executors, options.legacySource, request, observe); }, }, { at: "min" }, @@ -240,13 +256,13 @@ export function* installWorkflowLifecycle( // place for a capability that hands out transports. return { begin(executorLock, request) { - return beginRun(root, connections, executors, executorLock, request); + return beginRun(root, connections, executors, executorLock, request, options.legacySource); }, fork(executorLock, request) { - return forkRun(root, connections, executors, executorLock, request); + return forkRun(root, connections, executors, executorLock, request, options.legacySource); }, stageFork(request) { - return stageForkRun(root, connections, request); + return stageForkRun(root, connections, request, options.legacySource); }, *settle(executorLock, completion) { // Authorized before a connection exists: the path comes from the hold the @@ -273,6 +289,7 @@ function* beginRun( executors: ExecutorLockRegistry, executorLock: ExecutorLock, request: WorkflowBeginRequest, + legacySource: LegacyWorkflowSourceReader | undefined, ): Operation> { const checked = checkRunId(request.runId); if (!checked.ok) { @@ -292,6 +309,7 @@ function* beginRun( hold, () => executors.authorize(executorLock, checked.value), request, + legacySource, ); if (!begun.ok) { return begun; @@ -321,6 +339,7 @@ function* forkRun( executors: ExecutorLockRegistry, executorLock: ExecutorLock, request: WorkflowForkRequest, + legacySource: LegacyWorkflowSourceReader | undefined, ): Operation> { const checked = checkRunId(request.runId); if (!checked.ok) { @@ -351,6 +370,7 @@ function* forkRun( request, snapshot.value, forkHead(hold.runId, request), + legacySource, ); if (forked.ok) { @@ -397,6 +417,7 @@ function* stageForkRun( root: string, connections: WorkflowRunConnections, request: WorkflowForkRequest, + legacySource: LegacyWorkflowSourceReader | undefined, ): Operation> { const checked = checkRunId(request.runId); if (!checked.ok) { @@ -416,21 +437,38 @@ function* stageForkRun( request, snapshot.value, forkHead(checked.value, request), + legacySource, ); } -/** The two records a fork writes for itself, wherever it is being assembled. */ +/** + * The two records a fork writes for itself, wherever it is being assembled. + * + * The run value is the fork's own, in the shape its candidate's version has: a + * source-bundle fork records its bundle hash and exact target, and invents no + * base or pinned commit for a repository it never had. + */ function forkHead(runId: string, request: WorkflowForkRequest): ForkHeadEvents { return { - runRecord: forkRunRecordEvent({ - runId, - base: request.creation.base, - pinnedCommit: request.creation.definition.objectId, - }), + runRecord: forkRunRecordEvent(forkRunValue(runId, request.creation)), rootImport: request.rootImport, }; } +/** The fork's own run value, in the shape its candidate's version declares. */ +function forkRunValue(runId: string, creation: WorkflowRunCreation): WorkflowRun { + if (isGitWorkflowRunCreation(creation)) { + return { runId, base: creation.base, pinnedCommit: creation.definition.objectId }; + } + const { bundleHash, targetPath } = creation.definition; + return { + runId, + definitionVersion: 2, + bundleHash, + ...(targetPath === undefined ? {} : { targetPath }), + }; +} + /** The committed source prefix, read exactly the way inspection reads a run. */ function* readSource( root: string, @@ -597,7 +635,7 @@ function* acquire( function* exportRun( root: string, executors: ExecutorLockRegistry, - readDefinitionSource: WorkflowDefinitionSourceReader | undefined, + legacySource: LegacyWorkflowSourceReader | undefined, request: WorkflowExportRequest, observe: RecoveryObserver, ): Operation> { @@ -605,20 +643,13 @@ function* exportRun( if (!checked.ok) { return checked; } - if (readDefinitionSource === undefined) { - return Err( - new WorkflowRequestError( - "this host installs no way to read a retained definition's source, so it cannot export " + - "a run. An artifact carries the document the run was of, and one sealed without it " + - "would be evidence nobody could continue from.", - ), - ); - } - // The lock covers selection and detachment, and stops there. Reading the - // definition's source means opening a repository, and holding a run + // The lock covers selection and detachment, and stops there. Reading a + // version-1 definition's source means opening a repository, and holding a run // unrunnable for as long as that takes would buy nothing: the frontier is // already values in memory, and no later execution can change what they say. + // A version-2 run's source is read here too, out of its own store, because + // that is where it is. const selected = yield* scoped(function* (): Operation> { const hold = yield* executors.acquire(root, checked.value); if (hold === undefined) { @@ -631,6 +662,9 @@ function* exportRun( detached: readExportFrontier(database, record, path), definition: record.definition, retrieval: readRetrievalMetadata(database), + ...(record.definition.kind === "source-bundle" + ? { rows: readDefinitionSourceRows(database) } + : {}), }), observe, ); @@ -639,17 +673,21 @@ function* exportRun( return selected; } - const closure = yield* readDefinitionSource(selected.value.definition, selected.value.retrieval); + const definition = selected.value.definition; + // Routed by the version the run retains. Version 2 reads only its own source + // store; version 1 crosses the host-supplied legacy seam, and Workflow — not + // the adapter — decides whether what came back describes this definition. + const closure: Result = + definition.kind === "source-bundle" + ? selected.value.rows === undefined + ? Err(new WorkflowDefinitionSourceMissingError()) + : yield* verifyRetainedSources(definition, selected.value.rows) + : legacySource === undefined + ? Err(new LegacyWorkflowSourceReaderUnavailableError()) + : yield* fetchLegacyClosure(definition, selected.value.retrieval, legacySource); if (!closure.ok) { return closure; } - // Asked even though the host fetched it: a reader is host code, and the one - // thing the provider can still check is that what came back describes the - // definition this frontier retains rather than some other run's. - const matched = matchesRetainedDefinition(selected.value.definition, closure.value); - if (!matched.ok) { - return matched; - } const written = yield* writeXmdArtifact(request.stagingPath, { ...selected.value.detached, @@ -666,11 +704,25 @@ function* exportRun( }); } +function* fetchLegacyClosure( + definition: GitWorkflowDefinitionV1, + retrieval: Json | undefined, + legacySource: LegacyWorkflowSourceReader, +): Operation> { + const answered = yield* legacySource(definition, retrieval); + if (!answered.ok) { + return answered; + } + return validateLegacySources(definition, answered.value); +} + /** What one locked selection detached, before any source was fetched. */ interface SelectedFrontier { readonly detached: Omit; readonly definition: WorkflowDefinition; readonly retrieval: Json | undefined; + /** The retained source store, when the run is a source bundle. */ + readonly rows?: RetainedSourceRows; } function* inspectRun( @@ -1169,7 +1221,7 @@ function readRunRow(database: DatabaseSync, path: string): WorkflowRunRecord { if (row === undefined) { throw new WorkflowRequestError(`The workflow-run database at ${path} holds no workflow run.`); } - return readRunRecord(row); + return readRunRecord(row, liveSchemaVersion(database, path)); } function refusal(error: unknown, path: string): Result { diff --git a/packages/workflow/src/deno/provider.ts b/packages/workflow/src/deno/provider.ts index f868c3517..7325090ba 100644 --- a/packages/workflow/src/deno/provider.ts +++ b/packages/workflow/src/deno/provider.ts @@ -37,8 +37,9 @@ import { import { conflictingFields } from "../storage/compatibility.ts"; import { definitionToJson, + type GitWorkflowDefinitionV1, + isGitWorkflowDefinition, parseWorkflowDefinition, - type WorkflowDefinition, } from "../storage/definition.ts"; import { WorkflowRequestError, @@ -165,7 +166,7 @@ export function authorizedRoot(root: string): string { /** A request whose every member has been checked rather than believed. */ interface CheckedRequest { readonly runId: string; - readonly definition: WorkflowDefinition; + readonly definition: GitWorkflowDefinitionV1; readonly base: string; readonly props: JsonObject; } @@ -418,6 +419,20 @@ function checkRequest(offered: CreateWorkflowRunRequest): Result if (!definition.ok) { return definition; } + // Version 1 only, and refused rather than narrowed. This request carries a + // descriptor and no source, so admitting a source bundle would initialize a + // database whose authoritative content nobody ever supplied. Creating a + // version-2 run crosses the trusted lifecycle transition instead. + if (!isGitWorkflowDefinition(definition.value)) { + return Err( + new WorkflowRequestError( + "a source-bundle definition is not created through this request: its exact bytes are " + + "retained with it, and none were supplied here. Start the run through the workflow " + + "lifecycle instead.", + ), + ); + } + const git = definition.value; let props: JsonObject; try { @@ -429,7 +444,7 @@ function checkRequest(offered: CreateWorkflowRunRequest): Result throw error; } - return Ok({ runId: runId.value, definition: definition.value, base, props }); + return Ok({ runId: runId.value, definition: git, base, props }); } function requestFailure(reason: string, path: string): Error { diff --git a/packages/workflow/src/deno/rows.ts b/packages/workflow/src/deno/rows.ts index 30951826c..0245a35c8 100644 --- a/packages/workflow/src/deno/rows.ts +++ b/packages/workflow/src/deno/rows.ts @@ -19,13 +19,16 @@ import { describe, type Fail, type JsonObject, parseJsonValue } from "../storage import { type DefinitionRetrieval, type DocumentExecutionRecord, + type GitWorkflowRunRecordV1, parseRunId, parseWorkflowRunStatus, parseWorkflowStopReason, + type SourceBundleWorkflowRunRecordV2, type WorkflowRunRecord, type WorkflowRunStatus, type WorkflowStopReason, } from "../storage/record.ts"; +import type { WorkflowSchemaVersion } from "./schema.ts"; /** One row as SQLite hands it back. */ export type Row = Record; @@ -201,19 +204,58 @@ function runId(row: Row, table: string): string { return parseRunId(row["run_id"], "$", failure(`${table}.run_id`)); } -/** The run the singleton row describes. */ -export function readRunRecord(row: Row): WorkflowRunRecord { +/** + * The run the singleton row describes, read as the version it was recognized + * at. + * + * The version is the one structural recognition just proved rather than one + * inferred from the descriptor the row happens to hold: a version-1 table + * carrying a source-bundle descriptor is a file disagreeing with itself, and + * reading it as a version-2 record would quietly drop the `base` column that + * table still has. + */ +export function readRunRecord(row: Row, version: WorkflowSchemaVersion = 1): WorkflowRunRecord { const table = "workflow_run"; - const record: WorkflowRunRecord = { + const definition = readDefinition(row, table); + const shared = { runId: runId(row, table), - definition: readDefinition(row, table), - base: nonEmpty(row, "base", table), props: jsonObject(row, "props", table), status: readStatus(row, "status", table), createdAt: instant(row, "created_at", table), updatedAt: instant(row, "updated_at", table), }; const stopReason = readStopReason(row, table); + + if (version === 2) { + if (definition.kind !== "source-bundle") { + throw new WorkflowRecordMalformedError( + `${table}.definition`, + "expected a source-bundle definition", + ); + } + const record: SourceBundleWorkflowRunRecordV2 = { + runId: shared.runId, + definition, + props: shared.props, + status: shared.status, + createdAt: shared.createdAt, + updatedAt: shared.updatedAt, + }; + return Object.freeze(stopReason === undefined ? record : { ...record, stopReason }); + } + + if (definition.kind !== "git") { + throw new WorkflowRecordMalformedError(`${table}.definition`, "expected a Git definition"); + } + const record: GitWorkflowRunRecordV1 = { + runId: shared.runId, + definition, + base: nonEmpty(row, "base", table), + props: shared.props, + status: shared.status, + createdAt: shared.createdAt, + updatedAt: shared.updatedAt, + }; return Object.freeze(stopReason === undefined ? record : { ...record, stopReason }); } diff --git a/packages/workflow/src/deno/run-host.ts b/packages/workflow/src/deno/run-host.ts index c74ecfcf6..e1b503fbb 100644 --- a/packages/workflow/src/deno/run-host.ts +++ b/packages/workflow/src/deno/run-host.ts @@ -10,15 +10,34 @@ import type { Operation } from "effection"; import type { WorkflowExecutionTransitions } from "../lifecycle/execution.ts"; +import type { LegacyWorkflowSourceReader } from "../lifecycle/source.ts"; import { useWorkflowRunConnections } from "./connections.ts"; import { installWorkflowLifecycle } from "./lifecycle.ts"; import { installWorkflowRunStorage, type WorkflowRunStorageOptions } from "./provider.ts"; import { SavepointObservation } from "./savepoints.ts"; +/** What one host installs, and the version-1 source capability it may carry. */ +export interface WorkflowRunHostOptions extends WorkflowRunStorageOptions { + /** + * How this host turns a retained version-1 definition back into Markdown. + * + * Captured in the lifecycle provider's closure, never installed into a scope + * and never reachable by name. A host that supplies none can inspect and + * control runs and cannot begin, fork or export a Git one. + */ + readonly legacySource?: LegacyWorkflowSourceReader; +} + export function* useWorkflowRunHost( - options: WorkflowRunStorageOptions, + options: WorkflowRunHostOptions, ): Operation { const connections = yield* useWorkflowRunConnections(yield* SavepointObservation.get()); - yield* installWorkflowRunStorage(options, {}, connections); - return yield* installWorkflowLifecycle({ root: options.root }, connections); + yield* installWorkflowRunStorage({ root: options.root }, {}, connections); + return yield* installWorkflowLifecycle( + { + root: options.root, + ...(options.legacySource === undefined ? {} : { legacySource: options.legacySource }), + }, + connections, + ); } diff --git a/packages/workflow/src/deno/schema.ts b/packages/workflow/src/deno/schema.ts index f37558810..ec6f127be 100644 --- a/packages/workflow/src/deno/schema.ts +++ b/packages/workflow/src/deno/schema.ts @@ -44,9 +44,21 @@ import { initializeEmptyWorkspace, verifyWorkspace } from "./workspace/root.ts"; */ export const APPLICATION_ID = 0x584d4431; -/** The only schema version this build reads or writes. */ +/** + * The version a Git-definition run is written at, and reads back as. + * + * Still the value it always was. A version-2 database is a different inventory + * rather than a later amendment of this one, so redefining this constant to + * mean "latest" would silently move every version-1 assertion that names it. + */ export const SCHEMA_VERSION = 1; +/** The version a source-bundle run is written at, and reads back as. */ +export const SOURCE_BUNDLE_SCHEMA_VERSION = 2; + +/** Every live schema version this build recognizes. */ +export type WorkflowSchemaVersion = 1 | 2; + const STATUSES = "'running', 'suspended', 'interrupted', 'completed', 'failed', 'cancelled'"; /** @@ -499,12 +511,115 @@ const OBJECTS: ReadonlyMap = new Map([ ], ]); +/** + * The run table a source-bundle database holds instead of version 1's. + * + * No `base` column at all. A version-2 run started from exact bytes rather than + * from a repository state, so a column for one would be a value every row had + * to invent — and SQLite is where that invariant is held rather than in the + * code that writes rows. The `definition` CHECK pins the descriptor's own + * version and kind for the same reason. + */ +const SOURCE_BUNDLE_RUN: DeclaredObject = { + type: "table", + sql: `CREATE TABLE workflow_run ( + id INTEGER PRIMARY KEY CHECK (id = 1), + run_id TEXT NOT NULL, + definition TEXT NOT NULL CHECK ( + json_valid(definition) + AND json_extract(definition, '$.version') = 2 + AND json_extract(definition, '$.kind') = 'source-bundle' + ), + props TEXT NOT NULL CHECK (json_valid(props) AND json_type(props) = 'object'), + status TEXT NOT NULL CHECK (status IN (${STATUSES})), + stop_reason_kind TEXT CHECK (stop_reason_kind IS NULL OR stop_reason_kind IN ('host', 'journal')), + stop_reason_code TEXT, + stop_reason_event_id TEXT REFERENCES journal_events (event_id), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + ${coherentStopReason()} +) STRICT`, +}; + +/** + * The two tables a source-bundle database adds, and nothing else. + * + * Content is keyed by its own hash, so two logical paths holding identical + * bytes reference one blob. The manifest is the descriptor's `sources` array, + * one row per entry: the parser rather than SQLite collation enforces the + * logical-path grammar and the canonical order, because neither is something a + * collation can state. + */ +/** + * The two tables a source-bundle database adds, and nothing else. + * + * Content is keyed by its own hash, so two logical paths holding identical + * bytes reference one blob. The manifest is the descriptor's `sources` array, + * one row per entry: the parser rather than SQLite collation enforces the + * logical-path grammar and the canonical order, because neither is something a + * collation can state. + */ +const SOURCE_BUNDLE_BLOB: DeclaredObject = { + type: "table", + sql: `CREATE TABLE workflow_definition_blob ( + source_hash TEXT PRIMARY KEY CHECK ( + length(source_hash) = 64 + AND source_hash NOT GLOB '*[^0-9a-f]*' + ), + byte_length INTEGER NOT NULL CHECK ( + byte_length >= 0 AND byte_length <= 9007199254740991 + ), + content BLOB NOT NULL CHECK (length(content) = byte_length) +) STRICT, WITHOUT ROWID`, +}; + +const SOURCE_BUNDLE_MANIFEST: DeclaredObject = { + type: "table", + sql: `CREATE TABLE workflow_definition_source ( + path TEXT PRIMARY KEY CHECK (length(path) > 0), + source_hash TEXT NOT NULL REFERENCES workflow_definition_blob(source_hash) ON DELETE RESTRICT +) STRICT, WITHOUT ROWID`, +}; + +const SOURCE_BUNDLE_OBJECTS: ReadonlyMap = new Map([ + ["workflow_definition_blob", SOURCE_BUNDLE_BLOB], + ["workflow_definition_source", SOURCE_BUNDLE_MANIFEST], +]); + +/** + * Version 2: every version-1 object byte for byte, one replacement, two + * additions. + * + * Derived from the version-1 map rather than restated, so the two inventories + * cannot drift apart in the objects they are supposed to share — and so a + * version-1 amendment is not something that has to be applied twice. + */ +const OBJECTS_V2: ReadonlyMap = new Map([ + ...[...OBJECTS.entries()].map(([name, object]): [string, DeclaredObject] => [ + name, + name === "workflow_run" ? SOURCE_BUNDLE_RUN : object, + ]), + ...SOURCE_BUNDLE_OBJECTS.entries(), +]); + +/** The declared inventory of one live schema version. */ +function objectsFor(version: WorkflowSchemaVersion): ReadonlyMap { + return version === 2 ? OBJECTS_V2 : OBJECTS; +} + export const EXPECTED_SCHEMA = Object.freeze( [...OBJECTS.entries()].map(([name, object]) => Object.freeze({ name, type: object.type, sql: normalize(object.sql) }), ), ); +/** The same declarations, for the source-bundle inventory. */ +export const SOURCE_BUNDLE_EXPECTED_SCHEMA = Object.freeze( + [...OBJECTS_V2.entries()].map(([name, object]) => + Object.freeze({ name, type: object.type, sql: normalize(object.sql) }), + ), +); + /** * Objects a version-1 file must hold, including the pinned Cloudflare * structure. @@ -556,31 +671,62 @@ export function declaredObjectSql(name: string): string { return declared.sql; } +function schemaSql(version: WorkflowSchemaVersion): string { + return [...objectsFor(version).values()] + .filter((object) => object.type === "table" && !object.sql.startsWith("CREATE TABLE vfs_")) + .filter((object) => !object.sql.startsWith("CREATE TABLE _vfs_")) + .map((object) => `${object.sql};`) + .join("\n\n"); +} + /** Version 1 in full. */ -export const SCHEMA_SQL = [...OBJECTS.values()] - .filter((object) => object.type === "table" && !object.sql.startsWith("CREATE TABLE vfs_")) - .filter((object) => !object.sql.startsWith("CREATE TABLE _vfs_")) - .map((object) => `${object.sql};`) - .join("\n\n"); +export const SCHEMA_SQL = schemaSql(SCHEMA_VERSION); + +/** Version 2 in full. */ +export const SOURCE_BUNDLE_SCHEMA_SQL = schemaSql(SOURCE_BUNDLE_SCHEMA_VERSION); /** - * Write the version-1 schema into a database that holds nothing. + * Write one live schema into a database that holds nothing. * * Called inside the caller's transaction, so the application id, the version * and the tables appear together or not at all — a half-initialized file would * be indistinguishable from one this build must refuse. + * + * The version comes from the candidate definition that is about to be written, + * never from what a file already claims: this initializes an empty database and + * nothing here ever upgrades one. */ export function initializeSchema( database: DatabaseSync, dofs: CloudflareDatabase, initializeRun: () => void, + version: WorkflowSchemaVersion = SCHEMA_VERSION, ): void { database.exec(`PRAGMA application_id = ${APPLICATION_ID};`); - database.exec(SCHEMA_SQL); + database.exec(version === 2 ? SOURCE_BUNDLE_SCHEMA_SQL : SCHEMA_SQL); initializeCloudflareSchema(dofs, () => 0); initializeEmptyWorkspace(database); initializeRun(); - database.exec(`PRAGMA user_version = ${SCHEMA_VERSION};`); + database.exec(`PRAGMA user_version = ${version};`); +} + +/** + * The version a recognized database declares. + * + * Read from the pragma rather than inferred from the descriptor a row holds: + * the pragma is what structural recognition just held the whole inventory to, + * and a row parsed under a version its own file does not declare would be read + * with columns that file does not have. + */ +export function liveSchemaVersion(database: DatabaseSync, path: string): WorkflowSchemaVersion { + const version = readPragmaNumber(database, "user_version", path); + if (version === SOURCE_BUNDLE_SCHEMA_VERSION) { + return 2; + } + if (version === SCHEMA_VERSION) { + return 1; + } + throw new WorkflowSchemaVersionError(path, version, SOURCE_BUNDLE_SCHEMA_VERSION); } /** @@ -600,12 +746,33 @@ export function isUninitialized(database: DatabaseSync, path: string): boolean { } /** - * Refuse anything that is not a version-1 workflow-run database. + * Refuse anything that is not a workflow-run database this build recognizes. * * Structure only. Whether the rows describe the run that was asked for is a * separate question, asked after this one succeeds. + * + * The application id is read first, and only then the version — and the version + * selects one of exactly two immutable inventories. Nothing here repairs, + * migrates or reinterprets: a file whose objects are a mixture of the two, or + * that carries an extra one, disagrees with the version it declares and is left + * as it was found. */ export function verifySchema(database: DatabaseSync, path: string, dofs: CloudflareDatabase): void { + verifyRecognizedSchema(database, path, dofs); +} + +/** + * The same recognition, answering which version it recognized. + * + * A reader that has to parse a row differently per version asks this, so the + * version a record is read under is the one structural recognition just proved + * rather than one read again afterwards. + */ +export function verifyRecognizedSchema( + database: DatabaseSync, + path: string, + dofs: CloudflareDatabase, +): WorkflowSchemaVersion { checkIntegrity(database, path); const applicationId = readPragmaNumber(database, "application_id", path); @@ -623,52 +790,69 @@ export function verifySchema(database: DatabaseSync, path: string, dofs: Cloudfl } const version = readPragmaNumber(database, "user_version", path); + // Version 0 under either declared inventory is a partial initialization: the + // application identity is there and the version that completes it never was. if (version === 0) { throw new WorkflowDatabaseCorruptError( path, - "it carries the XMD application identity without a complete version-1 schema", + "it carries the XMD application identity without a complete schema version", ); } - if (version !== SCHEMA_VERSION) { - throw new WorkflowSchemaVersionError(path, version, SCHEMA_VERSION); + if (version !== SCHEMA_VERSION && version !== SOURCE_BUNDLE_SCHEMA_VERSION) { + throw new WorkflowSchemaVersionError(path, version, SOURCE_BUNDLE_SCHEMA_VERSION); } + const recognized: WorkflowSchemaVersion = version === 2 ? 2 : 1; - verifyStructure(database, path); + verifyStructure(database, path, recognized); checkForeignKeys(database, path); verifyWorkspace(database, dofs, path); + return recognized; } /** - * Hold a recognized database to the schema this build writes. + * Hold a recognized database to the schema this build writes for its version. * - * The header already claims version 1, so anything missing or differently + * The header already claims a version, so anything missing or differently * shaped is the file disagreeing with itself rather than a version this build - * has not learned yet. + * has not learned yet. A version-1 object inside a version-2 file — the + * `workflow_run` table with a `base` column, say — fails here as a shape that + * is not what this version declares, which is what keeps the two inventories + * from being read as one superset. */ -function verifyStructure(database: DatabaseSync, path: string): void { +function verifyStructure( + database: DatabaseSync, + path: string, + version: WorkflowSchemaVersion, +): void { const objects = schemaObjects(database, path); - if (isIncompletePreReleaseShape(objects)) { + // Only version 1 has pre-release shapes: nothing ever shipped claiming to be + // an incomplete version 2. + if (version === 1 && isIncompletePreReleaseShape(objects)) { throw new WorkflowIncompleteVersionOneError(path); } + const declared = objectsFor(version); for (const object of objects) { - const expected = OBJECTS.get(object.name); + const expected = declared.get(object.name); if (expected === undefined) { throw new WorkflowDatabaseCorruptError( path, - `it declares an object that version ${SCHEMA_VERSION} does not`, + `it declares an object that version ${version} does not`, ); } if (object.type !== expected.type || normalize(object.sql) !== normalize(expected.sql)) { throw new WorkflowDatabaseCorruptError( path, - `its ${object.name} object is not shaped the way version ${SCHEMA_VERSION} declares it`, + `its ${object.name} object is not shaped the way version ${version} declares it`, ); } } const present = new Set(objects.map((object) => object.name)); - const missing = REQUIRED_OBJECTS.filter((name) => !present.has(name)); + const missing = [...declared.entries()] + .filter(([, object]) => object.optional !== true) + .map(([name]) => name) + .filter((name) => !present.has(name)); if (missing.length > 0) { throw new WorkflowDatabaseCorruptError(path, `it is missing the table ${missing.join(", ")}`); } diff --git a/packages/workflow/src/deno/transitions.ts b/packages/workflow/src/deno/transitions.ts index 1c365834d..64a4e4496 100644 --- a/packages/workflow/src/deno/transitions.ts +++ b/packages/workflow/src/deno/transitions.ts @@ -27,17 +27,21 @@ import { randomUUID } from "node:crypto"; import type { DatabaseSync } from "node:sqlite"; import { ensure, Err, Ok, type Operation, resource, type Result, scoped } from "effection"; import { exists, rm } from "@effectionx/fs"; -import type { DurableEvent } from "@executablemd/durable-streams"; -import type { - WorkflowBeginRequest, - WorkflowExecutionBegun, - WorkflowForkRequest, - WorkflowRunCreation, +import type { Json } from "@executablemd/durable-streams"; +import { + isGitWorkflowRunCreation, + type WorkflowBeginRequest, + type WorkflowExecutionBegun, + type WorkflowForkRequest, + type WorkflowRunCreation, } from "../lifecycle/execution.ts"; +import type { LegacyWorkflowSourceReader, RetainedDefinitionSources } from "../lifecycle/source.ts"; import type { WorkflowRunDatabase } from "../storage/api.ts"; -import { conflictingFields } from "../storage/compatibility.ts"; -import { definitionToJson } from "../storage/definition.ts"; +import { conflictingFields, type WorkflowRunComparison } from "../storage/compatibility.ts"; +import { definitionToJson, type GitWorkflowDefinitionV1 } from "../storage/definition.ts"; import { + LegacyWorkflowSourceReaderUnavailableError, + WorkflowDefinitionSourceMissingError, WorkflowDocumentExecutionError, WorkflowRequestError, WorkflowRunConflictError, @@ -45,6 +49,10 @@ import { WorkflowRunNotFoundError, WorkflowStorageError, } from "../storage/errors.ts"; +import { + type SourceBundleSnapshotEntryV2, + verifySourceBundleSnapshot, +} from "../storage/source-bundle.ts"; import { canonicalJson, type DocumentExecutionCompletion, @@ -56,8 +64,16 @@ import { import type { RunConnection, RunTransaction, WorkflowRunConnections } from "./connections.ts"; import { openWorkflowRunDatabase, readRunRow } from "./database.ts"; import type { ExecutorLockHold } from "./executor.ts"; -import { reading } from "./reading.ts"; +import { reading, readTransaction } from "./reading.ts"; import { readJournalEntries } from "./journal.ts"; +import { readRetrievalMetadata } from "./artifact-frontier.ts"; +import { + readDefinitionSourceRows, + type RetainedSourceRows, + validateLegacySources, + verifyRetainedSources, + writeDefinitionSources, +} from "./definition-source.ts"; import type { ForkSourceSnapshot } from "./fork-source.ts"; import { readForkLineage, writeForkInheritance, type ForkHeadEvents } from "./fork-write.ts"; import { readDocumentExecution, readRetrieval, stopReasonColumns } from "./rows.ts"; @@ -65,7 +81,9 @@ import { initializeSchema, isSqliteForeignKeyConstraint, isUninitialized, + SOURCE_BUNDLE_SCHEMA_VERSION, translateSqliteError, + verifyRecognizedSchema, verifySchema, } from "./schema.ts"; @@ -73,6 +91,10 @@ import { export const INSERT_RUN = `INSERT INTO workflow_run (id, run_id, definition, base, props, status, created_at, updated_at) VALUES (1, ?, ?, ?, ?, 'running', ?, ?)`; +/** The same row in a version-2 database, which has no base column at all. */ +const INSERT_SOURCE_BUNDLE_RUN = `INSERT INTO workflow_run + (id, run_id, definition, props, status, created_at, updated_at) + VALUES (1, ?, ?, ?, 'running', ?, ?)`; const UPDATE_RUN_STATE = `UPDATE workflow_run SET status = ?, stop_reason_kind = ?, stop_reason_code = ?, stop_reason_event_id = ?, updated_at = ? @@ -106,6 +128,7 @@ export function* beginExecution( hold: ExecutorLockHold, authorize: () => ExecutorLockHold, request: WorkflowBeginRequest, + readLegacySource?: LegacyWorkflowSourceReader, ): Operation> { // Asked before a connection exists, because opening one creates the file. // A resume that found nothing would otherwise leave an empty database behind @@ -120,6 +143,16 @@ export function* beginExecution( // in between. const connection = yield* connections.at(path); + // Everything the source has to prove, proved first. This caller holds the + // executor lock, and nothing here writes — so a run whose retained bytes are + // missing, damaged or unobtainable is refused with its lifecycle and its + // journal exactly as they were, rather than after a recovery it then has to + // leave behind. + const authenticated = yield* authenticateBeforeBegin(connection, path, request, readLegacySource); + if (!authenticated.ok) { + return authenticated; + } + // One transaction. Recovery decides what the previous workflow executor's execution // became, admission decides whether this caller may continue, and an admitted // caller's execution is inserted — all or none. Splitting them would publish @@ -128,7 +161,7 @@ export function* beginExecution( const outcome = yield* scoped(function* (): Operation> { yield* connection.lock.hold(); return inLifecycleTransaction(connection, path, () => - beginOnce(connection, path, sameHold(authorize, hold), request), + beginOnce(connection, path, sameHold(authorize, hold), request, authenticated.value.owned), ); }); if (!outcome.ok) { @@ -141,6 +174,20 @@ export function* beginExecution( } const { record, execution, replay, closed } = outcome.value; + // A run this call created has no proved source yet: what it will execute is + // read back out of the transaction that committed it, so the closure a caller + // imports is the store's own bytes rather than the buffers it handed in. + const sources = yield* settleSources( + connection, + path, + record, + authenticated.value.proved, + readLegacySource, + ); + if (!sources.ok) { + return sources; + } + const database = yield* openWorkflowRunDatabase({ connection, connections, record }); return Ok({ kind: "begun", @@ -148,10 +195,206 @@ export function* beginExecution( record, execution, replay, + sources: sources.value, ...(closed === undefined ? {} : { closed }), }); } +/** What a begin proved about the run's source before it wrote anything. */ +interface AuthenticatedSource { + /** + * The source proved before anything moved, when it could be proved yet. + * + * Absent for exactly one case: a source-bundle run being created, whose + * content does not exist in any store until this call commits it. Every other + * path — an existing run of either version, and a Git run being created from + * a descriptor the reader can already be asked about — proves it here. + */ + readonly proved?: RetainedDefinitionSources; + /** The creation's snapshot, copied and checked against its descriptor. */ + readonly owned?: readonly SourceBundleSnapshotEntryV2[]; +} + +/** + * Prove the source before the lifecycle transaction opens. + * + * Two independent questions, both answered here so neither can be answered + * after a write. A version-2 creation's snapshot is copied and held to its own + * descriptor, so a caller that keeps mutating its arrays changes nothing and a + * snapshot that does not describe the descriptor never reaches storage. An + * existing run's retained source is re-derived from what it holds, or — for a + * version-1 run — fetched through the host's legacy reader and validated + * against the descriptor the run retains. + */ +function* authenticateBeforeBegin( + connection: RunConnection, + path: string, + request: WorkflowBeginRequest, + readLegacySource: LegacyWorkflowSourceReader | undefined, +): Operation> { + const { creation } = request; + let owned: readonly SourceBundleSnapshotEntryV2[] | undefined; + if (creation !== undefined && !isGitWorkflowRunCreation(creation)) { + const verified = yield* verifySourceBundleSnapshot( + creation.definition, + creation.sourceSnapshot, + ); + if (!verified.ok) { + return verified; + } + owned = verified.value; + } + + const stored = yield* readStoredSource(connection, path); + if (!stored.ok) { + return stored; + } + + // A run that is already there is held to what it retains. A Git run that is + // being created is held to the descriptor it is about to retain: the reader + // is asked now, with the creation's own descriptor and retrieval metadata, so + // a host that cannot obtain the source never reaches the transaction that + // would make the run exist. + if (stored.value === undefined) { + if (creation === undefined || !isGitWorkflowRunCreation(creation)) { + return Ok(owned === undefined ? {} : { owned }); + } + const proved = yield* readGitSource(creation.definition, creation.retrieval, readLegacySource); + if (!proved.ok) { + return proved; + } + return Ok({ proved: proved.value, ...(owned === undefined ? {} : { owned }) }); + } + + const proved = yield* authenticate(stored.value, readLegacySource); + if (!proved.ok) { + return proved; + } + return Ok({ proved: proved.value, ...(owned === undefined ? {} : { owned }) }); +} + +/** A run that is already there, and whatever its own store holds for it. */ +interface StoredSource { + readonly record: WorkflowRunRecord; + readonly retrieval: Json | undefined; + readonly rows?: RetainedSourceRows; +} + +/** + * Read what the store holds, deciding nothing about it. + * + * Read-only and inside its own transaction, because the questions that follow + * are operations and a lifecycle transaction body may not suspend. + */ +function* readStoredSource( + connection: RunConnection, + path: string, +): Operation> { + return yield* scoped(function* (): Operation> { + yield* connection.lock.hold(); + try { + return Ok( + readTransaction(connection.database, () => { + if (isUninitialized(connection.database, path)) { + return undefined; + } + const version = verifyRecognizedSchema(connection.database, path, connection.dofs); + const record = readRunRow(connection.database, path); + const retrieval = readRetrievalMetadata(connection.database); + if (version === 2) { + return { record, retrieval, rows: readDefinitionSourceRows(connection.database) }; + } + return { record, retrieval }; + }), + ); + } catch (error) { + return refusal(error, path); + } + }); +} + +/** + * The source a stored run is a run of, re-derived or fetched and validated. + * + * A source bundle's content is in this database, so it is always re-derived and + * a store that cannot produce it refuses here — before anything moves. + * + * A Git run's content is in a repository this package does not reach, so it is + * fetched through the host's reader and held to the descriptor at the same + * moment and for the same reason. A host that installed no reader cannot obtain + * version-1 Markdown at all, and that is a refusal here rather than a run + * admitted without knowing what it executes: every version-1 lifecycle + * admission is gated before it writes. + */ +function* authenticate( + stored: StoredSource, + readLegacySource: LegacyWorkflowSourceReader | undefined, +): Operation> { + const { definition } = stored.record; + if (definition.kind === "source-bundle") { + if (stored.rows === undefined) { + return Err(new WorkflowDefinitionSourceMissingError()); + } + return yield* verifyRetainedSources(definition, stored.rows); + } + return yield* readGitSource(definition, stored.retrieval, readLegacySource); +} + +/** + * A Git definition's Markdown, fetched through the host and held to it. + * + * The reader is a direct dependency the host captured, and a host that captured + * none cannot obtain version-1 source at all — so that is a refusal here rather + * than a run begun without knowing what it executes. Whatever comes back is + * validated against this descriptor before it counts as this run's. + */ +function* readGitSource( + definition: GitWorkflowDefinitionV1, + retrieval: Json | undefined, + readLegacySource: LegacyWorkflowSourceReader | undefined, +): Operation> { + if (readLegacySource === undefined) { + return Err(new LegacyWorkflowSourceReaderUnavailableError()); + } + const answered = yield* readLegacySource(definition, retrieval); + if (!answered.ok) { + return answered; + } + return validateLegacySources(definition, answered.value); +} + +/** + * The source the begun execution runs, read back from what is now committed. + * + * An existing run already proved its own before anything moved, and that is the + * value returned. A run this call created has one only now, so it is read out + * of storage and verified again — which is what makes the closure a caller + * imports the store's bytes rather than the buffers it supplied. + */ +function* settleSources( + connection: RunConnection, + path: string, + record: WorkflowRunRecord, + proved: RetainedDefinitionSources | undefined, + readLegacySource: LegacyWorkflowSourceReader | undefined, +): Operation> { + if (proved !== undefined) { + return Ok(proved); + } + // The one case left: a source-bundle run this call created. Its content + // existed nowhere until the transaction committed, so it is read back out of + // the store and verified again — which is what makes the closure a caller + // imports the retained bytes rather than the buffers it supplied. + const stored = yield* readStoredSource(connection, path); + if (!stored.ok) { + return stored; + } + if (stored.value === undefined) { + return Err(new WorkflowRunNotFoundError(record.runId)); + } + return yield* authenticate(stored.value, readLegacySource); +} + /** * What one begin transaction committed. * @@ -207,12 +450,7 @@ function recover( throw new WorkflowRunIdMismatchError(hold.runId, path); } if (request.creation !== undefined) { - const differing = conflictingFields(stored, { - runId: hold.runId, - definition: request.creation.definition, - base: request.creation.base, - props: request.creation.props, - }); + const differing = conflictingFields(stored, creationComparison(hold.runId, request.creation)); if (differing.length > 0) { throw new WorkflowRunConflictError(hold.runId, differing); } @@ -228,6 +466,7 @@ function beginOnce( path: string, hold: ExecutorLockHold, request: WorkflowBeginRequest, + owned: readonly SourceBundleSnapshotEntryV2[] | undefined, ): BegunRows | Refused { // An acquisition begins one execution. A second would find this workflow executor's own // live execution and, seeing it unfinished, reconcile it as a dead executor's @@ -260,7 +499,7 @@ function beginOnce( } const { database } = connection; - const begun = begin(connection, path, hold, request, recovery); + const begun = begin(connection, path, hold, request, recovery, owned); hold.execution = begun.execution.executionId; return { ...begun, @@ -292,13 +531,35 @@ export function* forkExecution( request: WorkflowForkRequest, snapshot: ForkSourceSnapshot, head: ForkHeadEvents, + readLegacySource?: LegacyWorkflowSourceReader, ): Operation> { const connection = yield* connections.at(path); + // The fork's own candidate, proved before its destination exists. A snapshot + // that does not describe its descriptor leaves nothing behind at all. + const authenticated = yield* authenticateBeforeBegin( + connection, + path, + { runId: request.runId, action: "start", creation: request.creation }, + readLegacySource, + ); + if (!authenticated.ok) { + return authenticated; + } + const outcome = yield* scoped(function* (): Operation> { yield* connection.lock.hold(); return inLifecycleTransaction(connection, path, (transaction) => - forkOnce(connection, path, sameHold(authorize, hold), request, snapshot, head, transaction), + forkOnce( + connection, + path, + sameHold(authorize, hold), + request, + snapshot, + head, + transaction, + authenticated.value.owned, + ), ); }); if (!outcome.ok) { @@ -311,6 +572,16 @@ export function* forkExecution( } const { record, execution, replay, closed } = outcome.value; + const sources = yield* settleSources( + connection, + path, + record, + authenticated.value.proved, + readLegacySource, + ); + if (!sources.ok) { + return sources; + } const database = yield* openWorkflowRunDatabase({ connection, connections, record }); return Ok({ kind: "begun", @@ -318,6 +589,7 @@ export function* forkExecution( record, execution, replay, + sources: sources.value, ...(closed === undefined ? {} : { closed }), }); } @@ -343,6 +615,7 @@ export function stageFork( request: WorkflowForkRequest, snapshot: ForkSourceSnapshot, head: ForkHeadEvents, + readLegacySource?: LegacyWorkflowSourceReader, ): Operation> { return resource(function* (provide) { yield* ensure(function* () { @@ -353,11 +626,40 @@ export function stageFork( // left by an attempt that did not finish, and it describes nothing. yield* rm(path, { force: true }); + // The staged fork retains the same source its admitted twin will, so the + // same snapshot is copied and checked here. A replay against buffers a + // caller still holds would prove compatibility of something else. + let owned: readonly SourceBundleSnapshotEntryV2[] | undefined; + if (isGitWorkflowRunCreation(request.creation)) { + // A staged fork executes the same candidate its admitted twin will, so + // the same reader gate applies before anything is assembled: a host that + // cannot obtain this definition's Markdown cannot replay it either. + const proved = yield* readGitSource( + request.creation.definition, + request.creation.retrieval, + readLegacySource, + ); + if (!proved.ok) { + yield* provide(proved); + return; + } + } else { + const verified = yield* verifySourceBundleSnapshot( + request.creation.definition, + request.creation.sourceSnapshot, + ); + if (!verified.ok) { + yield* provide(verified); + return; + } + owned = verified.value; + } + const connection = yield* connections.at(path); const built = yield* scoped(function* (): Operation> { yield* connection.lock.hold(); return inLifecycleTransaction(connection, path, (transaction) => { - const record = createRun(connection, path, request.runId, request.creation); + const record = createRun(connection, path, request.runId, request.creation, owned); writeForkInheritance(connection, transaction, snapshot, head); void record; return readRunRow(connection.database, path); @@ -384,6 +686,7 @@ function forkOnce( snapshot: ForkSourceSnapshot, head: ForkHeadEvents, transaction: RunTransaction, + owned: readonly SourceBundleSnapshotEntryV2[] | undefined, ): BegunRows | Refused { if (hold.execution !== undefined) { throw new WorkflowRequestError( @@ -411,17 +714,19 @@ function forkOnce( if (differing.length > 0) { throw new WorkflowRunConflictError(hold.runId, differing); } - return beginOnce(connection, path, hold, resumed); + return beginOnce(connection, path, hold, resumed, owned); } - const record = create(connection, path, hold, { - runId: hold.runId, - action: "start", - creation: request.creation, - }); + const record = create( + connection, + path, + hold, + { runId: hold.runId, action: "start", creation: request.creation }, + owned, + ); void record; writeForkInheritance(connection, transaction, snapshot, head); - const execution = insertExecution(database, path); + const execution = insertExecution(database); hold.execution = execution.executionId; return { kind: "begun", record: readRunRow(database, path), execution, replay: false }; } @@ -452,14 +757,7 @@ function forkConflicts( fields.push("checkpoint"); } } - fields.push( - ...conflictingFields(stored, { - runId, - definition: request.creation.definition, - base: request.creation.base, - props: request.creation.props, - }), - ); + fields.push(...conflictingFields(stored, creationComparison(runId, request.creation))); return fields; } @@ -534,6 +832,7 @@ function begin( hold: ExecutorLockHold, request: WorkflowBeginRequest, recovered: Recovery, + owned: readonly SourceBundleSnapshotEntryV2[] | undefined, ): BegunRows { const { database } = connection; @@ -542,12 +841,12 @@ function begin( // lock, so nothing has appeared since. return { kind: "begun", - record: create(connection, path, hold, request), - ...firstExecution(database, path), + record: create(connection, path, hold, request, owned), + ...firstExecution(database), }; } - const started = insertExecution(database, path); + const started = insertExecution(database); if (terminal(recovered.status)) { // A replay observes an outcome that already won. Publishing `running` would // make a settled run mutable again. @@ -562,46 +861,104 @@ function create( path: string, hold: ExecutorLockHold, request: WorkflowBeginRequest, + owned: readonly SourceBundleSnapshotEntryV2[] | undefined, ): WorkflowRunRecord { const { creation } = request; if (creation === undefined) { throw new WorkflowRunNotFoundError(hold.runId); } - return createRun(connection, path, hold.runId, creation); + return createRun(connection, path, hold.runId, creation, owned); } -/** The schema, the immutable run and its retrieval metadata, in one write. */ +/** + * The schema, the immutable run, its source and its retrieval metadata, in one + * write. + * + * The schema version comes from the creation variant, so a Git run initializes + * version 1 and a source-bundle run version 2. A version-2 creation writes its + * complete manifest and de-duplicated content inside the same initialization + * callback as the run row: descriptor and content are one fact, and a database + * holding one without the other is a run whose authoritative source nobody + * supplied. + * + * `owned` is the snapshot after it was copied and checked against the + * descriptor, never the caller's own arrays. + */ function createRun( connection: RunConnection, path: string, runId: string, creation: WorkflowRunCreation, + owned: readonly SourceBundleSnapshotEntryV2[] | undefined, ): WorkflowRunRecord { const { database } = connection; const stamp = new Date().toISOString(); - initializeSchema(database, connection.dofs, () => { - database - .prepare(INSERT_RUN) - .run( - runId, - canonicalJson(definitionToJson(creation.definition)), - creation.base, - canonicalJson(creation.props), - stamp, - stamp, + + if (isGitWorkflowRunCreation(creation)) { + initializeSchema(database, connection.dofs, () => { + database + .prepare(INSERT_RUN) + .run( + runId, + canonicalJson(definitionToJson(creation.definition)), + creation.base, + canonicalJson(creation.props), + stamp, + stamp, + ); + }); + } else { + if (owned === undefined) { + throw new WorkflowRequestError( + "a source-bundle run is created from its exact bytes, and none were verified for it.", ); - }); + } + initializeSchema( + database, + connection.dofs, + () => { + database + .prepare(INSERT_SOURCE_BUNDLE_RUN) + .run( + runId, + canonicalJson(definitionToJson(creation.definition)), + canonicalJson(creation.props), + stamp, + stamp, + ); + writeDefinitionSources(database, creation.definition, owned); + }, + SOURCE_BUNDLE_SCHEMA_VERSION, + ); + } + if (creation.retrieval !== undefined) { database.prepare(UPSERT_RETRIEVAL).run(canonicalJson(creation.retrieval), 1, stamp); } return readRunRow(database, path); } -function firstExecution( - database: DatabaseSync, - path: string, -): { execution: DocumentExecutionRecord; replay: boolean } { - return { execution: insertExecution(database, path), replay: false }; +/** The immutable terms a creation offers for the run id it names. */ +export function creationComparison( + runId: string, + creation: WorkflowRunCreation, +): WorkflowRunComparison { + if (isGitWorkflowRunCreation(creation)) { + return { + runId, + definition: creation.definition, + base: creation.base, + props: creation.props, + }; + } + return { runId, definition: creation.definition, props: creation.props }; +} + +function firstExecution(database: DatabaseSync): { + execution: DocumentExecutionRecord; + replay: boolean; +} { + return { execution: insertExecution(database), replay: false }; } /** @@ -637,11 +994,6 @@ function terminal(status: WorkflowRunStatus): boolean { return status === "completed" || status === "failed"; } -interface Reconciled { - readonly status: WorkflowRunStatus; - readonly of: { recovered?: DocumentExecutionRecord }; -} - /** * Close what the previous workflow executor left, on the evidence the run itself holds. * @@ -731,7 +1083,7 @@ function rootOutcome( return undefined; } -function insertExecution(database: DatabaseSync, path: string): DocumentExecutionRecord { +function insertExecution(database: DatabaseSync): DocumentExecutionRecord { const executionId = randomUUID(); database.prepare(INSERT_EXECUTION).run(executionId, new Date().toISOString()); return readExecution(database, executionId); diff --git a/packages/workflow/src/fork.ts b/packages/workflow/src/fork.ts index a564fc2e4..284192d1e 100644 --- a/packages/workflow/src/fork.ts +++ b/packages/workflow/src/fork.ts @@ -35,7 +35,12 @@ import type { DurableEvent } from "@executablemd/durable-streams"; import { Err, Ok, type Result } from "effection"; -import { describeWorkflowRun, WORKFLOW_RUN, type WorkflowRun } from "./journal.ts"; +import { + describeWorkflowRun, + WORKFLOW_RUN, + type WorkflowRun, + workflowRunValue, +} from "./journal.ts"; import type { Forkability } from "./lifecycle/forkability.ts"; import { WorkflowRequestError } from "./storage/errors.ts"; @@ -130,11 +135,8 @@ export function forkRunRecordEvent(run: WorkflowRun): DurableEvent { return { type: "yield", coroutineId: ROOT_COROUTINE, - description: describeWorkflowRun(run.base), - result: { - status: "ok", - value: { runId: run.runId, base: run.base, pinnedCommit: run.pinnedCommit }, - }, + description: describeWorkflowRun(run), + result: { status: "ok", value: workflowRunValue(run) }, }; } diff --git a/packages/workflow/src/journal.ts b/packages/workflow/src/journal.ts index 51a8920fd..236ca82a8 100644 --- a/packages/workflow/src/journal.ts +++ b/packages/workflow/src/journal.ts @@ -20,24 +20,94 @@ import { StaleInputError } from "@executablemd/durable-streams"; import type { DurableEvent, EffectDescription } from "@executablemd/durable-streams"; /** - * One workflow run: an opaque identifier, the base that was asked for, and the - * commit that base resolved to once. + * One workflow run of a Git definition: an opaque identifier, the base that was + * asked for, and the commit that base resolved to once. */ -export interface WorkflowRun { +export interface GitWorkflowRunV1 { readonly runId: string; readonly base: string; readonly pinnedCommit: string; } +/** + * One workflow run of a retained source bundle. + * + * No base and no pinned commit: this run started from exact bytes rather than + * from a repository state, and a synthetic Git field here would be a claim + * about a repository the run never had. The bundle hash is what the retained + * definition is, and the exact target is what distinguishes two runs over it. + */ +export interface SourceBundleWorkflowRunV2 { + readonly runId: string; + readonly definitionVersion: 2; + readonly bundleHash: string; + readonly targetPath?: string; +} + +/** The value a journal records, and the value `getWorkflowRun()` answers with. */ +export type WorkflowRun = GitWorkflowRunV1 | SourceBundleWorkflowRunV2; + export const WORKFLOW_RUN = "workflow_run"; /** The coroutine a document execution's own history belongs to. */ const ROOT_COROUTINE = "root"; const RUN_MEMBERS: readonly string[] = ["runId", "base", "pinnedCommit"]; +const SOURCE_BUNDLE_MEMBERS: readonly string[] = ["runId", "definitionVersion", "bundleHash"]; +const SOURCE_BUNDLE_TARGET = "targetPath"; + +/** Whether this run is the Git one, narrowing to it when it is. */ +export function isGitWorkflowRun(run: WorkflowRun): run is GitWorkflowRunV1 { + return !("definitionVersion" in run); +} + +/** + * The record as a plain value, in the order its version presents it. + * + * Object-member order is presentation and not identity — a canonical-JSON + * container sorts these keys under its own rule without changing the value — + * but one ordinary spelling per version is what keeps a retained record byte + * for byte what it always was. + */ +export function workflowRunValue(run: WorkflowRun): Record { + if (isGitWorkflowRun(run)) { + return { runId: run.runId, base: run.base, pinnedCommit: run.pinnedCommit }; + } + return { + runId: run.runId, + definitionVersion: run.definitionVersion, + bundleHash: run.bundleHash, + ...(run.targetPath === undefined ? {} : { targetPath: run.targetPath }), + }; +} + +/** + * How the record identifies itself. + * + * The members past the type and the name are for a reader and never for + * matching: divergence detection compares only type and name. A version-2 + * description says so and carries its bundle hash; it invents no Git field. + */ +export function describeWorkflowRun(run: WorkflowRun): EffectDescription { + if (isGitWorkflowRun(run)) { + return describeGitWorkflowRun(run.base); + } + return { + type: WORKFLOW_RUN, + name: WORKFLOW_RUN, + definitionVersion: run.definitionVersion, + bundleHash: run.bundleHash, + }; +} -/** How the record identifies itself. `base` is for a reader, never for matching. */ -export function describeWorkflowRun(base: string): EffectDescription { +/** + * The same description for a Git run that has not resolved its commit yet. + * + * A programmatic run allocates its identifier and resolves its base inside the + * durable operation this description names, so the description exists before + * the run does. Version 2 has no equivalent: its run arrives whole. + */ +export function describeGitWorkflowRun(base: string): EffectDescription { return { type: WORKFLOW_RUN, name: WORKFLOW_RUN, base }; } @@ -93,6 +163,14 @@ export function readWorkflowRun(value: unknown): WorkflowRun | undefined { if (record === undefined) { return undefined; } + // The exact member set of one version or the exact member set of the other, + // in any order. Nothing in between: a value carrying members of both, or one + // of either with something extra, is not a run read loosely — it is a value + // this version cannot account for. + return readGitRun(record) ?? readSourceBundleRun(record); +} + +function readGitRun(record: Record): WorkflowRun | undefined { if ( Object.keys(record).length !== RUN_MEMBERS.length || !RUN_MEMBERS.every((member) => Object.hasOwn(record, member)) @@ -106,6 +184,37 @@ export function readWorkflowRun(value: unknown): WorkflowRun | undefined { return Object.freeze({ runId, base, pinnedCommit }); } +function readSourceBundleRun(record: Record): WorkflowRun | undefined { + const names = Object.keys(record); + const targeted = names.length === SOURCE_BUNDLE_MEMBERS.length + 1; + if (!targeted && names.length !== SOURCE_BUNDLE_MEMBERS.length) { + return undefined; + } + if (!SOURCE_BUNDLE_MEMBERS.every((member) => Object.hasOwn(record, member))) { + return undefined; + } + if (targeted && !Object.hasOwn(record, SOURCE_BUNDLE_TARGET)) { + return undefined; + } + const { runId, definitionVersion, bundleHash } = record; + if ( + typeof runId !== "string" || + definitionVersion !== 2 || + typeof bundleHash !== "string" || + bundleHash === "" + ) { + return undefined; + } + if (!targeted) { + return Object.freeze({ runId, definitionVersion: 2, bundleHash }); + } + const targetPath = record[SOURCE_BUNDLE_TARGET]; + if (typeof targetPath !== "string") { + return undefined; + } + return Object.freeze({ runId, definitionVersion: 2, bundleHash, targetPath }); +} + /** What one retained event claims about the run, when it claims anything. */ interface RunClaim { readonly settled: boolean; diff --git a/packages/workflow/src/lifecycle/execution.ts b/packages/workflow/src/lifecycle/execution.ts index 9c4ec317b..9a180e62a 100644 --- a/packages/workflow/src/lifecycle/execution.ts +++ b/packages/workflow/src/lifecycle/execution.ts @@ -15,22 +15,56 @@ import type { Operation, Result } from "effection"; import type { DurableEvent, Json } from "@executablemd/durable-streams"; import type { ExecutorLock } from "./api.ts"; import type { WorkflowRunDatabase } from "../storage/api.ts"; -import type { WorkflowDefinition } from "../storage/definition.ts"; +import type { GitWorkflowDefinitionV1 } from "../storage/definition.ts"; import type { JsonObject } from "../storage/members.ts"; +import type { + SourceBundleSnapshotEntryV2, + SourceBundleWorkflowDefinitionV2, +} from "../storage/source-bundle.ts"; +import type { RetainedDefinitionSources } from "./source.ts"; import type { DocumentExecutionCompletion, DocumentExecutionRecord, WorkflowRunRecord, } from "../storage/record.ts"; -/** What a `start` creates a run from, and what compatible reuse is compared against. */ -export interface WorkflowRunCreation { - readonly definition: WorkflowDefinition; +/** What a `start` creates a Git run from, and what compatible reuse compares. */ +export interface GitWorkflowRunCreationV1 { + readonly definition: GitWorkflowDefinitionV1; readonly base: string; readonly props: JsonObject; readonly retrieval?: Json; } +/** + * What a `start` creates a source-bundle run from. + * + * The descriptor and the bytes arrive together, because they are one fact: a + * version-2 run's authoritative content is what its store holds, so a creation + * that carried only the descriptor would be asking storage to retain an + * identity for content nobody supplied. + * + * `sourceSnapshot` names exactly the descriptor's paths, in exactly its order. + * The transition copies every byte sequence before it validates anything, so + * mutating a caller-owned array afterwards cannot change the run. + */ +export interface SourceBundleWorkflowRunCreationV2 { + readonly definition: SourceBundleWorkflowDefinitionV2; + readonly sourceSnapshot: readonly SourceBundleSnapshotEntryV2[]; + readonly props: JsonObject; + readonly retrieval?: Json; +} + +/** What a `start` creates a run from, by the definition version it creates. */ +export type WorkflowRunCreation = GitWorkflowRunCreationV1 | SourceBundleWorkflowRunCreationV2; + +/** Whether this creation is the Git one, narrowing to it when it is. */ +export function isGitWorkflowRunCreation( + creation: WorkflowRunCreation, +): creation is GitWorkflowRunCreationV1 { + return creation.definition.kind === "git"; +} + /** Which committed checkpoint of which run a fork continues. */ export interface WorkflowForkSelection { readonly sourceRunId: string; @@ -81,6 +115,24 @@ export interface WorkflowExecutionBegun { readonly replay: boolean; /** What stale recovery closed on the way in, when it closed anything. */ readonly recovered?: DocumentExecutionRecord; + /** + * The source this run executes, authenticated against its own definition. + * + * Answered by the transition rather than fetched afterwards, so the check + * happens under the executor lock and before stale recovery, a new execution + * record, Workspace attachment or any other lifecycle write. A caller imports + * this closure; rereading the candidate path after admission would be a + * second source of truth about what the run is a run of. + * + * Always present. A source-bundle run's content comes out of its own store; + * a Git run's comes through the host's legacy reader, which is consulted + * against the descriptor about to be persisted — before persistence — when + * the run is being created, and against the retained descriptor when it + * already exists. A host that installed no reader cannot begin a Git run at + * all, which is the honest answer: it has no way to obtain what that run + * executes. + */ + readonly sources: RetainedDefinitionSources; } /** diff --git a/packages/workflow/src/lifecycle/source.ts b/packages/workflow/src/lifecycle/source.ts new file mode 100644 index 000000000..34e6c1067 --- /dev/null +++ b/packages/workflow/src/lifecycle/source.ts @@ -0,0 +1,105 @@ +/** + * The Markdown a retained definition names, and how a host that has it supplies + * it. + * + * A run's definition is identity; its source is content. Version 2 keeps both, + * so its content comes out of the run's own store. Version 1 keeps only + * identity — an object id and a path inside a commit — so its content has to be + * fetched, and fetching it means reaching a repository. + * + * The retained lifecycle does neither. It receives a closure and holds it to + * the descriptor it asked about, which is what makes "these are the bytes this + * run is a run of" a checkable claim rather than an assertion by whoever did + * the reading. + * + * ## The legacy reader is a closure, never a name + * + * `LegacyWorkflowSourceReader` is captured by the trusted host before document + * code exists. It is reachable through no Context, contextual Api, component, + * Plugin installation result or authored value: a source capability something + * in the process could reach by name is a way to decide what a run executes. + */ + +import type { Operation, Result } from "effection"; +import type { Json } from "@executablemd/durable-streams"; +import type { GitWorkflowDefinitionV1 } from "../storage/definition.ts"; +import type { SourceBundleWorkflowDefinitionV2 } from "../storage/source-bundle.ts"; + +/** + * The root document a version-1 definition names, and the bytes behind it. + * + * The descriptor members repeat what the definition already pins so the closure + * can be checked against it: a closure whose Markdown belongs to a different + * commit than the definition names is not a closure of that definition. + */ +export interface GitDefinitionSourceRootV1 { + readonly objectFormat: "sha1" | "sha256"; + /** The commit the definition pins, as the definition's own object id. */ + readonly pinnedCommit: string; + readonly rootDocumentPath: string; + /** One exact canonical document target, when the definition selects one. */ + readonly targetPath?: string; + /** The Git blob identity of `content`, under `objectFormat`. */ + readonly blobId: string; + readonly content: string; +} + +/** One declared component of a version-1 definition, including an unexpanded one. */ +export interface GitDefinitionSourceComponentV1 { + readonly name: string; + readonly path: string; + /** The Git blob identity of `content`, under the root's `objectFormat`. */ + readonly blobId: string; + readonly content: string; +} + +/** Everything a version-1 definition is closed over, without its repository. */ +export interface GitDefinitionSourceClosureV1 { + readonly root: GitDefinitionSourceRootV1; + readonly components: readonly GitDefinitionSourceComponentV1[]; +} + +/** One retained version-2 source: its logical path and its exact bytes. */ +export interface SourceBundleRetainedSourceV2 { + readonly path: string; + readonly bytes: Uint8Array; +} + +/** A version-1 run's authenticated source, with the definition it was checked against. */ +export interface GitRetainedDefinitionSourcesV1 { + readonly definitionVersion: 1; + readonly definition: GitWorkflowDefinitionV1; + readonly closure: GitDefinitionSourceClosureV1; +} + +/** + * A version-2 run's authenticated source, in the descriptor's canonical order. + * + * The bytes are the store's own, read back after they were committed rather + * than the buffers a caller offered — so what executes is what the run + * retained, not what somebody still holds a reference to. + */ +export interface SourceBundleRetainedDefinitionSourcesV2 { + readonly definitionVersion: 2; + readonly definition: SourceBundleWorkflowDefinitionV2; + readonly sources: readonly SourceBundleRetainedSourceV2[]; +} + +/** The complete verified source closure one run executes. */ +export type RetainedDefinitionSources = + | GitRetainedDefinitionSourcesV1 + | SourceBundleRetainedDefinitionSourcesV2; + +/** + * How a trusted host turns a retained version-1 definition back into Markdown. + * + * Supplied to the workflow host as a direct dependency and captured in its + * closure. It receives only the parsed descriptor and the run's replaceable + * retrieval metadata; whatever it does to reach a repository is the host's + * business. Workflow — not the adapter — decides whether what came back + * describes the definition it asked about. + */ +export type LegacyWorkflowSourceReader = ( + definition: GitWorkflowDefinitionV1, + retrieval: Json | undefined, +) => Operation>; diff --git a/packages/workflow/src/run.ts b/packages/workflow/src/run.ts index 97c9d978d..a30e9ef7a 100644 --- a/packages/workflow/src/run.ts +++ b/packages/workflow/src/run.ts @@ -72,10 +72,13 @@ import type { RetainedIdentity } from "./git-host/identities.ts"; import { admitWorkflowRunHistory, baseMismatch, + describeGitWorkflowRun, describeWorkflowRun, + isGitWorkflowRun, malformedRecord, readWorkflowRun, retainedRunMismatch, + workflowRunValue, } from "./journal.ts"; import type { RunHistoryRules, WorkflowRun } from "./journal.ts"; @@ -168,7 +171,13 @@ export function* getWorkflowRun(): Operation { * records a different one is not this run's journal. */ interface RunPreparation extends RunHistoryRules { - readonly base: string; + /** + * How the durable record identifies itself. + * + * Carried rather than derived from a base, because a version-2 run has none: + * its description says which version it is and what bundle it retains. + */ + readonly description: EffectDescription; /** The run this execution is of, reached only when nothing is recorded yet. */ allocate(): Operation; } @@ -179,14 +188,13 @@ function* record(description: EffectDescription, preparation: RunPreparation): W // Reached only when nothing is recorded yet: a replay hands the stored // value back without running this at all, so neither the identifier nor Git // is reached a second time. - const { runId, base, pinnedCommit } = yield* preparation.allocate(); - return { runId, base, pinnedCommit }; + return workflowRunValue(yield* preparation.allocate()); }); } function allocating(base: string): RunPreparation { return { - base, + description: describeGitWorkflowRun(base), // A base that would not resolve is recorded as a failed effect (§6), and a // history whose only record is that failure is this run's own. Requiring a // successful one would retry Git instead of replaying what happened. @@ -203,6 +211,11 @@ function allocating(base: string): RunPreparation { * against the stored *value* rather than against the entry's identity. */ agree(recorded: WorkflowRun): WorkflowRun { + // This installation allocates a Git run, so a recorded source bundle is + // not a base disagreement — it is a different kind of run entirely. + if (!isGitWorkflowRun(recorded)) { + throw retainedRunMismatch(["definition version"]); + } if (recorded.base !== base) { throw baseMismatch(recorded.base, base); } @@ -213,7 +226,7 @@ function allocating(base: string): RunPreparation { function retaining(run: WorkflowRun): RunPreparation { return { - base: run.base, + description: describeWorkflowRun(run), // The host created this run before anything executed, so a history of its // own is something it must have: none, or one that only failed, means the // recorded work is not this run's. @@ -223,9 +236,7 @@ function retaining(run: WorkflowRun): RunPreparation { return run; }, agree(recorded: WorkflowRun): WorkflowRun { - const differing = (["runId", "base", "pinnedCommit"] as const).filter( - (field) => recorded[field] !== run[field], - ); + const differing = differingFields(recorded, run); if (differing.length > 0) { throw retainedRunMismatch(differing); } @@ -234,6 +245,35 @@ function retaining(run: WorkflowRun): RunPreparation { }; } +/** + * Which terms of the retained run a recorded one disagrees with. + * + * Two versions are never the same run, and saying so as one field is what stops + * the comparison from reporting members one of them does not have. Within a + * version every term is compared, the exact target included: absent equals only + * absent, which is what keeps a whole-document run distinct from a targeted one. + */ +function differingFields(recorded: WorkflowRun, expected: WorkflowRun): string[] { + if (isGitWorkflowRun(recorded)) { + if (!isGitWorkflowRun(expected)) { + return ["definition version"]; + } + return [ + ...(recorded.runId === expected.runId ? [] : ["runId"]), + ...(recorded.base === expected.base ? [] : ["base"]), + ...(recorded.pinnedCommit === expected.pinnedCommit ? [] : ["pinnedCommit"]), + ]; + } + if (isGitWorkflowRun(expected)) { + return ["definition version"]; + } + return [ + ...(recorded.runId === expected.runId ? [] : ["runId"]), + ...(recorded.bundleHash === expected.bundleHash ? [] : ["bundleHash"]), + ...(recorded.targetPath === expected.targetPath ? [] : ["targetPath"]), + ]; +} + /** * Read one member set off a value a host supplied, or answer that it refused. * @@ -258,11 +298,7 @@ function held(stored: unknown, preparation: RunPreparation): WorkflowRun { } function same(left: WorkflowRun, right: WorkflowRun): boolean { - return ( - left.runId === right.runId && - left.base === right.base && - left.pinnedCommit === right.pinnedCommit - ); + return differingFields(left, right).length === 0; } /** @@ -281,7 +317,7 @@ function same(left: WorkflowRun, right: WorkflowRun): boolean { * run and the admission is what installs the recorded run. */ function* prepare(preparation: RunPreparation): Workflow { - const description = describeWorkflowRun(preparation.base); + const description = preparation.description; // Which run this is, and whether the journal agrees, are decided by the // captured `preparation` and the durable record — never by what the slot // happens to hold. @@ -430,14 +466,18 @@ function retainedRun(run: WorkflowRun): WorkflowRun { // Named through the same total read as a journal value: a host that hands // over a record whose members refuse to be read has supplied a value that // identifies no run, which is the sentence below rather than its exception. - const parsed = readWorkflowRun( - readingRetainedValue(() => ({ - runId: run?.runId, - base: run?.base, - pinnedCommit: run?.pinnedCommit, - })), - ); - if (parsed === undefined || parsed.runId === "" || parsed.base === "") { + const parsed = readWorkflowRun(readingRetainedValue(() => workflowRunValue(run))); + if (parsed === undefined || parsed.runId === "") { + throw new Error( + "retainedWorkflowInstallation() needs a complete retained run: a Git run's id, base and " + + "pinned commit, or a source-bundle run's id and bundle hash. A run installed without " + + "them identifies no workflow run.", + ); + } + if (!isGitWorkflowRun(parsed)) { + return parsed; + } + if (parsed.base === "") { throw new Error( "retainedWorkflowInstallation() needs the retained run's id, base and pinned commit. A run " + "installed without them identifies no workflow run.", diff --git a/packages/workflow/src/storage/api.ts b/packages/workflow/src/storage/api.ts index bab0e333d..ca14b794a 100644 --- a/packages/workflow/src/storage/api.ts +++ b/packages/workflow/src/storage/api.ts @@ -42,17 +42,25 @@ import { type Api, createApi } from "@effectionx/context-api"; import type { Operation, Result } from "effection"; import type { DurableEvent, DurableStream, Json } from "@executablemd/durable-streams"; -import type { WorkflowDefinition } from "./definition.ts"; +import type { GitWorkflowDefinitionV1 } from "./definition.ts"; import { WorkflowStorageError } from "./errors.ts"; import type { JsonObject } from "./members.ts"; import type { DefinitionRetrieval, DocumentExecutionRecord, WorkflowRunRecord } from "./record.ts"; -/** What a caller must decide before a run can exist. */ +/** + * What a caller must decide before a run can exist. + * + * Version 1 only, and deliberately. A source-bundle definition names bytes + * nobody has supplied here: this request carries a descriptor and no source, so + * admitting one would create a database whose authoritative content was never + * given to it. Creating a version-2 run crosses the trusted lifecycle + * transition instead, which takes the complete snapshot with the descriptor. + */ export interface CreateWorkflowRunRequest { /** The public run id. Retained inside the run, and the only way back to it. */ readonly runId: string; - /** The immutable definition this run is a run of. */ - readonly definition: WorkflowDefinition; + /** The immutable Git definition this run is a run of. */ + readonly definition: GitWorkflowDefinitionV1; /** The Git revision chosen as the run's starting repository state. */ readonly base: string; /** diff --git a/packages/workflow/src/storage/compatibility.ts b/packages/workflow/src/storage/compatibility.ts index e936ad309..a47f61c22 100644 --- a/packages/workflow/src/storage/compatibility.ts +++ b/packages/workflow/src/storage/compatibility.ts @@ -4,23 +4,58 @@ * Reuse of a run id is the mechanism a caller has for saying "the same run * again", so the question is not whether two requests are byte-identical but * whether they describe one run. Identity is the run id, the whole definition - * descriptor including its version, the base, and the normalized props. Values - * are compared canonically, so props that differ only in key order are the same - * props. + * descriptor including its version, the base a Git run started from, and the + * normalized props. Values are compared canonically, so props that differ only + * in key order are the same props. * * Everything a run accumulates is excluded: status, stop reason, retrieval * metadata, timestamps, document executions and journal records all change * while the run stays the run it was. A completed run that is asked for again * is found, not refused. + * + * ## Each version is compared by its own identity + * + * A Git run is its object, its path, its target and its declared bundle. A + * source-bundle run is its entrypoint, the complete canonical source manifest, + * its component mapping and its target. Neither comparison is the other's with + * a member missing, and two descriptors of different versions are never one + * run — so a cross-version request disagrees as `definition` and is not then + * asked about a base one of them does not have. */ -import type { CreateWorkflowRunRequest } from "./api.ts"; import { definitionComponents, + type GitWorkflowDefinitionV1, type WorkflowComponentEntry, type WorkflowDefinition, } from "./definition.ts"; -import { canonicalJson, type WorkflowRunRecord } from "./record.ts"; +import type { JsonObject } from "./members.ts"; +import { canonicalJson, isGitWorkflowRunRecord, type WorkflowRunRecord } from "./record.ts"; +import type { + SourceBundleComponentV2, + SourceBundleEntryV2, + SourceBundleWorkflowDefinitionV2, +} from "./source-bundle.ts"; + +/** What one reuse of a Git run id is compared against. */ +export interface GitWorkflowRunComparisonV1 { + readonly runId: string; + readonly definition: GitWorkflowDefinitionV1; + readonly base: string; + readonly props: JsonObject; +} + +/** What one reuse of a source-bundle run id is compared against. */ +export interface SourceBundleWorkflowRunComparisonV2 { + readonly runId: string; + readonly definition: SourceBundleWorkflowDefinitionV2; + readonly props: JsonObject; +} + +/** The immutable terms one request offers for the run id it names. */ +export type WorkflowRunComparison = + | GitWorkflowRunComparisonV1 + | SourceBundleWorkflowRunComparisonV2; /** * The immutable fields in which a stored run and a request disagree. @@ -31,7 +66,7 @@ import { canonicalJson, type WorkflowRunRecord } from "./record.ts"; */ export function conflictingFields( stored: WorkflowRunRecord, - request: CreateWorkflowRunRequest, + request: WorkflowRunComparison, ): string[] { const fields: string[] = []; @@ -41,8 +76,14 @@ export function conflictingFields( if (!sameDefinition(stored.definition, request.definition)) { fields.push("definition"); } - if (stored.base !== request.base) { - fields.push("base"); + // Only when both sides are Git runs. A source bundle has no base at all, and + // two descriptors of different versions have already disagreed as definitions + // — reporting a second field about a member one of them never had would name + // a disagreement the caller cannot act on. + if (isGitWorkflowRunRecord(stored) && isGitComparison(request)) { + if (stored.base !== request.base) { + fields.push("base"); + } } if (canonicalJson(stored.props) !== canonicalJson(request.props)) { fields.push("props"); @@ -51,6 +92,10 @@ export function conflictingFields( return fields; } +function isGitComparison(request: WorkflowRunComparison): request is GitWorkflowRunComparisonV1 { + return request.definition.kind === "git"; +} + /** * Compared member by member rather than canonically. * @@ -59,21 +104,25 @@ export function conflictingFields( * keeps a later variant from being admitted because it happened to serialize * the same way. * - * The exact target is one of those members. A run of one section and a run of - * the whole document are different runs, and so are runs of two different - * sections: they execute different content, so reusing one run id for the other - * would let a resumed run continue something it never started. Absent compares - * equal only to absent, which is what makes whole-document and targeted - * definitions incompatible rather than merely unequal. - * - * The component bundle is compared on the same terms, and the version is - * compared first: a run closed over a bundle is never the same run as one - * closed over none. + * The exact target is one of those members in both versions. A run of one + * section and a run of the whole document are different runs, and so are runs + * of two different sections: they execute different content, so reusing one run + * id for the other would let a resumed run continue something it never started. + * Absent compares equal only to absent. */ function sameDefinition(stored: WorkflowDefinition, requested: WorkflowDefinition): boolean { + if (stored.kind === "git") { + return requested.kind === "git" && sameGitDefinition(stored, requested); + } + return requested.kind === "source-bundle" && sameSourceBundle(stored, requested); +} + +function sameGitDefinition( + stored: GitWorkflowDefinitionV1, + requested: GitWorkflowDefinitionV1, +): boolean { return ( stored.version === requested.version && - stored.kind === requested.kind && stored.objectFormat === requested.objectFormat && stored.objectId === requested.objectId && stored.rootDocumentPath === requested.rootDocumentPath && @@ -82,6 +131,61 @@ function sameDefinition(stored: WorkflowDefinition, requested: WorkflowDefinitio ); } +/** + * The whole bundle, not the hash that commits to it. + * + * The bundle hash already names this manifest, and it is compared too — but a + * hash is not a reason to admit a retained structure that disagrees with + * itself, and a stored descriptor whose manifest no longer matches its hash + * must not be reused merely because a candidate carried the same hash. + */ +function sameSourceBundle( + stored: SourceBundleWorkflowDefinitionV2, + requested: SourceBundleWorkflowDefinitionV2, +): boolean { + return ( + stored.version === requested.version && + stored.hashAlgorithm === requested.hashAlgorithm && + stored.bundleHash === requested.bundleHash && + stored.entrypoint === requested.entrypoint && + stored.targetPath === requested.targetPath && + sameSources(stored.sources, requested.sources) && + sameMapping(stored.components ?? [], requested.components ?? []) && + (stored.components === undefined) === (requested.components === undefined) + ); +} + +function sameSources( + stored: readonly SourceBundleEntryV2[], + requested: readonly SourceBundleEntryV2[], +): boolean { + if (stored.length !== requested.length) { + return false; + } + return stored.every((source, index) => { + const other = requested[index]; + return ( + other !== undefined && + source.path === other.path && + source.sourceHash === other.sourceHash && + source.byteLength === other.byteLength + ); + }); +} + +function sameMapping( + stored: readonly SourceBundleComponentV2[], + requested: readonly SourceBundleComponentV2[], +): boolean { + if (stored.length !== requested.length) { + return false; + } + return stored.every((component, index) => { + const other = requested[index]; + return other !== undefined && component.name === other.name && component.path === other.path; + }); +} + /** * The bundle is compared whole, entry by entry, in the canonical order both * descriptors were parsed in. diff --git a/packages/workflow/src/storage/definition.ts b/packages/workflow/src/storage/definition.ts index cda336a5b..b60e85894 100644 --- a/packages/workflow/src/storage/definition.ts +++ b/packages/workflow/src/storage/definition.ts @@ -19,6 +19,11 @@ import { Err, Ok, type Result } from "effection"; import { isCanonicalDocumentTarget, isComponentName } from "@executablemd/core"; import type { Json } from "@executablemd/durable-streams"; import { WorkflowDefinitionError } from "./errors.ts"; +import { + parseSourceBundleDefinition, + sourceBundleDefinitionToJson, + type SourceBundleWorkflowDefinitionV2, +} from "./source-bundle.ts"; import { describe, type Members, @@ -74,7 +79,21 @@ export interface WorkflowComponentEntry { } /** Every descriptor this build understands. */ -export type WorkflowDefinition = GitWorkflowDefinitionV1; +export type WorkflowDefinition = GitWorkflowDefinitionV1 | SourceBundleWorkflowDefinitionV2; + +/** Whether this descriptor is the Git one, narrowing to it when it is. */ +export function isGitWorkflowDefinition( + definition: WorkflowDefinition, +): definition is GitWorkflowDefinitionV1 { + return definition.kind === "git"; +} + +/** Whether this descriptor is a source bundle, narrowing to it when it is. */ +export function isSourceBundleWorkflowDefinition( + definition: WorkflowDefinition, +): definition is SourceBundleWorkflowDefinitionV2 { + return definition.kind === "source-bundle"; +} /** Hexadecimal digits per object id, by the format that names them. */ const OBJECT_ID_LENGTHS: Readonly> = { @@ -104,8 +123,18 @@ function fail(reason: string, path: string): Error { * Parsed rather than asserted: a descriptor reaches storage from a host, and a * host that builds one by hand — or reads one from a file — can build one that * type-checks and does not describe a definition. + * + * `kind` chooses the parser, not `version`. A kind says what sort of thing a + * descriptor identifies and a version says which revision of that sort it is, + * so a Git descriptor carrying some other version is refused by the Git parser + * — which is where a reader looking at `objectId` and `rootDocumentPath` is + * told what went wrong. Dispatching on the version instead would answer a + * mis-numbered Git descriptor with the source bundle's member list. */ export function parseWorkflowDefinition(value: unknown): Result { + if (declaredKind(value) === "source-bundle") { + return parseSourceBundleDefinition(value); + } try { return Ok(parseDefinition(value)); } catch (error) { @@ -116,6 +145,25 @@ export function parseWorkflowDefinition(value: unknown): Result key === "kind")?.[1]; + return typeof kind === "string" ? kind : undefined; + } catch { + return undefined; + } +} + function parseDefinition(value: unknown): WorkflowDefinition { const members = parseMembers(value, "$", fail); requireMemberNames(members, MEMBER_NAMES, "$", fail); @@ -272,6 +320,9 @@ function parseTargetPath(members: Members): string | undefined { * stored shape and the parsed shape one decision. */ export function definitionToJson(definition: WorkflowDefinition): Json { + if (isSourceBundleWorkflowDefinition(definition)) { + return sourceBundleDefinitionToJson(definition); + } return { version: definition.version, kind: definition.kind, @@ -297,13 +348,26 @@ export function definitionToJson(definition: WorkflowDefinition): Json { }; } -/** The bundle this definition is closed over, empty when it is closed over none. */ +/** + * The Git bundle this definition is closed over, empty when it is closed over + * none. + * + * A Git entry names a blob inside the pinned tree and a source-bundle entry + * names a retained source, so the two mappings are not one list with a + * different member. A caller holding the union asks whichever question its + * version has. + */ export function definitionComponents( - definition: WorkflowDefinition, + definition: GitWorkflowDefinitionV1, ): readonly WorkflowComponentEntry[] { return definition.components ?? []; } +/** The exact document target this definition names, whichever version it is. */ +export function definitionTargetPath(definition: WorkflowDefinition): string | undefined { + return definition.targetPath; +} + function parseObjectFormat(value: unknown): GitWorkflowDefinitionV1["objectFormat"] { if (value === "sha1" || value === "sha256") { return value; diff --git a/packages/workflow/src/storage/errors.ts b/packages/workflow/src/storage/errors.ts index 16196ad51..23ba2ea8e 100644 --- a/packages/workflow/src/storage/errors.ts +++ b/packages/workflow/src/storage/errors.ts @@ -255,6 +255,97 @@ export class WorkflowDatabaseClosedError extends WorkflowStorageError { } } +/** + * A version-2 run's own retained source is not there. + * + * Its own category because there is nothing to fall back to and nothing to + * repair: the descriptor names a manifest entry or a blob the live store does + * not contain, so the run's authoritative content is gone. The original file, + * the optional provenance and the legacy Git reader are all deliberately not + * consulted — each would execute bytes this run is not a run of. + */ +export class WorkflowDefinitionSourceMissingError extends WorkflowStorageError { + override name = "WorkflowDefinitionSourceMissingError"; + + constructor() { + super( + "This workflow run retains a source bundle whose content its own store no longer holds. " + + "The run is left unchanged, and no other source is substituted for it: restore the " + + "run's database from a backup, or start a new run.", + ); + } +} + +/** + * A version-2 run's retained source is there and does not describe itself. + * + * Distinct from absence because an operator acts on it differently. A path, a + * length, a source hash, a component mapping or the bundle hash disagrees with + * what the descriptor states, so what is retained is no longer evidence of the + * definition it claims. No partial closure is returned. + */ +export class WorkflowDefinitionCorruptError extends WorkflowStorageError { + override name = "WorkflowDefinitionCorruptError"; + + constructor(reason: string) { + super( + `This workflow run's retained source bundle disagrees with its own descriptor: ${reason}. ` + + "No partial source is returned and the run is left unchanged.", + ); + } +} + +/** + * A version-1 run needs its Git source and this host installed no reader. + * + * The retained lifecycle performs no Git operation of its own. A host that can + * reach the repository supplies that capability directly when it installs the + * workflow host, and one that did not cannot obtain a version-1 definition's + * Markdown at all. + */ +export class LegacyWorkflowSourceReaderUnavailableError extends WorkflowStorageError { + override name = "LegacyWorkflowSourceReaderUnavailableError"; + + constructor() { + super( + "This workflow run retains a Git definition, and this host installed no legacy workflow " + + "source reader. Run it from a Git-capable XMD host. No execution record was created.", + ); + } +} + +/** The installed legacy reader cannot obtain the object a v1 definition pins. */ +export class LegacyWorkflowSourceUnavailableError extends WorkflowStorageError { + override name = "LegacyWorkflowSourceUnavailableError"; + + constructor(reason: string) { + super( + `This workflow run's retained Git definition could not be read: ${reason}. Current \`HEAD\` ` + + "and working-tree bytes are not substituted for it, and the run is left unchanged.", + ); + } +} + +/** + * The legacy reader answered, and its answer is not this definition's. + * + * Workflow validates every returned closure against the descriptor it asked + * about rather than trusting the adapter that produced it. A reader that + * returned another commit's bytes, another path, or a different component set + * has not obtained this run's source, and none of what it returned executes. + */ +export class LegacyWorkflowSourceMismatchError extends WorkflowStorageError { + override name = "LegacyWorkflowSourceMismatchError"; + + constructor(reason: string) { + super( + `The legacy workflow source reader returned a source closure that does not describe this ` + + `run's retained Git definition: ${reason}. None of it executes and the run is left ` + + "unchanged.", + ); + } +} + /** A value offered as a workflow definition descriptor does not describe one. */ export class WorkflowDefinitionError extends WorkflowStorageError { override name = "WorkflowDefinitionError"; diff --git a/packages/workflow/src/storage/record.ts b/packages/workflow/src/storage/record.ts index 37c78d0f5..54122788a 100644 --- a/packages/workflow/src/storage/record.ts +++ b/packages/workflow/src/storage/record.ts @@ -16,7 +16,8 @@ import { Err, Ok, type Result } from "effection"; import { canonicalize } from "@executablemd/core"; import type { Json } from "@executablemd/durable-streams"; -import type { WorkflowDefinition } from "./definition.ts"; +import type { GitWorkflowDefinitionV1 } from "./definition.ts"; +import type { SourceBundleWorkflowDefinitionV2 } from "./source-bundle.ts"; import { WorkflowRequestError } from "./errors.ts"; import { describe, @@ -64,10 +65,15 @@ export type WorkflowStopReason = | { readonly kind: "host"; readonly code: string } | { readonly kind: "journal"; readonly eventId: string }; -/** One workflow run's retained metadata. */ -export interface WorkflowRunRecord { +/** + * One workflow run of a Git definition, and the base it started from. + * + * Unchanged: this is the record every version-1 run has retained, in the + * members and the order it retained them. + */ +export interface GitWorkflowRunRecordV1 { readonly runId: string; - readonly definition: WorkflowDefinition; + readonly definition: GitWorkflowDefinitionV1; readonly base: string; readonly props: JsonObject; readonly status: WorkflowRunStatus; @@ -76,6 +82,39 @@ export interface WorkflowRunRecord { readonly updatedAt: string; } +/** + * One workflow run of a retained source bundle. + * + * No `base` and no pinned commit. Those are Git version-1 fields, and a + * synthetic one here would be a repository state this run never had — read back + * by anything comparing identity as though the run had named it. + */ +export interface SourceBundleWorkflowRunRecordV2 { + readonly runId: string; + readonly definition: SourceBundleWorkflowDefinitionV2; + readonly props: JsonObject; + readonly status: WorkflowRunStatus; + readonly stopReason?: WorkflowStopReason; + readonly createdAt: string; + readonly updatedAt: string; +} + +/** One workflow run's retained metadata, by the definition version it retains. */ +export type WorkflowRunRecord = GitWorkflowRunRecordV1 | SourceBundleWorkflowRunRecordV2; + +/** + * Whether this record is a Git run, narrowing to it when it is. + * + * The discriminator is the definition's own kind rather than a member repeated + * on the record: one value decides what a run is, and a second copy of it could + * disagree with the first. + */ +export function isGitWorkflowRunRecord( + record: WorkflowRunRecord, +): record is GitWorkflowRunRecordV1 { + return record.definition.kind === "git"; +} + /** * Where this host can fetch the definition from, as of now. * diff --git a/packages/workflow/tests/public-entrypoint.test.ts b/packages/workflow/tests/public-entrypoint.test.ts index 4095eaa82..2232c1d65 100644 --- a/packages/workflow/tests/public-entrypoint.test.ts +++ b/packages/workflow/tests/public-entrypoint.test.ts @@ -29,6 +29,8 @@ import { withWorkflowWorkspace } from "@executablemd/workflow/deno"; import type { WorkflowWorkspaceOptions } from "@executablemd/workflow/deno"; import * as published from "@executablemd/workflow/deno"; import { useInvokingHome } from "./support/credential-home.ts"; +import { readdir, readTextFile, stat } from "@effectionx/fs"; +import type { Operation } from "effection"; /** * A compile-time proof, not a runtime one. @@ -160,3 +162,173 @@ describe("workflow published Deno entrypoint", () => { expect(yield* until(Promise.resolve(true))).toBe(true); }); }); + +/** + * What the retained path is allowed to depend on. + * + * A version-1 definition's Markdown lives in a repository, and #443's whole + * point is that the modules which retain, recognize, resume, journal and seal a + * run do not reach one themselves — a trusted host supplies that capability as + * a direct closure instead. So this reads the source tree rather than the + * module graph: an import is a fact about a file, and a test that only exercised + * behaviour would pass right up until something imported Git and never used it. + * + * One exception, and it is pinned rather than granted. `src/run.ts` holds the + * public `workflowInstallation({ base })` convenience, which resolves a base + * through `Git.revParse()`. What this permits is that one named import and its + * one use inside the allocation path — not the file. A second Git operation + * reaching `run.ts`, or `revParse()` moving out of `allocating()` and into the + * retained path, fails here as surely as an import anywhere else would. + * #822 moves that adapter into the bundled Git Plugin. + */ +describe("workflow retained modules and the Git capability", () => { + /** The directories whose modules retain, recognize, resume or seal a run. */ + const RETAINED = ["src/storage", "src/lifecycle", "src/deno/artifact", "src/deno/workspace"]; + + /** Single files on that same path, beside the directories above. */ + const RETAINED_FILES = ["src/journal.ts", "src/fork.ts", "src/bundle.ts"]; + + /** + * The one module #822 has not moved yet. + * + * Named as a path rather than allowed by pattern: an exception that matched a + * shape would quietly cover the next file that happened to fit it. + */ + const EXCEPTION = "src/run.ts"; + + /** The exact import that exception is, and the one operation it names. */ + const EXCEPTION_IMPORT = "./git.ts"; + const EXCEPTION_OPERATION = "revParse"; + + function packageFile(relative: string): string { + return fileURLToPath(new URL(`../${relative}`, import.meta.url)); + } + + /** + * Every module specifier a file names, in every form that reaches one. + * + * `from "x"` is only one of them. A bare `import "x"` runs a module for its + * effects, `import("x")` reaches one at runtime, and `require("x")` reaches + * one from CommonJS — so the specifier is extracted from all four rather than + * the statement matched in one. + */ + function specifiers(source: string): string[] { + const found: string[] = []; + const pattern = /(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\brequire\s*\(\s*)["']([^"']+)["']/g; + for (const match of source.matchAll(pattern)) { + const specifier = match[1]; + if (specifier !== undefined) { + found.push(specifier); + } + } + return found; + } + + /** Whether a specifier names the local Git capability or its package. */ + function namesGit(specifier: string): boolean { + return /(?:^|\/)git\.ts$/.test(specifier) || /^@executablemd\/git(?:\/|$)/.test(specifier); + } + + function gitSpecifiers(source: string): string[] { + return specifiers(source).filter(namesGit); + } + + /** Every `.ts` file under one directory of the package, recursively. */ + function* moduleFiles(relative: string): Operation { + const found: string[] = []; + for (const name of yield* readdir(packageFile(relative))) { + const child = `${relative}/${name}`; + const stats = yield* stat(packageFile(child)); + if (stats.isDirectory()) { + found.push(...(yield* moduleFiles(child))); + } else if (name.endsWith(".ts")) { + found.push(child); + } + } + return found; + } + + function* retainedModules(): Operation { + const scanned: string[] = [...RETAINED_FILES]; + for (const directory of RETAINED) { + scanned.push(...(yield* moduleFiles(directory))); + } + // The deno adapter's own modules, without the repository-composition + // subsystem: those implement `` and are a capability rather than + // part of what a run retains. + for (const name of yield* readdir(packageFile("src/deno"))) { + if (name.endsWith(".ts")) { + scanned.push(`src/deno/${name}`); + } + } + return scanned; + } + + it("names Git in no retained module, in any import form", function* () { + const scanned = yield* retainedModules(); + + // The scan has to be looking at something: a glob that matched nothing + // would pass this case every time. + expect(scanned.length).toBeGreaterThan(40); + expect(scanned).toContain("src/deno/transitions.ts"); + expect(scanned).toContain("src/storage/source-bundle.ts"); + expect(scanned).toContain("src/lifecycle/source.ts"); + expect(scanned).toContain("src/deno/definition-source.ts"); + expect(scanned).not.toContain(EXCEPTION); + + // And the matcher has to recognize what it is looking for. Each of these is + // a way a module could reach Git without writing `from`. + for (const form of [ + 'import { revParse } from "./git.ts";', + 'import "../git.ts";', + 'const git = await import("./git.ts");', + 'const git = require("@executablemd/git");', + 'export { revParse } from "../../git.ts";', + ]) { + expect({ form, git: gitSpecifiers(form).length }).toEqual({ form, git: 1 }); + } + expect(gitSpecifiers('import { reading } from "./reading.ts";')).toEqual([]); + + const importing: string[] = []; + for (const relative of scanned) { + const source = yield* readTextFile(packageFile(relative)); + if (gitSpecifiers(source).length > 0) { + importing.push(relative); + } + } + expect(importing).toEqual([]); + }); + + it("permits one Git import in run.ts, used once inside the allocation path", function* () { + const source = yield* readTextFile(packageFile(EXCEPTION)); + + // One specifier, and it is the local capability rather than the package. + expect(gitSpecifiers(source)).toEqual([EXCEPTION_IMPORT]); + // Named, so the import states which operation it is the exception for. + expect(source).toContain(`import { ${EXCEPTION_OPERATION} } from "${EXCEPTION_IMPORT}";`); + + // Called once in the whole module. A bare identifier rather than any + // mention of the name: the module's own prose says `Git.revParse()`, and a + // sentence about the exception is not a second use of it. + const calls = [...source.matchAll(/(? { + const bytes = new TextEncoder().encode(FIXTURE_BUNDLE_SOURCE); + const sources = [ + { + path: FIXTURE_BUNDLE_PATH, + sourceHash: yield* sourceContentHash(bytes), + byteLength: bytes.byteLength, + }, + ]; + const bundleHash = yield* sourceBundleHash({ entrypoint: FIXTURE_BUNDLE_PATH, sources }); + const parsed = parseSourceBundleDefinition({ + version: 2, + kind: "source-bundle", + hashAlgorithm: "sha256", + bundleHash, + entrypoint: FIXTURE_BUNDLE_PATH, + sources, + }); + if (!parsed.ok) { + throw parsed.error; + } + + const base = richArtifact(); + const { runId, props, status, stopReason, createdAt, updatedAt } = base.run; + return { + ...base, + // Rebuilt member by member rather than spread past a deleted `base`: a + // version-2 record has no such member, and writing it out is what keeps the + // fixture the shape the contract declares. + run: { + runId, + definition: parsed.value, + props, + status, + ...(stopReason === undefined ? {} : { stopReason }), + createdAt, + updatedAt, + }, + definition: { + definitionVersion: 2, + definition: parsed.value, + sources: [{ path: FIXTURE_BUNDLE_PATH, bytes }], + }, + }; +} diff --git a/packages/workflow/tests/support/composition.ts b/packages/workflow/tests/support/composition.ts index 94fd2a9cc..80ea13d6d 100644 --- a/packages/workflow/tests/support/composition.ts +++ b/packages/workflow/tests/support/composition.ts @@ -50,6 +50,7 @@ import { } from "../../src/deno/composition/materialize.ts"; import type { StoredRepository } from "../../src/deno/workspace/repositories.ts"; import type { WorktreeRecord } from "../../src/composition/records.ts"; +import { isGitWorkflowRunRecord, type WorkflowRun, type WorkflowRunRecord } from "../../mod.ts"; /** What one execution did, at the boundaries a claim can be made about. */ export interface CompositionCounters { @@ -187,11 +188,7 @@ export function runWorkflowDocument( return yield* around(function* () { return yield* collect( yield* executeInstalled({ ...inlineSource(source), stream: database.journal }, [ - retainedWorkflowInstallation({ - runId: database.record.runId, - base: database.record.base, - pinnedCommit: database.record.definition.objectId, - }), + retainedWorkflowInstallation(retainedRunValue(database.record)), ]), ); }); @@ -758,3 +755,27 @@ export function* writeCheckoutFile( throw written.error; } } + +/** + * The retained run value a record installs under, in its own version's shape. + * + * A Git record installs the base and pinned commit it retains; a source-bundle + * record installs its bundle hash and exact target and invents neither. + */ +function retainedRunValue(record: WorkflowRunRecord): WorkflowRun { + if (isGitWorkflowRunRecord(record)) { + return { + runId: record.runId, + base: record.base, + pinnedCommit: record.definition.objectId, + }; + } + return { + runId: record.runId, + definitionVersion: 2, + bundleHash: record.definition.bundleHash, + ...(record.definition.targetPath === undefined + ? {} + : { targetPath: record.definition.targetPath }), + }; +} diff --git a/packages/workflow/tests/support/executor-holder.ts b/packages/workflow/tests/support/executor-holder.ts index ea6318170..832da48ca 100644 --- a/packages/workflow/tests/support/executor-holder.ts +++ b/packages/workflow/tests/support/executor-holder.ts @@ -10,6 +10,7 @@ import { main } from "effection"; import process from "node:process"; import { WorkflowLifecycle } from "../../mod.ts"; import { useWorkflowLifecycle } from "../../deno.ts"; +import { legacySourceReader } from "./legacy-source.ts"; await main(function* () { // `process.argv` rather than `Deno.args`: this file is Deno-only to run, and @@ -19,7 +20,7 @@ await main(function* () { throw new Error("usage: executor-holder.ts "); } - yield* useWorkflowLifecycle({ root }); + yield* useWorkflowLifecycle({ root, legacySource: legacySourceReader() }); const acquired = yield* WorkflowLifecycle.operations.acquireExecutor(runId); if (!acquired.ok) { throw acquired.error; diff --git a/packages/workflow/tests/support/legacy-source.ts b/packages/workflow/tests/support/legacy-source.ts new file mode 100644 index 000000000..b98598ca0 --- /dev/null +++ b/packages/workflow/tests/support/legacy-source.ts @@ -0,0 +1,73 @@ +/** + * The legacy source reader a test host installs when it executes a v1 run. + * + * A version-1 definition names a Git object and a path inside it, so the bytes + * behind it come from a repository the retained lifecycle does not reach. A + * host that executes one supplies this capability directly; these suites are + * such hosts, and this is the smallest honest one. + * + * It answers about whatever descriptor it is asked, deriving the root's blob + * identity from the bytes it returns — which is what Workflow recomputes on the + * way in. A declared component is answered with content whose identity really + * is the one the descriptor names, so a fixture that pins component hashes must + * register the bytes behind them. + */ + +import { Ok, type Operation, type Result } from "effection"; +import { gitBlobIdentity } from "../../deno.ts"; +import type { LegacyWorkflowSourceReader, RetainedDefinitionSources } from "../../deno.ts"; +import type { GitWorkflowDefinitionV1 } from "../../mod.ts"; + +/** The document a reader answers with when a fixture names no other. */ +export const LEGACY_ROOT_DOCUMENT = "# Release\n\nthis document is the run's retained source\n"; + +/** The bytes one fixture wants behind a declared component, by its blob id. */ +export type LegacyComponentSources = ReadonlyMap; + +/** + * A reader that returns this run's source, derived from its own descriptor. + * + * `components` maps a declared `sourceHash` to the bytes behind it. A fixture + * whose definition declares no components needs none: the root is the whole + * closure, and its identity is computed from the content returned rather than + * pinned by the descriptor. + */ +export function legacySourceReader( + root: string = LEGACY_ROOT_DOCUMENT, + components: LegacyComponentSources = new Map(), +): LegacyWorkflowSourceReader { + // deno-lint-ignore require-yield + return function* ( + definition: GitWorkflowDefinitionV1, + ): Operation> { + return Ok({ + definitionVersion: 1, + definition, + closure: { + root: { + objectFormat: definition.objectFormat, + pinnedCommit: definition.objectId, + rootDocumentPath: definition.rootDocumentPath, + ...(definition.targetPath === undefined ? {} : { targetPath: definition.targetPath }), + blobId: gitBlobIdentity(root, definition.objectFormat), + content: root, + }, + components: (definition.components ?? []).map((component) => { + const content = components.get(component.sourceHash); + if (content === undefined) { + throw new Error( + `this fixture declares the component "${component.name}" and registered no bytes ` + + "for the object id it pins, so no closure can satisfy it", + ); + } + return { + name: component.name, + path: component.path, + blobId: component.sourceHash, + content, + }; + }), + }, + }); + }; +} diff --git a/packages/workflow/tests/support/restart-child.ts b/packages/workflow/tests/support/restart-child.ts index 54fc21ca7..5037f774f 100644 --- a/packages/workflow/tests/support/restart-child.ts +++ b/packages/workflow/tests/support/restart-child.ts @@ -31,6 +31,7 @@ import type { Workflow } from "@executablemd/durable-streams"; import { main, until } from "effection"; import { WorkflowLifecycle, WorkflowStorageError } from "../../mod.ts"; import { useWorkflowRunHost } from "../../deno.ts"; +import { legacySourceReader } from "./legacy-source.ts"; const DEFINITION = { version: 1, @@ -71,7 +72,7 @@ main(function* () { // The whole host, because beginning a run is a lifecycle transition and the // executor lock is what authorizes it — here exactly as in production. - const transitions = yield* useWorkflowRunHost({ root }); + const transitions = yield* useWorkflowRunHost({ root, legacySource: legacySourceReader() }); const acquired = yield* WorkflowLifecycle.operations.acquireExecutor(runId); if (!acquired.ok) { diff --git a/packages/workflow/tests/support/storage.ts b/packages/workflow/tests/support/storage.ts index 92d1c6d63..e6d0fbe87 100644 --- a/packages/workflow/tests/support/storage.ts +++ b/packages/workflow/tests/support/storage.ts @@ -27,10 +27,10 @@ import { type WorkflowStopReason, } from "../../mod.ts"; import type { + GitWorkflowRunCreationV1, WorkflowBeginRequest, WorkflowExecutionTransitions, WorkflowExecutionBegun, - WorkflowRunCreation, } from "../../deno.ts"; import { installWorkflowLifecycle } from "../../src/deno/lifecycle.ts"; import { workflowRunPath } from "../../deno.ts"; @@ -38,6 +38,11 @@ import { useWorkflowRunConnections } from "../../src/deno/connections.ts"; import { SavepointObservation } from "../../src/deno/savepoints.ts"; import { installWorkflowRunStorage } from "../../src/deno/provider.ts"; import type { PrivateWorkspaceOptions } from "../../src/deno/workspace/private.ts"; +import { isGitWorkflowDefinition } from "../../mod.ts"; +import { legacySourceReader } from "./legacy-source.ts"; +import { parseSourceBundleDefinition, sourceBundleHash, sourceContentHash } from "../../mod.ts"; +import type { SourceBundleWorkflowRunCreationV2 } from "../../deno.ts"; +import type { Json } from "@executablemd/durable-streams"; export const SHA1 = "9fceb02d0ae598e95dc970b74767f19372d61af8"; @@ -60,6 +65,11 @@ export function definition( if (!result.ok) { throw result.error; } + // Narrowed rather than asserted: the parser answers with either version, and + // these fixtures describe the Git one. + if (!isGitWorkflowDefinition(result.value)) { + throw new Error("expected a Git workflow definition"); + } return result.value; } @@ -222,7 +232,10 @@ export function withRunHost( return scoped(function* () { const connections = yield* useWorkflowRunConnections(yield* SavepointObservation.get()); yield* installWorkflowRunStorage({ root }, internal, connections); - const transitions = yield* installWorkflowLifecycle({ root }, connections); + const transitions = yield* installWorkflowLifecycle( + { root, legacySource: legacySourceReader() }, + connections, + ); return yield* body(transitions); }); } @@ -258,8 +271,10 @@ export function withExecutorRun( }); } -/** The creation a `start` supplies, for a fixture that does not care which. */ -export function creation(overrides: Partial = {}): WorkflowRunCreation { +/** The Git creation a `start` supplies, for a fixture that does not care which. */ +export function creation( + overrides: Partial = {}, +): GitWorkflowRunCreationV1 { return { definition: definition(), base: "main", props: { channel: "stable" }, ...overrides }; } @@ -309,3 +324,87 @@ export function withBegunRun( ); }); } + +/** The Markdown a source-bundle fixture retains, and its logical path. */ +export const BUNDLE_ENTRYPOINT = "release.md"; +export const BUNDLE_SOURCE = "# Release\n\nthis run retains these exact bytes\n"; + +/** + * A complete source-bundle creation, descriptor and bytes together. + * + * Built the way a host builds one: the source hashes come from the bytes, and + * the bundle hash from the manifest those hashes make — so the descriptor + * describes itself before anything is asked to retain it. + */ +export function* sourceBundleCreation( + options: { + readonly content?: string; + readonly targetPath?: string; + readonly props?: { [key: string]: Json }; + } = {}, +): Operation { + const text = options.content ?? BUNDLE_SOURCE; + const bytes = new TextEncoder().encode(text); + const sources = [ + { + path: BUNDLE_ENTRYPOINT, + sourceHash: yield* sourceContentHash(bytes), + byteLength: bytes.byteLength, + }, + ]; + const bundleHash = yield* sourceBundleHash({ entrypoint: BUNDLE_ENTRYPOINT, sources }); + const parsed = parseSourceBundleDefinition({ + version: 2, + kind: "source-bundle", + hashAlgorithm: "sha256", + bundleHash, + entrypoint: BUNDLE_ENTRYPOINT, + sources, + ...(options.targetPath === undefined ? {} : { targetPath: options.targetPath }), + }); + if (!parsed.ok) { + throw parsed.error; + } + return { + definition: parsed.value, + sourceSnapshot: [{ path: BUNDLE_ENTRYPOINT, bytes }], + props: options.props ?? { channel: "stable" }, + }; +} + +/** + * One executor lock, held for the body and released with it. + * + * `withExecutorRun` begins an execution and raises a refusal; a case whose + * subject *is* the refusal needs the lock without the begin, so it can look at + * the answer rather than at an exception. + */ +export function withExecutor( + runId: string, + body: (executorLock: ExecutorLock) => Operation, +): Operation { + return scoped(function* () { + const acquisition = yield* WorkflowLifecycle.operations.acquireExecutor(runId); + if (!acquisition.ok) { + throw acquisition.error; + } + if (acquisition.value.kind !== "acquired") { + throw new Error(`the run ${runId} already has a live workflow executor`); + } + return yield* body(acquisition.value.lock); + }); +} + +/** + * One stored byte column, checked rather than coerced. + * + * A row is whatever SQLite handed back, so a column that is not bytes is a + * failure about the row rather than a `TextDecoder` throwing somewhere else. + */ +export function storedBytes(row: Record | undefined, column: string): Uint8Array { + const value = row?.[column]; + if (!(value instanceof Uint8Array)) { + throw new Error(`the row carries no ${column}`); + } + return value; +} diff --git a/packages/workflow/tests/workflow-definition.test.ts b/packages/workflow/tests/workflow-definition.test.ts index a8322a391..15751966a 100644 --- a/packages/workflow/tests/workflow-definition.test.ts +++ b/packages/workflow/tests/workflow-definition.test.ts @@ -13,6 +13,7 @@ import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; +import type { Result } from "effection"; import { isCanonicalDocumentTarget } from "@executablemd/core"; import type { Json } from "@executablemd/durable-streams"; import { @@ -22,6 +23,8 @@ import { definitionComponents, definitionToJson, type GitWorkflowDefinitionV1, + type GitWorkflowRunRecordV1, + isGitWorkflowDefinition, parseSourceBundleDefinition, parseStopReasonInput, parseWorkflowDefinition, @@ -35,7 +38,6 @@ import { WORKFLOW_RUN_STATUSES, WorkflowDefinitionError, WorkflowRequestError, - type WorkflowRunRecord, type WorkflowDefinition, WorkflowRunStorage, WorkflowStorageProviderError, @@ -58,10 +60,23 @@ function definition(overrides: Record = {}): Record = {}): GitWorkflowDefinitionV1 { - const result = parseWorkflowDefinition(definition(overrides)); + return git(parseWorkflowDefinition(definition(overrides))); +} + +/** + * The Git descriptor a result holds, narrowed rather than asserted. + * + * `parseWorkflowDefinition` now answers with either version, and these cases + * are about the Git one: a fixture that parsed as a source bundle would be a + * fixture this suite is not describing. + */ +function git(result: Result): GitWorkflowDefinitionV1 { if (!result.ok) { throw result.error; } + if (!isGitWorkflowDefinition(result.value)) { + throw new Error("expected a Git workflow definition"); + } return result.value; } @@ -84,11 +99,7 @@ function bundled(overrides: Record = {}): Record = {}): GitWorkflowDefinitionV1 { - const result = parseWorkflowDefinition(bundled(overrides)); - if (!result.ok) { - throw result.error; - } - return result.value; + return git(parseWorkflowDefinition(bundled(overrides))); } function refusal(value: unknown): WorkflowDefinitionError { @@ -102,7 +113,7 @@ function refusal(value: unknown): WorkflowDefinitionError { return result.error; } -function record(overrides: Partial = {}): WorkflowRunRecord { +function record(overrides: Partial = {}): GitWorkflowRunRecordV1 { return { runId: "release-1.4", definition: parsed(), @@ -468,7 +479,7 @@ describe("Tier WD — a definition's exact document target", () => { const section = record({ definition: parsed({ targetPath: "Release/Publish" }) }); const other = record({ definition: parsed({ targetPath: "Release/Announce" }) }); - const asking = (stored: WorkflowRunRecord, definition: WorkflowDefinition) => + const asking = (stored: GitWorkflowRunRecordV1, definition: GitWorkflowDefinitionV1) => conflictingFields(stored, { runId: stored.runId, definition, @@ -607,8 +618,8 @@ describe("Tier WD — the component bundle a definition is closed over", () => { const again = parseWorkflowDefinition(retained); expect(again.ok).toBe(true); - expect(again.ok && definitionComponents(again.value)).toEqual([]); - expect(again.ok && definitionToJson(again.value)).toEqual(retained); + expect(definitionComponents(git(again))).toEqual([]); + expect(definitionToJson(git(again))).toEqual(retained); // Presence is the member being written at all: a descriptor that wrote it // and named no bundle asked for one and failed to say which. @@ -637,7 +648,7 @@ describe("Tier WD — the component bundle a definition is closed over", () => { describe("Tier WD — a bundle decides compatible reuse", () => { const stored = record({ definition: parsedBundle() }); - const asking = (definition: WorkflowDefinition) => + const asking = (definition: GitWorkflowDefinitionV1) => conflictingFields(stored, { runId: stored.runId, definition, diff --git a/packages/workflow/tests/workflow-export.test.ts b/packages/workflow/tests/workflow-export.test.ts index e327b27e5..e9cc90f4c 100644 --- a/packages/workflow/tests/workflow-export.test.ts +++ b/packages/workflow/tests/workflow-export.test.ts @@ -28,8 +28,7 @@ import { Err, Ok, scoped } from "effection"; import type { Result } from "effection"; import type { DurableEvent } from "@executablemd/durable-streams"; import { WorkflowLifecycle } from "../mod.ts"; -import type { WorkflowDefinition, WorkflowRunDatabase } from "../mod.ts"; -import type { WorkflowDefinitionSourceReader } from "../src/deno/artifact/source.ts"; +import type { WorkflowRunDatabase } from "../mod.ts"; import { installWorkflowLifecycle } from "../src/deno/lifecycle.ts"; import type { WorkflowExecutionTransitions } from "../deno.ts"; import { installWorkflowRunStorage } from "../src/deno/provider.ts"; @@ -43,6 +42,11 @@ import { gitBlobId } from "./support/artifact-fixture.ts"; import { runDocument } from "./support/composition.ts"; import { git, gitOutput, useBareRemote } from "./support/git-remotes.ts"; import { creation, definition, SHA1, useStorageRoot, withExecutorRun } from "./support/storage.ts"; +import type { GitDefinitionSourceClosureV1, RetainedDefinitionSources } from "../deno.ts"; +import type { LegacyWorkflowSourceReader } from "../deno.ts"; +import type { GitWorkflowDefinitionV1 } from "../mod.ts"; +import { DatabaseSync } from "node:sqlite"; +import { BUNDLE_ENTRYPOINT, BUNDLE_SOURCE, sourceBundleCreation } from "./support/storage.ts"; const ROOT_DOCUMENT = "# Release\n\nnothing to see here\n"; @@ -78,8 +82,24 @@ function* exportRun(runId: string, stagingPath: string, forged?: XmdArtifactDefi /** The reader a host installs: it returns this run's real retained source. */ // deno-lint-ignore require-yield -function* honestSource(): Operation> { - return Ok(closure()); +function* honestSource( + retained: GitWorkflowDefinitionV1, +): Operation> { + return Ok(legacy(retained, closure())); +} + +/** + * One Git closure, as the versioned answer a legacy reader gives. + * + * The reader answers with the retained-source union now, so a fixture says + * which member it is producing. What Workflow does with it is unchanged: the + * closure is still held to the descriptor it was asked about. + */ +function legacy( + definition: GitWorkflowDefinitionV1, + closure: GitDefinitionSourceClosureV1, +): RetainedDefinitionSources { + return { definitionVersion: 1, definition, closure }; } /** @@ -91,14 +111,14 @@ function* honestSource(): Operation> { */ function withExportHost( root: string, - source: WorkflowDefinitionSourceReader | undefined, + source: LegacyWorkflowSourceReader | undefined, body: (transitions: WorkflowExecutionTransitions) => Operation, ): Operation { return scoped(function* () { const connections = yield* useWorkflowRunConnections(yield* SavepointObservation.get()); yield* installWorkflowRunStorage({ root }, {}, connections); const transitions = yield* installWorkflowLifecycle( - { root, ...(source === undefined ? {} : { definitionSource: source }) }, + { root, ...(source === undefined ? {} : { legacySource: source }) }, connections, ); return yield* body(transitions); @@ -178,7 +198,7 @@ describe("exporting a workflow run", () => { expect(opened.value.identity).toBe(sealed.value.identity); expect(opened.value.run.runId).toBe("release-1.4"); expect(opened.value.run.status).toBe("completed"); - expect(opened.value.definition.root.content).toBe(ROOT_DOCUMENT); + expect(gitClosure(opened.value.definition).root.content).toBe(ROOT_DOCUMENT); expect(opened.value.frontier).toEqual(sealed.value.frontier); // The run is still there, still readable, and still says what it said. @@ -197,15 +217,22 @@ describe("exporting a workflow run", () => { const other = closure(); // deno-lint-ignore require-yield - const wrongRun: WorkflowDefinitionSourceReader = function* () { - return Ok({ ...other, root: { ...other.root, rootDocumentPath: "workflows/other.md" } }); + const wrongRun: LegacyWorkflowSourceReader = function* (retained) { + return Ok( + legacy(retained, { + ...other, + root: { ...other.root, rootDocumentPath: "workflows/other.md" }, + }), + ); }; const refused = yield* withExportHost(root, wrongRun, function* () { return yield* exportRun("release-1.4", target); }); expect(refused.ok).toBe(false); - expect(refused.ok ? "" : refused.error.name).toBe("WorkflowRequestError"); + // Its own category now: a reader that answered about some other definition + // is a mismatched response rather than a malformed request. + expect(refused.ok ? "" : refused.error.name).toBe("LegacyWorkflowSourceMismatchError"); expect(yield* exists(target)).toBe(false); }); @@ -233,8 +260,8 @@ describe("exporting a workflow run", () => { throw opened.error; } // The request reached nothing: what was sealed is what the host read. - expect(opened.value.definition.root.content).toBe(ROOT_DOCUMENT); - expect(opened.value.definition.root.content).not.toBe(lie); + expect(gitClosure(opened.value.definition).root.content).toBe(ROOT_DOCUMENT); + expect(gitClosure(opened.value.definition).root.content).not.toBe(lie); }); it("XE5 refuses to export at all when the host installed no reader", function* () { @@ -365,9 +392,9 @@ function useDefinitionRepository(source: string): Operation { ); // --- The definition, and the component nothing expanded --------------- - expect(artifact.definition.root).toEqual({ + expect(gitClosure(artifact.definition).root).toEqual({ objectFormat: "sha1", pinnedCommit: repository.commit, rootDocumentPath: "flows/rich.md", @@ -688,8 +719,8 @@ describe("exporting a run that really ran", () => { content: source, }); // The bytes that were sealed are the bytes that executed. - expect(artifact.definition.root.content).toBe(source); - expect(artifact.definition.components).toEqual([ + expect(gitClosure(artifact.definition).root.content).toBe(source); + expect(gitClosure(artifact.definition).components).toEqual([ { name: "Unused", path: "flows/Unused.md", @@ -699,3 +730,95 @@ describe("exporting a run that really ran", () => { ]); }); }); + +/** + * The Git closure an artifact carries, narrowed rather than asserted. + * + * The retained source is a closed union now, so a format-1 case says which + * member it is describing: an artifact that came back carrying a source bundle + * is not the one these cases are about. + */ +function gitClosure(sources: RetainedDefinitionSources): GitDefinitionSourceClosureV1 { + if (sources.definitionVersion !== 1) { + throw new Error("expected a Git definition source closure"); + } + return sources.closure; +} + +/** + * A source-bundle run seals format 2, and reads back as itself. + * + * The physical container is unchanged — same application id, same tables, same + * `user_version` — and what moves is the semantic format inside it: its own + * header version, its own manifest version, its own identity domain, and a + * closed inventory carrying source pairs instead of a Git closure. + */ +describe("exporting a source-bundle workflow run", () => { + it("XE40: seals format 2 inside the unchanged container", function* () { + const root = yield* useStorageRoot(); + const creation = yield* sourceBundleCreation(); + const target = join(root, "bundle.xmd"); + + // No legacy reader at all: a version-2 run's source is its own store's, and + // an export that needed a repository would be reaching for one it never had. + yield* scoped(function* () { + const connections = yield* useWorkflowRunConnections(yield* SavepointObservation.get()); + yield* installWorkflowRunStorage({ root }, {}, connections); + const transitions = yield* installWorkflowLifecycle({ root }, connections); + yield* withExecutorRun( + transitions, + { runId: "bundle-export", action: "start", creation }, + function* (begun, executorLock) { + const settled = yield* transitions.settle(executorLock, { + executionId: begun.execution.executionId, + status: "completed", + }); + if (!settled.ok) { + throw settled.error; + } + }, + ); + const sealed = yield* WorkflowLifecycle.operations.export({ + runId: "bundle-export", + stagingPath: target, + }); + if (!sealed.ok) { + throw sealed.error; + } + }); + + // The container is version 1 and the semantic format is 2. + const header = ((): Record => { + const database = new DatabaseSync(target, { readOnly: true }); + try { + const row = database + .prepare("SELECT artifact_version, container_version FROM xmd_artifact_header") + .get(); + const version = database.prepare("PRAGMA user_version").get()?.["user_version"]; + return { ...row, userVersion: version }; + } finally { + database.close(); + } + })(); + expect(header["artifact_version"]).toBe(2); + expect(header["container_version"]).toBe(1); + expect(header["userVersion"]).toBe(1); + + const opened = yield* readXmdArtifact(target); + if (!opened.ok) { + throw opened.error; + } + const sources = opened.value.definition; + expect(sources.definitionVersion).toBe(2); + if (sources.definitionVersion !== 2) { + throw new Error("expected a source bundle"); + } + expect(sources.sources.map((source) => source.path)).toEqual([BUNDLE_ENTRYPOINT]); + expect(new TextDecoder().decode(sources.sources[0]?.bytes)).toBe(BUNDLE_SOURCE); + expect(sources.definition.bundleHash).toBe(creation.definition.bundleHash); + + // The run entry carries no Git fields at all. + expect(opened.value.run.definition.kind).toBe("source-bundle"); + expect("base" in opened.value.run).toBe(false); + }); +}); diff --git a/packages/workflow/tests/workflow-fork.test.ts b/packages/workflow/tests/workflow-fork.test.ts index 50c8f4457..4fec2992c 100644 --- a/packages/workflow/tests/workflow-fork.test.ts +++ b/packages/workflow/tests/workflow-fork.test.ts @@ -25,6 +25,37 @@ import { selectForkPrefix, } from "@executablemd/workflow"; import type { ForkCandidate } from "@executablemd/workflow"; +import { scoped } from "effection"; +import type { Operation } from "effection"; +import { + forkRunRecordEvent, + LegacyWorkflowSourceReaderUnavailableError, + WorkflowLifecycle, + WorkflowRequestError, +} from "@executablemd/workflow"; +import type { ExecutorLock } from "@executablemd/workflow"; +import type { WorkflowExecutionBegun, WorkflowExecutionTransitions } from "../deno.ts"; +import { useWorkflowRunConnections } from "../src/deno/connections.ts"; +import { SavepointObservation } from "../src/deno/savepoints.ts"; +import { installWorkflowRunStorage } from "../src/deno/provider.ts"; +import { installWorkflowLifecycle } from "../src/deno/lifecycle.ts"; +import { legacySourceReader } from "./support/legacy-source.ts"; +import { + BUNDLE_ENTRYPOINT, + BUNDLE_SOURCE, + creation, + runPath, + SHA1, + sourceBundleCreation, + storedBytes, + tamper, + useStorageRoot, + withExecutor, + withExecutorRun, + withRunHost, +} from "./support/storage.ts"; +import { DatabaseSync } from "node:sqlite"; +import { workflowForkStaging } from "../deno.ts"; const ROOT_A = "a".repeat(64); const ROOT_B = "b".repeat(64); @@ -257,3 +288,276 @@ describe("Tier WFK — forkability and fork selection", () => { } }); }); + +/** + * Tier WFK — admitting a fork of a retained source bundle. + * + * A fork's candidate is its own definition, so a version-2 fork is created from + * its own exact bytes exactly as a version-2 start is: the snapshot is copied + * and held to the descriptor before the destination exists, and what the fork + * retains afterwards is the store's copy rather than the caller's array. + * + * The same reader gate applies to every version-1 lifecycle admission. `begin`, + * `fork` and the private staging path each obtain and validate the Markdown a + * Git definition names before they write, so a host that cannot obtain it + * leaves no destination behind at all. + */ +describe("Tier WFK — a source-bundle fork", () => { + /** One settled source run, and the checkpoint a fork of it may select. */ + function* useForkSource( + root: string, + transitions: WorkflowExecutionTransitions, + ): Operation<{ checkpointEventId: string; rootImport: DurableEvent }> { + const creation = yield* sourceBundleCreation(); + return yield* withExecutorRun( + transitions, + { runId: "fork-source", action: "start", creation }, + function* (begun, executorLock) { + const rootImport: DurableEvent = { + type: "yield", + coroutineId: "root", + description: { type: "import_component", name: "__root__" }, + result: { status: "ok", value: { source: BUNDLE_SOURCE } }, + }; + yield* begun.database.journal.append( + forkRunRecordEvent({ + runId: "fork-source", + definitionVersion: 2, + bundleHash: creation.definition.bundleHash, + }), + ); + yield* begun.database.journal.append(rootImport); + yield* begun.database.journal.append(retained("checkpoint")); + + const entries = yield* begun.database.readJournalEntries(); + if (!entries.ok) { + throw entries.error; + } + const last = entries.value.at(-1); + if (last === undefined) { + throw new Error("the source run retained no checkpoint"); + } + const settled = yield* transitions.settle(executorLock, { + executionId: begun.execution.executionId, + status: "suspended", + }); + if (!settled.ok) { + throw settled.error; + } + void root; + return { checkpointEventId: last.eventId, rootImport }; + }, + ); + } + + it("WFK40: a fork is admitted from its own bytes, and retains its own copy", function* () { + const root = yield* useStorageRoot(); + const candidate = yield* sourceBundleCreation({ content: "# Forked\n\nits own bytes\n" }); + + yield* withRunHost(root, function* (transitions) { + const source = yield* useForkSource(root, transitions); + const forked = yield* withExecutor("fork-destination", function* (executorLock) { + return yield* transitions.fork(executorLock, { + runId: "fork-destination", + selection: { sourceRunId: "fork-source", checkpointEventId: source.checkpointEventId }, + creation: candidate, + rootImport: source.rootImport, + }); + }); + if (!forked.ok) { + throw forked.error; + } + + // The fork's own definition, and the closure it returns is the one its + // store now holds rather than the buffers the caller supplied. + expect(forked.value.record.definition.kind).toBe("source-bundle"); + const sources = forked.value.sources; + expect(sources.definitionVersion).toBe(2); + if (sources.definitionVersion !== 2) { + throw new Error("expected a source bundle"); + } + expect(new TextDecoder().decode(sources.sources[0]?.bytes)).toBe( + "# Forked\n\nits own bytes\n", + ); + expect(sources.definition.bundleHash).toBe(candidate.definition.bundleHash); + expect(sources.definition.bundleHash).not.toBe( + (yield* sourceBundleCreation()).definition.bundleHash, + ); + }); + + // Its schema is version 2, with the source store beside it. + tamper(runPath(root, "fork-destination"), (database) => { + expect(database.prepare("PRAGMA user_version").get()?.["user_version"]).toBe(2); + const stored = database.prepare("SELECT content FROM workflow_definition_blob").get(); + expect(new TextDecoder().decode(storedBytes(stored, "content"))).toBe( + "# Forked\n\nits own bytes\n", + ); + }); + }); + + it("WFK41: a candidate snapshot that is not its descriptor's leaves no fork", function* () { + const root = yield* useStorageRoot(); + const honest = yield* sourceBundleCreation(); + const lying = { + ...honest, + sourceSnapshot: [ + { path: BUNDLE_ENTRYPOINT, bytes: new TextEncoder().encode("# Something else\n") }, + ], + }; + + yield* withRunHost(root, function* (transitions) { + const source = yield* useForkSource(root, transitions); + const refused = yield* withExecutor("fork-lying", function* (executorLock) { + return yield* transitions.fork(executorLock, { + runId: "fork-lying", + selection: { sourceRunId: "fork-source", checkpointEventId: source.checkpointEventId }, + creation: lying, + rootImport: source.rootImport, + }); + }); + + expect(refused.ok).toBe(false); + expect(!refused.ok && refused.error).toBeInstanceOf(WorkflowRequestError); + const found = yield* WorkflowLifecycle.operations.inspect("fork-lying"); + expect(found.ok).toBe(false); + }); + }); + + it("WFK42: a staged fork retains the candidate's own bytes, discoverable by nobody", function* () { + const root = yield* useStorageRoot(); + const staging = "# Staged\n\nthe candidate's own bytes\n"; + const candidate = yield* sourceBundleCreation({ content: staging }); + + yield* withRunHost(root, function* (transitions) { + const source = yield* useForkSource(root, transitions); + const staged = yield* transitions.stageFork({ + runId: "fork-staged", + selection: { sourceRunId: "fork-source", checkpointEventId: source.checkpointEventId }, + creation: candidate, + rootImport: source.rootImport, + }); + if (!staged.ok) { + throw staged.error; + } + expect(staged.value.record.definition.kind).toBe("source-bundle"); + + // What it assembled, read out of the staging file while the resource that + // owns it is still alive. The kind alone would be satisfied by a staging + // copy that retained some other document; the bytes are what a + // compatibility replay would actually run. + const database = new DatabaseSync(workflowForkStaging(root, "fork-staged"), { + readOnly: true, + }); + try { + expect(database.prepare("PRAGMA user_version").get()?.["user_version"]).toBe(2); + const blob = database.prepare("SELECT content FROM workflow_definition_blob").get(); + expect(new TextDecoder().decode(storedBytes(blob, "content"))).toBe(staging); + + const manifest = database + .prepare("SELECT path FROM workflow_definition_source") + .all() + .map((row) => row["path"]); + expect(manifest).toEqual([BUNDLE_ENTRYPOINT]); + } finally { + database.close(); + } + + // And none of it is a run: staging assembles a Workspace to replay + // against, not a destination a host would find. + const found = yield* WorkflowLifecycle.operations.inspect("fork-staged"); + expect(found.ok).toBe(false); + }); + }); + + it("WFK43: every v1 admission is reader-gated, and leaves no destination", function* () { + const root = yield* useStorageRoot(); + + yield* scoped(function* () { + const connections = yield* useWorkflowRunConnections(yield* SavepointObservation.get()); + yield* installWorkflowRunStorage({ root }, {}, connections); + // A host with the reader, so a version-1 source run can exist to fork. + const capable = yield* installWorkflowLifecycle( + { root, legacySource: legacySourceReader() }, + connections, + ); + const source = yield* withExecutorRun( + capable, + { runId: "git-source", action: "start", creation: creation() }, + function* (begun, executorLock) { + const rootImport: DurableEvent = { + type: "yield", + coroutineId: "root", + description: { type: "import_component", name: "__root__" }, + result: { status: "ok", value: { source: "# Release\n" } }, + }; + yield* begun.database.journal.append( + forkRunRecordEvent({ runId: "git-source", base: "main", pinnedCommit: SHA1 }), + ); + yield* begun.database.journal.append(rootImport); + yield* begun.database.journal.append(retained("checkpoint")); + const entries = yield* begun.database.readJournalEntries(); + if (!entries.ok) { + throw entries.error; + } + const last = entries.value.at(-1); + if (last === undefined) { + throw new Error("the source run retained no checkpoint"); + } + const settled = yield* transitionsSettle(capable, executorLock, begun); + void settled; + return { checkpointEventId: last.eventId, rootImport }; + }, + ); + + // And a second host over the same storage with no reader at all. + const blind = yield* installWorkflowLifecycle({ root }, connections); + const request = { + selection: { sourceRunId: "git-source", checkpointEventId: source.checkpointEventId }, + creation: creation(), + rootImport: source.rootImport, + }; + + const resumed = yield* withExecutor("git-source", function* (executorLock) { + return yield* blind.begin(executorLock, { runId: "git-source", action: "resume" }); + }); + expect(resumed.ok).toBe(false); + expect(!resumed.ok && resumed.error).toBeInstanceOf( + LegacyWorkflowSourceReaderUnavailableError, + ); + + const forked = yield* withExecutor("git-fork", function* (executorLock) { + return yield* blind.fork(executorLock, { ...request, runId: "git-fork" }); + }); + expect(forked.ok).toBe(false); + expect(!forked.ok && forked.error).toBeInstanceOf(LegacyWorkflowSourceReaderUnavailableError); + + const staged = yield* blind.stageFork({ ...request, runId: "git-staged" }); + expect(staged.ok).toBe(false); + expect(!staged.ok && staged.error).toBeInstanceOf(LegacyWorkflowSourceReaderUnavailableError); + + // None of the three left a destination anything recognizes, and the + // source run is exactly as it was. + for (const runId of ["git-fork", "git-staged"]) { + const found = yield* WorkflowLifecycle.operations.inspect(runId); + expect({ runId, found: found.ok }).toEqual({ runId, found: false }); + } + const intact = yield* WorkflowLifecycle.operations.inspect("git-source"); + expect(intact.ok).toBe(true); + }); + }); +}); + +/** Settle one begun execution, so a source run stops before it is forked. */ +function* transitionsSettle( + transitions: WorkflowExecutionTransitions, + executorLock: ExecutorLock, + begun: WorkflowExecutionBegun, +): Operation { + const settled = yield* transitions.settle(executorLock, { + executionId: begun.execution.executionId, + status: "suspended", + }); + if (!settled.ok) { + throw settled.error; + } +} diff --git a/packages/workflow/tests/workflow-lifecycle-authority.test.ts b/packages/workflow/tests/workflow-lifecycle-authority.test.ts index a906cbf4a..d10ac4ffc 100644 --- a/packages/workflow/tests/workflow-lifecycle-authority.test.ts +++ b/packages/workflow/tests/workflow-lifecycle-authority.test.ts @@ -32,6 +32,27 @@ import { useStorageRoot, withRunHost, } from "./support/storage.ts"; +import { legacySourceReader } from "./support/legacy-source.ts"; +import { Ok } from "effection"; +import { + BUNDLE_ENTRYPOINT, + BUNDLE_SOURCE, + sourceBundleCreation, + storedBytes, + withExecutor, +} from "./support/storage.ts"; +import { useWorkflowRunConnections } from "../src/deno/connections.ts"; +import { SavepointObservation } from "../src/deno/savepoints.ts"; +import { installWorkflowRunStorage } from "../src/deno/provider.ts"; +import { installWorkflowLifecycle } from "../src/deno/lifecycle.ts"; +import { gitBlobIdentity } from "../deno.ts"; +import { + LegacyWorkflowSourceMismatchError, + WorkflowRunStorage, + LegacyWorkflowSourceReaderUnavailableError, + WorkflowRequestError, +} from "../mod.ts"; +import { withStorage } from "./support/storage.ts"; const { acquireExecutor } = WorkflowLifecycle.operations; @@ -39,7 +60,7 @@ const HOLDER = fileURLToPath(new URL("./support/executor-holder.ts", import.meta function withLifecycle(root: string, body: () => Operation): Operation { return scoped(function* () { - yield* useWorkflowLifecycle({ root }); + yield* useWorkflowLifecycle({ root, legacySource: legacySourceReader() }); return yield* body(); }); } @@ -420,3 +441,188 @@ function* holder(root: string, runId: string): Operation { } return result.stdout.trim(); } + +/** + * Tier WLA — creating a run from the bytes it retains. + * + * The transition is the only thing that can turn a source-bundle descriptor + * into storage, and what it retains is a copy it took before it checked + * anything. So these cases are about the two moments that decide what a run is + * a run of: what was copied, and what was proved before anything was written. + */ +describe("Tier WLA — a source-bundle creation", () => { + it("WLA40: begin answers with the source the store now holds", function* () { + const root = yield* useStorageRoot(); + const creation = yield* sourceBundleCreation(); + + yield* withRunHost(root, function* (transitions) { + yield* withExecutorRun( + transitions, + { runId: "bundle-begin", action: "start", creation }, + // deno-lint-ignore require-yield + function* (begun) { + const sources = begun.sources; + expect(sources.definitionVersion).toBe(2); + if (sources.definitionVersion !== 2) { + throw new Error("expected a source-bundle closure"); + } + expect(sources.sources.map((source) => source.path)).toEqual([BUNDLE_ENTRYPOINT]); + expect(new TextDecoder().decode(sources.sources[0]?.bytes)).toBe(BUNDLE_SOURCE); + expect(sources.definition.bundleHash).toBe(creation.definition.bundleHash); + }, + ); + }); + }); + + it("WLA41: what it retains is the descriptor's bytes, re-derived from the store", function* () { + const root = yield* useStorageRoot(); + const creation = yield* sourceBundleCreation(); + const mine = creation.sourceSnapshot[0]?.bytes; + + yield* withRunHost(root, function* (transitions) { + yield* withExecutorRun( + transitions, + { runId: "bundle-copy", action: "start", creation }, + // deno-lint-ignore require-yield + function* () { + if (mine === undefined) { + throw new Error("the fixture offered no bytes"); + } + mine[0] = 0x21; + }, + ); + }); + + // The caller's array really did change afterwards, and the run did not. + // + // This is the weaker of the two ownership claims: SQLite binds a parameter + // synchronously, so a transition that retained the caller's array rather + // than a copy would still store these bytes. What the copy protects is the + // window between verification and persistence, and the observable that + // discriminates it is `verifySourceBundleSnapshot`'s own answer — WD49. + expect(mine?.[0]).toBe(0x21); + tamper(runPath(root, "bundle-copy"), (database) => { + const stored = database.prepare("SELECT content FROM workflow_definition_blob").get(); + expect(new TextDecoder().decode(storedBytes(stored, "content"))).toBe(BUNDLE_SOURCE); + }); + + // And the run still reads back as the definition it claims, which is the + // check a drifted retained byte would fail. + const resumed = yield* withRunHost(root, function* (transitions) { + return yield* withExecutor("bundle-copy", function* (executorLock) { + return yield* transitions.begin(executorLock, { runId: "bundle-copy", action: "resume" }); + }); + }); + expect(resumed.ok).toBe(true); + }); + + it("WLA42: a snapshot that is not the descriptor's retains nothing", function* () { + const root = yield* useStorageRoot(); + const honest = yield* sourceBundleCreation(); + const lying = { + ...honest, + sourceSnapshot: [ + { path: BUNDLE_ENTRYPOINT, bytes: new TextEncoder().encode("# Not this document\n") }, + ], + }; + + const refused = yield* withRunHost(root, function* (transitions) { + return yield* withExecutor("bundle-lie", function* (executorLock) { + return yield* transitions.begin(executorLock, { + runId: "bundle-lie", + action: "start", + creation: lying, + }); + }); + }); + + expect(refused.ok).toBe(false); + expect(!refused.ok && refused.error).toBeInstanceOf(WorkflowRequestError); + // Nothing was retained, so nothing recognizes the id: opening a connection + // leaves an empty file behind, and an empty file is not a run. + expect(yield* undiscoverable(root, "bundle-lie")).toBe(true); + }); + + it("WLA43: a host with no legacy reader cannot begin a Git run", function* () { + const root = yield* useStorageRoot(); + + const refused = yield* scoped(function* () { + const connections = yield* useWorkflowRunConnections(yield* SavepointObservation.get()); + yield* installWorkflowRunStorage({ root }, {}, connections); + // Deliberately no `legacySource`: this host cannot obtain the Markdown a + // version-1 definition names, so it cannot say what such a run executes. + const transitions = yield* installWorkflowLifecycle({ root }, connections); + return yield* withExecutor("git-no-reader", function* (executorLock) { + return yield* transitions.begin(executorLock, { + runId: "git-no-reader", + action: "start", + creation: creation(), + }); + }); + }); + + expect(refused.ok).toBe(false); + expect(!refused.ok && refused.error).toBeInstanceOf(LegacyWorkflowSourceReaderUnavailableError); + // The check precedes the creation transaction, so nothing was written and + // the id is still one a later start can take. + expect(yield* undiscoverable(root, "git-no-reader")).toBe(true); + }); + + it("WLA44: a reader answering about another definition is a mismatch", function* () { + const root = yield* useStorageRoot(); + + const refused = yield* scoped(function* () { + const connections = yield* useWorkflowRunConnections(yield* SavepointObservation.get()); + yield* installWorkflowRunStorage({ root }, {}, connections); + const transitions = yield* installWorkflowLifecycle( + { + root, + // deno-lint-ignore require-yield + *legacySource(definition) { + return Ok({ + definitionVersion: 1, + definition, + closure: { + root: { + objectFormat: definition.objectFormat, + pinnedCommit: definition.objectId, + rootDocumentPath: "workflows/somewhere-else.md", + blobId: gitBlobIdentity("# Elsewhere\n", definition.objectFormat), + content: "# Elsewhere\n", + }, + components: [], + }, + }); + }, + }, + connections, + ); + return yield* withExecutor("git-mismatch", function* (executorLock) { + return yield* transitions.begin(executorLock, { + runId: "git-mismatch", + action: "start", + creation: creation(), + }); + }); + }); + + expect(refused.ok).toBe(false); + expect(!refused.ok && refused.error).toBeInstanceOf(LegacyWorkflowSourceMismatchError); + expect(yield* undiscoverable(root, "git-mismatch")).toBe(true); + }); +}); + +/** + * Whether this run id names nothing a host would find. + * + * Not "whether a file is there": opening a connection creates the file before + * any decision is made, so a refused creation routinely leaves an empty one + * behind. What the refusal has to preserve is that nothing recognizes the id, + * which is what a later start reusing it depends on. + */ +function* undiscoverable(root: string, runId: string): Operation { + return yield* withStorage(root, function* () { + const found = yield* WorkflowRunStorage.operations.lookup(runId); + return !found.ok; + }); +} diff --git a/packages/workflow/tests/workflow-lifecycle-control.test.ts b/packages/workflow/tests/workflow-lifecycle-control.test.ts index a20d634fe..65976062c 100644 --- a/packages/workflow/tests/workflow-lifecycle-control.test.ts +++ b/packages/workflow/tests/workflow-lifecycle-control.test.ts @@ -21,12 +21,13 @@ import { WorkflowLifecycle, WorkflowRunNotFoundError } from "../mod.ts"; import type { WorkflowRunRecord, WorkflowRunStatus } from "../mod.ts"; import { useWorkflowLifecycle, workflowRunLock, workflowRunPath } from "../deno.ts"; import { creation, useStorageRoot, withExecutorRun, withRunHost } from "./support/storage.ts"; +import { legacySourceReader } from "./support/legacy-source.ts"; const { cancel } = WorkflowLifecycle.operations; function withLifecycle(root: string, body: () => Operation): Operation { return scoped(function* () { - yield* useWorkflowLifecycle({ root }); + yield* useWorkflowLifecycle({ root, legacySource: legacySourceReader() }); return yield* body(); }); } diff --git a/packages/workflow/tests/workflow-run-journal.test.ts b/packages/workflow/tests/workflow-run-journal.test.ts index a66be76e8..19da1ba26 100644 --- a/packages/workflow/tests/workflow-run-journal.test.ts +++ b/packages/workflow/tests/workflow-run-journal.test.ts @@ -67,6 +67,7 @@ import { useStorageRoot, withStorage, } from "./support/storage.ts"; +import { describeWorkflowRun, readWorkflowRun, workflowRunValue } from "../src/journal.ts"; const { create } = WorkflowRunStorage.operations; @@ -1601,3 +1602,86 @@ describe("Tier WJ — surviving a process", () => { ]); }); }); + +/** + * Tier WJ — the run value a source-bundle journal records. + * + * The record is one closed union now. A reader accepts either member's exact + * set in any key order and no other, and the ordinary serializer writes one + * spelling per version — which is what keeps a retained version-1 record byte + * for byte what it always was while version 2 invents no Git field. + */ +describe("Tier WJ — the source-bundle run record", () => { + const BUNDLE_HASH = "a".repeat(64); + + it("WJ40: version 2 serializes its own members, in its own order", function* () { + const whole = workflowRunValue({ + runId: "bundle-1", + definitionVersion: 2, + bundleHash: BUNDLE_HASH, + }); + expect(Object.keys(whole)).toEqual(["runId", "definitionVersion", "bundleHash"]); + + const section = workflowRunValue({ + runId: "bundle-1", + definitionVersion: 2, + bundleHash: BUNDLE_HASH, + targetPath: "Release/Publish", + }); + expect(Object.keys(section)).toEqual([ + "runId", + "definitionVersion", + "bundleHash", + "targetPath", + ]); + + // Version 1 is untouched, in members and in order. + expect(Object.keys(workflowRunValue({ runId: "r", base: "main", pinnedCommit: "c" }))).toEqual([ + "runId", + "base", + "pinnedCommit", + ]); + }); + + it("WJ41: either exact member set reads, in any order, and nothing else", function* () { + // Reordered keys are the same value. + expect( + readWorkflowRun({ bundleHash: BUNDLE_HASH, definitionVersion: 2, runId: "bundle-1" }), + ).toEqual({ runId: "bundle-1", definitionVersion: 2, bundleHash: BUNDLE_HASH }); + + for (const refused of [ + // A member set that is neither version's. + { runId: "r", definitionVersion: 2 }, + { runId: "r", definitionVersion: 2, bundleHash: BUNDLE_HASH, base: "main" }, + // Half of each. + { runId: "r", base: "main", definitionVersion: 2 }, + // A synthetic version, and a synthetic target. + { runId: "r", definitionVersion: 1, bundleHash: BUNDLE_HASH }, + { runId: "r", definitionVersion: 2, bundleHash: BUNDLE_HASH, targetPath: 7 }, + { runId: "r", definitionVersion: 2, bundleHash: "" }, + ]) { + expect({ refused, run: readWorkflowRun(refused) }).toEqual({ refused, run: undefined }); + } + }); + + it("WJ42: the effect description says which version, and invents no base", function* () { + const described = describeWorkflowRun({ + runId: "bundle-1", + definitionVersion: 2, + bundleHash: BUNDLE_HASH, + }); + expect(described).toEqual({ + type: "workflow_run", + name: "workflow_run", + definitionVersion: 2, + bundleHash: BUNDLE_HASH, + }); + expect("base" in described).toBe(false); + + expect(describeWorkflowRun({ runId: "r", base: "main", pinnedCommit: "c" })).toEqual({ + type: "workflow_run", + name: "workflow_run", + base: "main", + }); + }); +}); diff --git a/packages/workflow/tests/workflow-run-storage.test.ts b/packages/workflow/tests/workflow-run-storage.test.ts index 6e53c7ae5..e315e189a 100644 --- a/packages/workflow/tests/workflow-run-storage.test.ts +++ b/packages/workflow/tests/workflow-run-storage.test.ts @@ -64,6 +64,22 @@ import { withBegunRun, withStorage, } from "./support/storage.ts"; +import { + type GitWorkflowRunRecordV1, + isGitWorkflowRunRecord, + type WorkflowRunRecord, +} from "../mod.ts"; +import { + BUNDLE_ENTRYPOINT, + BUNDLE_SOURCE, + sourceBundleCreation, + storedBytes, + withExecutor, + withExecutorRun, + withRunHost, +} from "./support/storage.ts"; +import { WorkflowDefinitionCorruptError, WorkflowDefinitionSourceMissingError } from "../mod.ts"; +import { sourceBundleDefinitionToJson } from "../mod.ts"; const { create, lookup } = WorkflowRunStorage.operations; @@ -413,7 +429,7 @@ describe("Tier WS — creating and finding a run", () => { expect(record.runId).toBe("release-1.4"); expect(record.definition).toEqual(definition()); - expect(record.base).toBe("main"); + expect(gitRecord(record).base).toBe("main"); expect(record.props).toEqual({ channel: "stable" }); expect(record.status).toBe("running"); @@ -981,10 +997,12 @@ describe("Tier WS — refusing what is not this run's database", () => { const root = yield* useStorageRoot(); const path = runPath(root, "run-2"); + // Version 3: versions 1 and 2 are both ones this build implements, so a + // version it does not is the one that proves nothing is migrated. const result = yield* withStorage(root, function* () { yield* createRun({ runId: "run-2" }); tamper(path, (database) => { - database.exec("PRAGMA user_version = 2"); + database.exec("PRAGMA user_version = 3"); }); return yield* lookup("run-2"); }); @@ -993,7 +1011,7 @@ describe("Tier WS — refusing what is not this run's database", () => { expect(!result.ok && result.error).toBeInstanceOf(WorkflowSchemaVersionError); tamper(path, (database) => { - expect(database.prepare("PRAGMA user_version").get()?.["user_version"]).toBe(2); + expect(database.prepare("PRAGMA user_version").get()?.["user_version"]).toBe(3); expect(database.prepare("PRAGMA application_id").get()?.["application_id"]).toBe( APPLICATION_ID, ); @@ -1715,7 +1733,7 @@ describe("Tier WS — version 1 amended in place", () => { } }); - it("WS23e: there is no version 2 to migrate to", function* () { + it("WS23e: a version-1 inventory stamped version 2 is a hybrid, not a migration", function* () { const root = yield* useStorageRoot(); yield* withStorage(root, function* () { yield* createRun(); @@ -1731,8 +1749,13 @@ describe("Tier WS — version 1 amended in place", () => { return yield* lookup("release-1.4"); }); + // Version 2 is a version this build implements, and this file is not one: + // its `workflow_run` still has the base column version 1 declares, and its + // two definition-source tables were never created. A header claiming the + // other version over version 1's objects is the file disagreeing with + // itself, which is damage rather than a version to upgrade from. expect(result.ok).toBe(false); - expect(!result.ok && result.error).toBeInstanceOf(WorkflowSchemaVersionError); + expect(!result.ok && result.error).toBeInstanceOf(WorkflowDatabaseCorruptError); // Described and left exactly as found: nothing upgraded it, and nothing // downgraded it either. expect(yield* until(readFile(path))).toEqual(before); @@ -1839,3 +1862,178 @@ describe("Tier WS — version 1 amended in place", () => { expect(!result.ok && result.error).toBeInstanceOf(WorkflowDatabaseCorruptError); }); }); + +/** + * The Git record a lookup answered with, narrowed rather than asserted. + * + * The retained record is a closed union now, and the base these cases assert + * about is a member only the Git one has. + */ +function gitRecord(record: WorkflowRunRecord): GitWorkflowRunRecordV1 { + if (!isGitWorkflowRunRecord(record)) { + throw new Error("expected a Git workflow run record"); + } + return record; +} + +/** + * Tier WS — the source-bundle schema, and the bytes it retains. + * + * Version 2 is a second immutable inventory rather than an amendment of the + * first. So the questions here are what a version-2 file is required to hold, + * what it is required *not* to hold, and whether a reader hands anything back + * when what it holds no longer describes itself. + */ +describe("Tier WS — a source-bundle run's own schema", () => { + it("WS40: a version-2 run declares version 2 and no base column", function* () { + const root = yield* useStorageRoot(); + const creation = yield* sourceBundleCreation(); + + yield* withRunHost(root, function* (transitions) { + const begun = yield* withExecutorRun( + transitions, + { runId: "bundle-1", action: "start", creation }, + // deno-lint-ignore require-yield + function* (begun) { + return begun; + }, + ); + expect(begun.record.definition.kind).toBe("source-bundle"); + }); + + tamper(runPath(root, "bundle-1"), (database) => { + expect(database.prepare("PRAGMA user_version").get()?.["user_version"]).toBe(2); + + // A base column would be a repository state this run never had, and the + // two definition-source tables are what version 2 adds instead. + const columns = database + .prepare("SELECT name FROM pragma_table_info('workflow_run')") + .all() + .map((row) => row["name"]); + expect(columns).not.toContain("base"); + + const objects = database + .prepare("SELECT name FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'") + .all() + .map((row) => row["name"]); + expect(objects).toContain("workflow_definition_blob"); + expect(objects).toContain("workflow_definition_source"); + }); + }); + + it("WS41: it retains the exact bytes, keyed by their own hash", function* () { + const root = yield* useStorageRoot(); + const creation = yield* sourceBundleCreation(); + const entry = creation.definition.sources[0]; + + yield* withRunHost(root, function* (transitions) { + yield* withExecutorRun( + transitions, + { runId: "bundle-2", action: "start", creation }, + // deno-lint-ignore require-yield + function* () {}, + ); + }); + + tamper(runPath(root, "bundle-2"), (database) => { + const manifest = database + .prepare("SELECT path, source_hash FROM workflow_definition_source") + .all(); + expect(manifest).toEqual([{ path: BUNDLE_ENTRYPOINT, source_hash: entry?.sourceHash }]); + + const blob = database + .prepare("SELECT byte_length, content FROM workflow_definition_blob") + .get(); + expect(blob?.["byte_length"]).toBe(entry?.byteLength); + expect(new TextDecoder().decode(storedBytes(blob, "content"))).toBe(BUNDLE_SOURCE); + }); + }); + + it("WS42: public creation still admits only version 1", function* () { + const root = yield* useStorageRoot(); + const creation = yield* sourceBundleCreation(); + + const refused = yield* withStorage(root, function* () { + // Round-tripped through JSON rather than cast past the declared type. + // That is not a trick: it is the value a host that read its request from + // a file hands over, and the reason the provider parses the request it + // was given instead of trusting the signature it was called through. + const offered: CreateWorkflowRunRequest = JSON.parse( + JSON.stringify({ + runId: "bundle-3", + // The descriptor alone, which is exactly what this request cannot + // retain: its bytes were never supplied here. + definition: sourceBundleDefinitionToJson(creation.definition), + base: "main", + props: {}, + }), + ); + return yield* WorkflowRunStorage.operations.create(offered); + }); + + expect(refused.ok).toBe(false); + expect(!refused.ok && refused.error).toBeInstanceOf(WorkflowRequestError); + expect(yield* exists(runPath(root, "bundle-3"))).toBe(false); + }); + + it("WS43: a blob the store no longer holds is missing, not partial", function* () { + const root = yield* useStorageRoot(); + const creation = yield* sourceBundleCreation(); + yield* withRunHost(root, function* (transitions) { + yield* withExecutorRun( + transitions, + { runId: "bundle-4", action: "start", creation }, + // deno-lint-ignore require-yield + function* () {}, + ); + }); + + // The manifest row and its content together, so the store stays + // structurally consistent — no dangling reference, no unreferenced blob — + // and what is wrong is only that the descriptor names a source it no longer + // holds. A dangling reference would be caught as damage before any of this. + tamper(runPath(root, "bundle-4"), (database) => { + database.exec("DELETE FROM workflow_definition_source"); + database.exec("DELETE FROM workflow_definition_blob"); + }); + + const resumed = yield* withRunHost(root, function* (transitions) { + return yield* withExecutor("bundle-4", function* (executorLock) { + return yield* transitions.begin(executorLock, { runId: "bundle-4", action: "resume" }); + }); + }); + + expect(resumed.ok).toBe(false); + expect(!resumed.ok && resumed.error).toBeInstanceOf(WorkflowDefinitionSourceMissingError); + }); + + it("WS44: content that is no longer what it names is corrupt, not missing", function* () { + const root = yield* useStorageRoot(); + const creation = yield* sourceBundleCreation(); + yield* withRunHost(root, function* (transitions) { + yield* withExecutorRun( + transitions, + { runId: "bundle-5", action: "start", creation }, + // deno-lint-ignore require-yield + function* () {}, + ); + }); + + // The same length, different bytes: the row still satisfies every CHECK the + // table declares, and recomputing its hash is what catches it. + tamper(runPath(root, "bundle-5"), (database) => { + database + .prepare("UPDATE workflow_definition_blob SET content = ?") + .run(new TextEncoder().encode(BUNDLE_SOURCE.replace("Release", "Reverse"))); + }); + + const resumed = yield* withRunHost(root, function* (transitions) { + return yield* withExecutor("bundle-5", function* (executorLock) { + return yield* transitions.begin(executorLock, { runId: "bundle-5", action: "resume" }); + }); + }); + + expect(resumed.ok).toBe(false); + expect(!resumed.ok && resumed.error).toBeInstanceOf(WorkflowDefinitionCorruptError); + }); +}); diff --git a/packages/workflow/tests/workflow-run.test.ts b/packages/workflow/tests/workflow-run.test.ts index c1fe84456..4b011ad02 100644 --- a/packages/workflow/tests/workflow-run.test.ts +++ b/packages/workflow/tests/workflow-run.test.ts @@ -34,6 +34,7 @@ import type { ExecutionInstallation } from "@executablemd/core/host"; import { Git } from "../src/git.ts"; import { getWorkflowRun, workflowInstallation } from "../src/run.ts"; import type { WorkflowRun } from "../src/run.ts"; +import { type GitWorkflowRunV1, isGitWorkflowRun } from "../mod.ts"; const COMMIT = "9fceb02d0ae598e95dc970b74767f19372d61af8"; const OTHER_COMMIT = "1111111111111111111111111111111111111111"; @@ -182,11 +183,11 @@ describe("Tier WR — workflow runs", () => { yield* b; }); - expect(first[0]?.base).toBe("main"); - expect(first[0]?.pinnedCommit).toBe(COMMIT); - expect(second[0]?.base).toBe("release"); - expect(second[0]?.pinnedCommit).toBe(OTHER_COMMIT); - expect(first[0]?.runId).not.toBe(second[0]?.runId); + expect(gitRun(first[0]).base).toBe("main"); + expect(gitRun(first[0]).pinnedCommit).toBe(COMMIT); + expect(gitRun(second[0]).base).toBe("release"); + expect(gitRun(second[0]).pinnedCommit).toBe(OTHER_COMMIT); + expect(gitRun(first[0]).runId).not.toBe(gitRun(second[0]).runId); }); it("WR20: one installation value reused by two executions gives each its own run", function* () { @@ -388,8 +389,8 @@ describe("Tier WR — workflow runs", () => { }); expect(asked).toHaveLength(0); - expect(restored[0]?.pinnedCommit).toBe(COMMIT); - expect(restored[0]?.runId).toBe(live[0]?.runId); + expect(gitRun(restored[0]).pinnedCommit).toBe(COMMIT); + expect(gitRun(restored[0]).runId).toBe(live[0]?.runId); }); it("WR7: a failure to resolve the base records no run and expands no document", function* () { @@ -912,7 +913,7 @@ describe("Tier WR — workflow runs", () => { origin: "tier-wr", props: { type: "object", properties: {}, additionalProperties: false }, *fn() { - order.push(`expanded:${(yield* getWorkflowRun()).pinnedCommit === COMMIT}`); + order.push(`expanded:${gitRun(yield* getWorkflowRun()).pinnedCommit === COMMIT}`); return ""; }, }, @@ -1027,6 +1028,20 @@ describe("Tier WR — workflow runs", () => { yield* slow; }); - expect(seen.map((run) => run.base).sort()).toEqual(["fast", "slow"]); + expect(seen.map((run) => gitRun(run).base).sort()).toEqual(["fast", "slow"]); }); }); + +/** + * The Git run a value holds, narrowed rather than asserted. + * + * `WorkflowRun` is a closed union now, and these cases are about the Git + * member: a value that came back as a source-bundle run is not the one the + * assertion below is describing. + */ +function gitRun(run: WorkflowRun | undefined): GitWorkflowRunV1 { + if (run === undefined || !isGitWorkflowRun(run)) { + throw new Error("expected a Git workflow run"); + } + return run; +} diff --git a/packages/workflow/tests/xmd-artifact.test.ts b/packages/workflow/tests/xmd-artifact.test.ts index 1a28ea12b..573b380be 100644 --- a/packages/workflow/tests/xmd-artifact.test.ts +++ b/packages/workflow/tests/xmd-artifact.test.ts @@ -63,6 +63,12 @@ import { import { serializeDurableEvent } from "@executablemd/durable-streams"; import type { DurableEvent, Json } from "@executablemd/durable-streams"; import { SUSPENSION_ANSWER } from "../src/suspension/answer.ts"; +import type { GitDefinitionSourceClosureV1, RetainedDefinitionSources } from "../deno.ts"; +import { + FIXTURE_BUNDLE_PATH, + FIXTURE_BUNDLE_SOURCE, + sourceBundleArtifact, +} from "./support/artifact-fixture.ts"; const encoder = new TextEncoder(); @@ -402,9 +408,19 @@ function* resealed( encoding: encodingOf(textColumn(row, "encoding")), content: bytesColumn(row, "content"), })); - const built = buildXmdArtifactManifest(entries, (kind) => { - throw new Error(`two ${kind} records under one identity`); - }); + // Resealed under the format the header declares: the manifest version and + // the identity domain are the format's, so a format-2 file resealed as + // format 1 would be turned away by the identity comparison rather than by + // the gate a case is aiming at. + const format = database.prepare("SELECT artifact_version FROM xmd_artifact_header").get(); + const declared = format?.["artifact_version"] === 2 ? 2 : 1; + const built = buildXmdArtifactManifest( + entries, + (kind) => { + throw new Error(`two ${kind} records under one identity`); + }, + declared, + ); database .prepare("UPDATE xmd_artifact_header SET manifest = ?, identity = ? WHERE id = 1") .run(built.bytes, built.identity); @@ -600,7 +616,7 @@ describe("XMD artifact container version 1", () => { } expect(read.manifests.length).toBe(contents.manifests.length); // One component the run never expanded is still in the closure. - expect(read.definition.components.map((component) => component.name)).toEqual([ + expect(gitClosure(read.definition).components.map((component) => component.name)).toEqual([ "Checklist", "Unused", ]); @@ -664,7 +680,16 @@ describe("XMD artifact container version 1", () => { target.exec("PRAGMA user_version = 2"); }, ); + // Version 3: formats 1 and 2 are both ones this build implements, so a + // format it does not is what proves an unsupported one is left alone. const futureFormat = yield* damaged(path, join(directory, "future-format.xmd"), (target) => { + target.exec("UPDATE xmd_artifact_header SET artifact_version = 3"); + }); + // The neighbouring claim: a format this build *does* implement, over the + // other format's records. Its closed inventory is the one the header names, + // so a format-1 closure inside a format-2 header is content that format + // does not declare rather than a superset either verifier could complete. + const wrongFormat = yield* damaged(path, join(directory, "wrong-format.xmd"), (target) => { target.exec("UPDATE xmd_artifact_header SET artifact_version = 2"); }); const extraView = yield* damaged(path, join(directory, "extra-view.xmd"), (target) => { @@ -769,6 +794,7 @@ describe("XMD artifact container version 1", () => { [notADatabase, "XmdArtifactForeignContainerError"], [futureContainer, "XmdArtifactContainerVersionError"], [futureFormat, "XmdArtifactFormatVersionError"], + [wrongFormat, "XmdArtifactInventoryError"], [extraView, "XmdArtifactSchemaError"], [extraIndex, "XmdArtifactSchemaError"], [extraTrigger, "XmdArtifactSchemaError"], @@ -801,7 +827,7 @@ describe("XMD artifact container version 1", () => { path, join(directory, "future-format-and-schema.xmd"), (target) => { - target.exec("UPDATE xmd_artifact_header SET artifact_version = 2"); + target.exec("UPDATE xmd_artifact_header SET artifact_version = 3"); target.exec("CREATE VIEW later AS SELECT kind FROM xmd_artifact_content"); }, ); @@ -1501,7 +1527,7 @@ describe("XMD artifact version 1 Agent portability evidence", () => { expect(read.run.status).toBe(frozen.status); expect(read.journal.length).toBe(frozen.journal); expect(read.frontier.finalEventId).toBe(frozen.finalEventId); - expect(read.definition.root.content.length).toBeGreaterThan(0); + expect(gitClosure(read.definition).root.content.length).toBeGreaterThan(0); // No record, no token, no bundle and no marker is reconstructed for it. expect(read.agentEvidence).toBeUndefined(); expect(rowsOfKind(frozen.path, "agent-session-portability")).toBe(0); @@ -2046,3 +2072,125 @@ function walk( walk(member, visit, seen); } } + +/** + * The Git closure an artifact carries, narrowed rather than asserted. + * + * The retained source is a closed union now, so a format-1 case says which + * member it is describing: an artifact that came back carrying a source bundle + * is not the one these cases are about. + */ +function gitClosure(sources: RetainedDefinitionSources): GitDefinitionSourceClosureV1 { + if (sources.definitionVersion !== 1) { + throw new Error("expected a Git definition source closure"); + } + return sources.closure; +} + +/** + * A format-2 artifact's own statement about its content, held to the descriptor. + * + * Each `definition-source-entry` declares a source hash and a byte length + * beside the content it names. Those are read, so a forger who re-seals the + * file cannot move them: the descriptor is what the run retains, and an entry + * that disagrees with it describes a source this artifact does not hold. + */ +describe("XMD artifact format 2 definition sources", () => { + const ENTRY = "definition-source-entry"; + const IDENTITY = JSON.stringify(FIXTURE_BUNDLE_PATH); + + function* useSourceBundleArtifact(): Operation<{ directory: string; path: string }> { + const directory = yield* useArtifactDirectory(); + const { path } = yield* sealed(directory, "bundle.xmd", yield* sourceBundleArtifact()); + return { directory, path }; + } + + /** + * The entry row's canonical JSON, with one member replaced. + * + * Parsed back into the object shape the row holds rather than asserted into + * it: a row that is not an object is a fixture that has already stopped + * describing what this case is about. + */ + function forgedEntry(path: string, replace: (entry: JsonObject) => Json): string { + const parsed: unknown = JSON.parse(storedText(path, ENTRY, IDENTITY)); + const entry = parseJsonObject(parsed, "$", (reason) => new Error(reason)); + return canonicalJsonText(replace(entry)); + } + + it("F40: opens when its entries agree with the descriptor", function* () { + const { path } = yield* useSourceBundleArtifact(); + const artifact = yield* opened(path); + + expect(artifact.definition.definitionVersion).toBe(2); + if (artifact.definition.definitionVersion !== 2) { + throw new Error("expected a source bundle"); + } + expect(new TextDecoder().decode(artifact.definition.sources[0]?.bytes)).toBe( + FIXTURE_BUNDLE_SOURCE, + ); + }); + + it("F41: a resealed artifact cannot move a declared byte length", function* () { + const { directory, path } = yield* useSourceBundleArtifact(); + + // Re-sealed in full: the row's own length and digest are corrected, and the + // manifest and identity are rebuilt over what the damage left. Every gate + // before semantic recognition therefore passes, and what refuses the file + // is the entry disagreeing with the definition the run retains. + const forged = yield* resealed(path, join(directory, "forged-length.xmd"), (database) => { + rewrite( + database, + ENTRY, + IDENTITY, + forgedEntry(path, (entry) => ({ ...entry, byteLength: 1 })), + ); + }); + + const refused = yield* readXmdArtifact(forged); + expect(refused.ok).toBe(false); + expect(refused.ok ? "" : refused.error.name).toBe("XmdArtifactRecordError"); + expect(refused.ok ? "" : refused.error.message).toContain("declared byte length"); + }); + + it("F42: nor a declared source hash", function* () { + const { directory, path } = yield* useSourceBundleArtifact(); + + const forged = yield* resealed(path, join(directory, "forged-hash.xmd"), (database) => { + rewrite( + database, + ENTRY, + IDENTITY, + forgedEntry(path, (entry) => ({ ...entry, sourceHash: "b".repeat(64) })), + ); + }); + + const refused = yield* readXmdArtifact(forged); + expect(refused.ok).toBe(false); + expect(refused.ok ? "" : refused.error.name).toBe("XmdArtifactRecordError"); + expect(refused.ok ? "" : refused.error.message).toContain("declared source hash"); + }); + + it("F43: nor a path the descriptor does not retain", function* () { + const { directory, path } = yield* useSourceBundleArtifact(); + + const forged = yield* resealed(path, join(directory, "forged-path.xmd"), (database) => { + database + .prepare("UPDATE xmd_artifact_content SET identity = ? WHERE kind = ? AND identity = ?") + .run(JSON.stringify("elsewhere.md"), ENTRY, IDENTITY); + database + .prepare("UPDATE xmd_artifact_content SET identity = ? WHERE kind = ? AND identity = ?") + .run(JSON.stringify("elsewhere.md"), "definition-source-content", IDENTITY); + rewrite( + database, + ENTRY, + JSON.stringify("elsewhere.md"), + forgedEntry(path, (entry) => ({ ...entry, path: "elsewhere.md" })), + ); + }); + + const refused = yield* readXmdArtifact(forged); + expect(refused.ok).toBe(false); + expect(refused.ok ? "" : refused.error.name).toBe("XmdArtifactInventoryError"); + }); +}); From a0d35f4f5c786b54bef40152fa1e1d60a8456fbd Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Tue, 15 Sep 2026 16:27:57 -0400 Subject: [PATCH 3/8] =?UTF-8?q?=E2=9C=A8=20Start=20workflows=20from=20reta?= =?UTF-8?q?ined=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `xmd workflow start ./notes.md` now runs the file's current bytes. A file outside a repository starts, an untracked file starts, and a modified tracked file runs what it says rather than what its last commit said — because the run retains those bytes before it becomes durable, and a record naming a commit while executing something else was the thing that could not be true. Establishment reads the supplied root once. The logical entrypoint is the file's own final segment, normalized; the containing directory, the absolute path and the invocation working directory never reach the descriptor. Declared components are resolved against the root's own directory, read as bytes, decoded strictly and parsed — so a declaration this command cannot read refuses the start before any storage exists, rather than surfacing the first time a document writes the name. Every source's identity is the hash of the bytes that were actually read, and the descriptor is parsed back through storage's own closed parser before it is offered to anything. Git is provenance now, not retrieval. A start records where it happened when that is cheaply available, as replaceable metadata; a directory that is not a working tree simply has none, and the start is an ordinary start. Nothing reads it back to find a source. After admission the CLI executes the closure the lifecycle authenticated and nothing it read itself. The run id is reported only once the creation transaction has committed, and the post-admission reloads are gone: a candidate that remained a second path to execution would be a second answer to what the run is a run of. `workflow-source.ts` is now only the legacy version-1 Git reader, captured by the Deno host on both the run-host and lifecycle installations and reachable through no Context, Api, component or Plugin. It maps an unreadable object into the categorical failure Workflow declares for it and judges nothing else: Workflow recomputes the returned blob identities itself. Status and history render a source-bundle run by its entrypoint and bundle hash, with no base line and no fabricated commit. Refs #443 --- packages/cli/src/cli.ts | 26 +- packages/cli/src/deno-workflow.ts | 10 +- packages/cli/src/workflow-bundle.ts | 188 +++++-- packages/cli/src/workflow-definition.ts | 515 +++++++++++++----- packages/cli/src/workflow-fork.ts | 20 +- packages/cli/src/workflow-management.ts | 22 +- packages/cli/src/workflow-source.ts | 74 +-- packages/cli/src/workflow.ts | 162 +++--- packages/cli/tests/workflow-cli.test.ts | 289 +++++++--- .../cli/tests/workflow-declaration.test.ts | 244 +++++++++ .../tests/workflow-export-publication.test.ts | 4 +- packages/cli/tests/workflow-fork.test.ts | 38 +- .../cli/tests/workflow-inspection.test.ts | 13 +- .../cli/tests/workflow-installation.test.ts | 37 +- packages/cli/tests/workflow-retention.test.ts | 154 +++++- scripts/tests/plugin-compiled.test.ts | 73 ++- 16 files changed, 1434 insertions(+), 435 deletions(-) diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 21d933ca5..f7e938153 100755 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -1624,9 +1624,9 @@ interface PropsPhase { * The immutable definition a `workflow start` established, when it did. * * Established here rather than later because the props a run is created with - * are the ones the *pinned* document declares: reading the working tree to - * build the bindings and then executing the commit would let help and parsing - * describe a document that is not the one running. + * are the ones the *retained* document declares: binding against one reading + * of the file and then executing another would let help and parsing describe + * a document that is not the one running. */ established?: EstablishedDefinition; } @@ -1961,7 +1961,7 @@ function exactRoot(root: RootDocumentSource, target: string | undefined): RootDo /** * The props phase of a `workflow` invocation. * - * `start` reads what the pinned definition declares, so its generated + * `start` reads what the definition it establishes declares, so its generated * `--props-*` arguments are exactly `xmd run`'s for that document. A `fork` * reads the same way, from the definition it names as its third argument: the * fork is a run of that document, so its props are that document's — merged @@ -2035,10 +2035,10 @@ function* prepareWorkflowProps( return { args, bindings: [], workflow, error: established.error.message }; } - const root = retainedSource( - established.value.definition.rootDocumentPath, - established.value.source, - ); + const { definition } = established.value; + const root = retainedSource(definition.entrypoint, established.value.source, { + ...(definition.targetPath === undefined ? {} : { target: definition.targetPath }), + }); try { const document = yield* inspectDocument(root); const bindings = buildBindings(document.props); @@ -3044,11 +3044,11 @@ function* runCommand( const helpRequest = takeHelpFlag(evalFlags.rest); // Before the props phase, because that phase establishes a workflow start's - // definition from Git in order to read what the *pinned* document declares. - // On a host without workflow support the first thing a caller would otherwise - // see is whatever Git said about their directory, which is not the reason the - // command is not going to run. Help is exempt: the grammar is the same - // everywhere, and describing it costs nothing. + // definition in order to read what the *retained* document declares. On a + // host without workflow support the first thing a caller would otherwise see + // is a refusal about their document, which is not the reason the command is + // not going to run. Help is exempt: the grammar is the same everywhere, and + // describing it costs nothing. let workflowHost: WorkflowHost | undefined; if (!helpRequest.requested && namesWorkflow(helpRequest.args)) { try { diff --git a/packages/cli/src/deno-workflow.ts b/packages/cli/src/deno-workflow.ts index 88f40ac46..c3fd8d355 100644 --- a/packages/cli/src/deno-workflow.ts +++ b/packages/cli/src/deno-workflow.ts @@ -32,7 +32,7 @@ import { import type { WorkflowExecutionTransitions } from "@executablemd/workflow/deno"; import type { WorkflowRunDatabase } from "@executablemd/workflow"; import type { HelperAssembly } from "@executablemd/workflow/credential-helper"; -import { readDefinitionSource } from "./workflow-source.ts"; +import { readLegacyDefinitionSource } from "./workflow-source.ts"; import type { WorkflowHost } from "./workflow.ts"; import { gitHubIssuesConfiguration } from "./github-issues-config.ts"; import { gitHubPullRequestsConfiguration } from "./github-pull-requests-config.ts"; @@ -57,13 +57,17 @@ export function* useDenoWorkflowHost(helper: HelperAssembly): Operation { - return useWorkflowRunHost({ root }); + // The same reader the lifecycle installation captures. A version-1 run + // needs its Markdown to begin, fork or stage, and the transitions this + // returns are what admit those — so a host that could export a legacy run + // and not resume one would be two hosts wearing one name. + return useWorkflowRunHost({ root, legacySource: readLegacyDefinitionSource }); }, useLifecycle(): Operation { // The reader goes into the provider's closure, not onto a request. An // export seals the document a run was of, and a caller that could hand // that in would be sealing its own bytes as somebody else's evidence. - return useWorkflowLifecycle({ root, definitionSource: readDefinitionSource }); + return useWorkflowLifecycle({ root, legacySource: readLegacyDefinitionSource }); }, useDelivery(): Operation { return useWorkflowInputDelivery({ root }); diff --git a/packages/cli/src/workflow-bundle.ts b/packages/cli/src/workflow-bundle.ts index 69b4b8bb1..370f3a584 100644 --- a/packages/cli/src/workflow-bundle.ts +++ b/packages/cli/src/workflow-bundle.ts @@ -1,5 +1,5 @@ /** - * The components a workflow root is closed over, established from Git. + * The components a workflow root is closed over, and where they come from. * * A workflow root may declare a fixed bundle of authored Markdown components in * its own frontmatter: @@ -11,27 +11,32 @@ * Planning: ./Planning.md * ``` * - * The declaration is authored beside the root and read from the same pinned - * commit the root came from, so the whole procedure — root and components — is - * one immutable object graph. A working tree with uncommitted edits runs the - * committed components for exactly the reason it runs the committed root: a - * record that named a commit while executing something else would be a claim - * about something that never happened. + * The declaration is authored beside the root and read from beside it: each + * path is resolved against the root's own directory, read as bytes, and + * retained with the run. So the whole procedure — root and components — is one + * immutable set of bytes from the moment the run becomes durable, and a working + * tree with uncommitted edits runs what it says rather than what its last + * commit said. * * What each declaration produces is two views of one bundle. The **identity** - * view is what the workflow definition retains: name, canonical - * repository-relative path, and the blob's own object id. The **execution** - * view is the same entries plus the exact source read from the commit, which is - * what canonical core resolves the names against. Because the hash is - * identity, changing what a component says changes the definition rather than - * changing what a retained definition executes. + * view is what the workflow definition retains: name, logical path inside the + * bundle, and the source hash of the bytes that were read. The **execution** + * view is the same entries plus that source as text, which is what canonical + * core resolves the names against. Because the hash is identity, changing what + * a component says changes the definition rather than changing what a retained + * definition executes. * - * Everything here goes through the contextual `Git` capability and the - * engine's own Markdown parser. Nothing reads the filesystem, resolves a - * module, or searches a directory. + * ## The legacy half + * + * `reconstructBundle()` rebuilds a version-1 run's bundle out of the commit it + * pinned, through the contextual `Git` capability. It is reached only from the + * legacy source reader this host supplies to the Workflow lifecycle; nothing + * establishing a new bundle goes near it. */ -import { Err, Ok } from "effection"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Err, Ok, until } from "effection"; import type { Operation, Result } from "effection"; import { CORE_COMPONENT_NAMES, @@ -40,8 +45,14 @@ import { RESERVED_STRUCTURAL, } from "@executablemd/core"; import type { WorkflowBundleComponent } from "@executablemd/core/host"; -import { readGitObject, revParse } from "@executablemd/workflow"; +import { + decodeSourceText, + readGitObject, + revParse, + sourceContentHash, +} from "@executablemd/workflow"; import type { GitObjectFormat, WorkflowComponentEntry } from "@executablemd/workflow"; +import type { EstablishedComponent } from "./workflow-definition.ts"; /** Hexadecimal digits per object id, by the format that names them. */ const OBJECT_ID_LENGTHS: Readonly> = { sha1: 40, sha256: 64 }; @@ -61,10 +72,11 @@ function unavailable(message: string, cause?: unknown): WorkflowBundleUnavailabl /** * One declared component, normalized against the root's own directory. * - * `path` is already canonical: repository-relative, POSIX, with the single - * optional leading `./` removed. It is what the definition retains and what - * Git is asked for; the spelling the document wrote is not kept, because two - * spellings of one path would be two identities for one bundle. + * `path` is already canonical: bundle-relative, POSIX, with the single optional + * leading `./` removed. It is the logical path the definition retains and the + * path this host reads beside the root; the spelling the document wrote is not + * kept, because two spellings of one path would be two identities for one + * bundle. */ export interface DeclaredComponent { readonly name: string; @@ -183,15 +195,15 @@ function usableName(name: string): WorkflowBundleUnavailableError | undefined { } /** - * The repository-relative path a declaration names, or the refusal saying why - * it names none. + * The bundle-relative path a declaration names, or the refusal saying why it + * names none. * - * A declared path locates a Markdown blob beside the root inside one commit, so - * it is deliberately the narrowest thing that can do that: relative, POSIX, - * forward only, and Markdown. Everything else — an absolute path, a - * backslash, a URL, a package specifier, a glob, a directory, a traversal that - * would land back inside the repository anyway — is refused rather than - * repaired, because a repaired path runs a file the author did not write down. + * A declared path locates one Markdown file beside the root, so it is + * deliberately the narrowest thing that can do that: relative, POSIX, forward + * only, and Markdown. Everything else — an absolute path, a backslash, a URL, a + * package specifier, a glob, a directory, a traversal that would land back + * under the root's own directory anyway — is refused rather than repaired, + * because a repaired path runs a file the author did not write down. */ function canonicalPath(value: string, directory: string, name: string): Result { const refuse = (reason: string): Result => @@ -210,7 +222,7 @@ function canonicalPath(value: string, directory: string, name: string): Result { } /** - * Read every declared component out of one commit, and parse it. + * Read every declared component from the directory the root sits in, and parse + * it. + * + * A declared path is a logical path inside the bundle *and* the path the file + * has beside the root, which is what makes a declaration portable: the run + * retains `Discovery.md`, and this host happens to find it next to the document + * that named it. The host directory is joined on here and nowhere else. * - * Each source is read as a blob, so a declaration that names a directory, a - * submodule, or a path the commit does not hold fails here rather than - * executing as whatever Git chose to print. Each blob's own object id becomes - * the source hash, under the repository's object format, so there is one hash - * algorithm and it is Git's. + * Each source's identity is the source-bundle hash of its own bytes, so what + * the descriptor names and what executes come from one read of one file. * * Parsing happens here too. A bundled component that is not Markdown the engine * can read refuses the start before storage is created and before any component * code runs — rather than surfacing the first time a document writes its name. */ export function* readBundle( - pinnedCommit: string, + directory: string, declared: readonly DeclaredComponent[], - objectFormat: GitObjectFormat, -): Operation> { - const components: WorkflowBundleComponent[] = []; +): Operation> { + const components: EstablishedComponent[] = []; for (const component of declared) { - const loaded = yield* readComponent(pinnedCommit, component, objectFormat); + const loaded = yield* readComponent(directory, component); if (!loaded.ok) { return loaded; } @@ -282,39 +296,49 @@ export function* readBundle( } function* readComponent( - commit: string, + directory: string, declared: DeclaredComponent, - objectFormat: GitObjectFormat, -): Operation> { +): Operation> { const { name, path } = declared; - let content: string; - let sourceHash: string; + // Joined from the root's own directory. The declaration was already refused + // if it was absolute, a URL, a glob, or walked the tree, so what is joined + // here stays beneath the directory the document lives in. + const host = join(directory, ...path.split("/")); + + let bytes: Uint8Array; try { - // `cat-file blob` first: it refuses a tree, so a declaration that names a - // directory fails before its object id is taken as a component's identity. - content = yield* readGitObject(commit, path); - sourceHash = (yield* revParse(`${commit}:${path}`)).toLowerCase(); + bytes = new Uint8Array(yield* until(readFile(host))); } catch (error) { return Err( unavailable( - `the component "${name}" is not a file this workflow's commit holds at ${path}. ` + - "Commit the component beside the document that declares it.", + `the component "${name}" is not a file beside the document that declares it at ` + + `${path}. Put the component next to the document, or correct the path it declares.`, error, ), ); } - if (sourceHash.length !== OBJECT_ID_LENGTHS[objectFormat] || !/^[0-9a-f]+$/.test(sourceHash)) { + + const text = decodeSourceText(bytes); + if (!text.ok) { return Err( unavailable( - `the component "${name}" did not resolve to an object this repository's format names.`, + `the component "${name}" at ${path} is not well-formed UTF-8, so it is not Markdown ` + + "this command can read.", ), ); } - const parsed = yield* parseComponent(name, path, content); + + const parsed = yield* parseComponent(name, path, text.value); if (!parsed.ok) { return parsed; } - return Ok({ name, path, sourceHash, content }); + return Ok({ + name, + path, + sourceHash: yield* sourceContentHash(bytes), + content: text.value, + bytes, + }); } function* parseComponent(name: string, path: string, content: string): Operation> { @@ -346,7 +370,7 @@ export function* reconstructBundle( retained: readonly WorkflowComponentEntry[], objectFormat: GitObjectFormat, ): Operation> { - const loaded = yield* readBundle( + const loaded = yield* readGitBundle( pinnedCommit, retained.map((entry) => ({ name: entry.name, path: entry.path })), objectFormat, @@ -366,3 +390,49 @@ export function* reconstructBundle( } return loaded; } + +/** + * Read every retained component out of one commit, for a version-1 resume. + * + * The legacy half, and the only place in this module that reaches Git. A + * version-1 definition pins a commit and a path per component, so its bundle is + * rebuilt from objects rather than from files — a working tree edited since the + * run started continues the run it started. + */ +function* readGitBundle( + pinnedCommit: string, + declared: readonly DeclaredComponent[], + objectFormat: GitObjectFormat, +): Operation> { + const components: WorkflowBundleComponent[] = []; + for (const { name, path } of declared) { + let content: string; + let sourceHash: string; + try { + // `cat-file blob` first: it refuses a tree, so a declaration that names a + // directory fails before its object id is taken as a component's identity. + content = yield* readGitObject(pinnedCommit, path); + sourceHash = (yield* revParse(`${pinnedCommit}:${path}`)).toLowerCase(); + } catch (error) { + return Err( + unavailable( + `the component "${name}" is not a file this workflow's commit holds at ${path}.`, + error, + ), + ); + } + if (sourceHash.length !== OBJECT_ID_LENGTHS[objectFormat] || !/^[0-9a-f]+$/.test(sourceHash)) { + return Err( + unavailable( + `the component "${name}" did not resolve to an object this repository's format names.`, + ), + ); + } + const parsed = yield* parseComponent(name, path, content); + if (!parsed.ok) { + return parsed; + } + components.push({ name, path, sourceHash, content }); + } + return Ok(Object.freeze(components)); +} diff --git a/packages/cli/src/workflow-definition.ts b/packages/cli/src/workflow-definition.ts index f8f9eb799..4d384adcf 100644 --- a/packages/cli/src/workflow-definition.ts +++ b/packages/cli/src/workflow-definition.ts @@ -1,71 +1,89 @@ /** - * What a workflow run is a run of, established from Git. + * What a workflow run is a run of, established from the bytes the caller named. * - * `xmd workflow start notes.md` names a file in a working tree. A working tree - * changes, so it cannot be a run's identity — a resume months later has to mean - * the same document. What becomes identity is the object: the repository's - * object format, the full commit id `HEAD` resolved to once, and the document's - * repository-relative path inside it. + * `xmd workflow start notes.md` names a file, and that file's current bytes are + * what the run is of. They are read once, hashed, and retained with the run + * before it becomes durable — so a file outside a repository, an untracked + * file, and a file edited since its last commit all start, and all start from + * what they actually say. * - * The consequence is the part worth stating plainly. **The bytes that execute - * come from that commit, not from the file the caller pointed at.** A working - * tree with uncommitted edits runs the committed document, because running the - * edited one while recording the commit as identity would make the record a - * claim about something that never ran. + * **Where the file is has nothing to do with what the run is.** The containing + * directory, the absolute path and the invocation working directory are all + * retrieval facts; what the descriptor holds is a portable logical path — the + * file's own final segment for the root, and each declared component's + * canonical path relative to it. Two machines holding the same bytes under the + * same logical entrypoint hold the same definition. * - * Where the repository is *checked out* is not identity. It is retrieval - * metadata: replaceable, credential-free, excluded from the comparison that - * decides whether a reused run id addresses the same run, and reauthorized - * before it is used again. A run that moves between machines is the same run. + * Git is optional provenance and never identity. A run records where it was + * started from when that is cheaply available, as replaceable metadata; failing + * to learn it does not fail a start, and nothing ever reads it back to find the + * source. The source is in the run. * - * Everything here goes through the contextual `Git` capability, so nothing - * below runs a command of its own or names a host. + * ## The legacy path + * + * Version-1 runs still exist, and their Markdown still lives in a repository. + * `loadRetainedDefinition()` is what reaches it, and it is used from exactly one + * place: the adapter this host hands the Workflow lifecycle as its legacy + * source reader. Nothing establishes a version-1 definition any more. */ -import { isAbsolute, relative, resolve, sep } from "node:path"; -import { Err, Ok, scoped } from "effection"; +import { readFile } from "node:fs/promises"; +import { basename, dirname, resolve } from "node:path"; +import { Err, Ok, scoped, until } from "effection"; import type { Operation, Result } from "effection"; import type { Json } from "@executablemd/durable-streams"; import { API } from "@executablemd/runtime"; -import { parseMarkdownDefinition } from "@executablemd/core"; +import { + asDocumentTargetError, + fileSource, + inspectDocument, + parseMarkdownDefinition, + retainedSource, +} from "@executablemd/core"; +import type { DocumentInfo, FileRootDocument } from "@executablemd/core"; import type { WorkflowBundleComponent } from "@executablemd/core/host"; import { + decodeSourceText, definitionComponents, gitObjectFormat, - parseWorkflowDefinition, + parseSourceBundleDefinition, readGitObject, repositoryRoot, revParse, + sourceBundleHash, + sourceContentHash, +} from "@executablemd/workflow"; +import type { + GitWorkflowDefinitionV1, + SourceBundleEntryV2, + SourceBundleSnapshotEntryV2, + SourceBundleWorkflowDefinitionV2, } from "@executablemd/workflow"; -import type { WorkflowDefinition } from "@executablemd/workflow"; import { declaredBundle, readBundle, reconstructBundle } from "./workflow-bundle.ts"; -/** The base a `start` records. The command has no base option, so it is this. */ -export const DEFINITION_BASE = "HEAD"; - -/** How this host will find the definition again. Replaceable, never a credential. */ +/** How this host recorded where a run was started from. Never read back. */ export const RETRIEVAL_KIND = "local-checkout"; /** Everything one `start` establishes before a run can exist. */ export interface EstablishedDefinition { - readonly definition: WorkflowDefinition; - readonly base: string; - readonly pinnedCommit: string; - readonly retrieval: Json; - /** The document as the pinned commit holds it. */ - readonly source: string; + readonly definition: SourceBundleWorkflowDefinitionV2; /** - * The execution view of the declared component bundle, empty when the root - * declares none. + * The exact bytes behind every logical path, in the descriptor's own order. * - * The same entries the definition retains, plus the exact source each was - * read from — so what identity names and what executes come from one read of - * one commit. + * Owned copies from the one read of each file. They travel to the lifecycle + * transition, which copies them again before it validates — so nothing + * between here and storage can change what the run is of. */ + readonly sourceSnapshot: readonly SourceBundleSnapshotEntryV2[]; + /** Credential-free provenance, when it was cheaply available. */ + readonly retrieval?: Json; + /** The entrypoint as text, for the caller that is about to import it. */ + readonly source: string; + /** The execution view of the declared bundle, empty when none is declared. */ readonly components: readonly WorkflowBundleComponent[]; } -/** The pinned sources one execution runs: the root, and the bundle it is closed over. */ +/** The sources one execution runs: the root, and the bundle it is closed over. */ export interface RetainedSources { readonly source: string; readonly components: readonly WorkflowBundleComponent[]; @@ -81,14 +99,24 @@ function unavailable(message: string, cause?: unknown): WorkflowDefinitionUnavai } /** - * Run `body` with the repository as the contextual working directory. + * One file's exact bytes. + * + * `@effectionx/fs` reads text and this needs bytes, so the runtime's own + * asynchronous primitive is adapted as an operation. Never synchronous: a read + * that blocked the host would stall every other operation in the scope. + */ +function* readBytes(path: string): Operation { + return new Uint8Array(yield* until(readFile(path))); +} + +/** + * Run `body` with a directory as the contextual working directory. * * Git answers about the directory it is asked in, so every question about one * repository is asked from the same place rather than from wherever the process - * started. Keeping Git's own output out of the caller's is the capability's - * own business and is done there. + * started. */ -function inRepository(directory: string, body: () => Operation): Operation { +function inDirectory(directory: string, body: () => Operation): Operation { return scoped(function* () { yield* API.Env.around( { @@ -104,116 +132,324 @@ function inRepository(directory: string, body: () => Operation): Operation } /** - * The document's path inside its repository, as a definition may hold it. + * Establish the immutable definition of a run that is starting. * - * Repository-relative, POSIX-separated, and refused rather than repaired when - * it leaves the working tree: a path outside the repository names no object in - * the commit, and normalizing one would silently run a different document. + * The argument is a document reference, not a path: `notes.md#Release/Publish` + * names one section of one file, and a filename that really holds a `#` writes + * it `%23`. The reference is taken apart first, because resolving the whole + * argument as a path would address a file nobody has. + * + * Every phase happens before storage exists: the reference is parsed, the file + * read once, its logical entrypoint derived from its own final segment, its + * bytes decoded strictly and parsed, any selector resolved against those exact + * bytes to the one canonical target core produces, the declared bundle resolved + * against the file's own directory and read, and the descriptor built and + * verified. A file this command cannot read, cannot decode, cannot parse, whose + * selector resolves to nothing, or that declares a component it cannot read, is + * refused here — with no run, no id and nothing on disk. */ -function repositoryRelativePath(root: string, documentPath: string): Result { - const absolute = resolve(documentPath); - const within = relative(resolve(root), absolute); - if (within === "" || within.startsWith("..") || isAbsolute(within)) { +export function* establishDefinition(reference: string): Operation> { + let requested: FileRootDocument; + try { + requested = fileSource(reference); + } catch (error) { return Err( unavailable( - "the document is not inside the repository this command resolved, so no commit in it " + - "holds the document. Run the command from the repository the document belongs to.", + "the workflow definition reference could not be read. A reference is a document path, " + + "optionally followed by # and one target selector; write a literal # in a filename " + + "as %23.", + error, ), ); } - return Ok(within.split(sep).join("/")); -} -/** - * Establish the immutable definition of a run that is starting. - * - * The order matters: the repository is located from the document's own - * directory, `HEAD` is resolved once, and the object format is read from the - * same repository — so a definition never mixes one repository's commit with - * another's format. - */ -export function* establishDefinition( - documentPath: string, -): Operation> { + const documentPath = requested.path; const absolute = resolve(documentPath); + const directory = dirname(absolute); + + // The logical entrypoint is the file's own name, normalized. Where it sits is + // this machine's arrangement; what it is called is the run's. + const entrypoint = basename(absolute).normalize("NFC"); + + let bytes: Uint8Array; try { - const root = yield* inRepository(absolute.slice(0, absolute.lastIndexOf(sep)) || sep, () => - repositoryRoot(), + bytes = yield* readBytes(absolute); + } catch (error) { + return Err( + unavailable( + `the workflow definition could not be read from ${documentPath}: ` + describeCause(error), + error, + ), ); + } - return yield* inRepository(root, function* (): Operation> { - const rootDocumentPath = repositoryRelativePath(root, absolute); - if (!rootDocumentPath.ok) { - return rootDocumentPath; - } + const text = decodeSourceText(bytes); + if (!text.ok) { + return Err( + unavailable( + `the workflow definition at ${documentPath} is not well-formed UTF-8, so it is not a ` + + "Markdown document this command can run.", + ), + ); + } - const pinnedCommit = yield* revParse(`${DEFINITION_BASE}^{commit}`); - const objectFormat = yield* gitObjectFormat(); - const source = yield* readGitObject(pinnedCommit, rootDocumentPath.value); - - // The bundle is established from the same commit, and before the run - // exists: a declaration this command cannot read, or a component this - // commit does not hold, refuses the start rather than being discovered - // the first time a document writes the name. - const declared = declaredBundle( - (yield* parseMarkdownDefinition("__root__", rootDocumentPath.value, source)).meta, - rootDocumentPath.value, - ); - if (!declared.ok) { - return declared; - } - const components = - declared.value.length === 0 - ? Ok([]) - : yield* readBundle(pinnedCommit, declared.value, objectFormat); - if (!components.ok) { - return components; - } + let meta: Record; + try { + meta = (yield* parseMarkdownDefinition("__root__", entrypoint, text.value)).meta; + } catch (error) { + return Err( + unavailable( + `the workflow definition at ${documentPath} is not a Markdown document this version ` + + "can read: " + + describeCause(error), + error, + ), + ); + } - const definition = parseWorkflowDefinition({ - version: 1, - kind: "git", - objectFormat, - objectId: pinnedCommit.toLowerCase(), - rootDocumentPath: rootDocumentPath.value, - // Written only when the root declared one, so a document with no - // bundle stores the descriptor it always stored. - ...(components.value.length === 0 - ? {} - : { - components: components.value.map((component) => ({ - name: component.name, - path: component.path, - sourceHash: component.sourceHash, - })), - }), - }); - if (!definition.ok) { - return definition; - } + // Resolved against the bytes that are about to be retained, by core, once. + // What the descriptor keeps is the exact canonical target core produced — + // never the selector the caller wrote, which a later resolution against + // other bytes could answer differently. + const targetPath = yield* resolveTarget(entrypoint, text.value, requested.target); + if (!targetPath.ok) { + return targetPath; + } - return Ok({ - definition: definition.value, - base: DEFINITION_BASE, - pinnedCommit, - retrieval: { version: 1, kind: RETRIEVAL_KIND, checkout: root }, - source, - components: components.value, - }); - }); + // The bundle is resolved against the root's own directory and read before the + // run exists: a declaration this command cannot read, or a component that is + // not there, refuses the start rather than being discovered the first time a + // document writes the name. + const declared = declaredBundle(meta, entrypoint); + if (!declared.ok) { + return declared; + } + const bundle = yield* readBundle(directory, declared.value); + if (!bundle.ok) { + return bundle; + } + + const built = yield* buildSourceBundle(entrypoint, bytes, bundle.value, targetPath.value); + if (!built.ok) { + return built; + } + + return Ok({ + definition: built.value.definition, + sourceSnapshot: built.value.sourceSnapshot, + ...withProvenance(yield* provenance(directory)), + source: text.value, + components: bundle.value.map((component) => ({ + name: component.name, + path: component.path, + sourceHash: component.sourceHash, + content: component.content, + })), + }); +} + +/** Written only when there is some: an absent locator is an absent member. */ +function withProvenance(retrieval: Json | undefined): { retrieval?: Json } { + return retrieval === undefined ? {} : { retrieval }; +} + +/** What one established candidate is, descriptor and bytes together. */ +interface BuiltBundle { + readonly definition: SourceBundleWorkflowDefinitionV2; + readonly sourceSnapshot: readonly SourceBundleSnapshotEntryV2[]; +} + +/** + * The one exact target a selector names in these bytes, or none. + * + * Core resolves it, because what counts as a target is core's decision and a + * rule restated here could disagree with the one the document layer applies. + * The answer is the canonical target, never the glob or alias that asked for + * it: two spellings of one request are one run, and a glob re-resolved against + * different bytes would name a different section. + */ +function* resolveTarget( + entrypoint: string, + source: string, + selector: string | undefined, +): Operation> { + if (selector === undefined) { + return Ok(undefined); + } + let described: DocumentInfo; + try { + described = yield* inspectDocument(retainedSource(entrypoint, source, { target: selector })); } catch (error) { + const failure = asDocumentTargetError(error); return Err( unavailable( - `the workflow definition could not be established from ${documentPath}: ` + - (error instanceof Error ? error.message : String(error)), + failure === undefined + ? "the workflow definition's target could not be resolved: " + describeCause(error) + : failure.message, error, ), ); } + if (described.target === undefined) { + return Err( + unavailable( + "the workflow definition's target selector resolved to no section of the document.", + ), + ); + } + return Ok(described.target); +} + +/** + * The canonical descriptor these bytes produce, and the snapshot beside it. + * + * The manifest is sorted by the UTF-8 bytes of each logical path, because that + * is the order the descriptor is canonical in — and the snapshot is built in + * the same order, because the transition requires exactly the descriptor's + * paths in exactly its order. + * + * The target is outside the bundle hash and inside the descriptor: selecting a + * section does not change the bytes, and a run of one section is still not a + * run of the whole document. + */ +function* buildSourceBundle( + entrypoint: string, + root: Uint8Array, + components: readonly EstablishedComponent[], + targetPath: string | undefined, +): Operation> { + const byPath = new Map([[entrypoint, root]]); + for (const component of components) { + const existing = byPath.get(component.path); + if (existing === undefined) { + byPath.set(component.path, component.bytes); + continue; + } + // One logical path, one source. A component declared at the entrypoint's + // own path is the root, and two declarations of one path are one entry. + if (!sameBytes(existing, component.bytes)) { + return Err( + unavailable( + `the component "${component.name}" and another source both claim the logical path ` + + `${component.path} with different content.`, + ), + ); + } + } + + const ordered = [...byPath.keys()].sort(compareUtf8); + const sources: SourceBundleEntryV2[] = []; + const sourceSnapshot: SourceBundleSnapshotEntryV2[] = []; + for (const path of ordered) { + const bytes = byPath.get(path); + if (bytes === undefined) { + return Err(unavailable("a source this command read is no longer in hand")); + } + sources.push({ + path, + sourceHash: yield* sourceContentHash(bytes), + byteLength: bytes.byteLength, + }); + sourceSnapshot.push({ path, bytes: Uint8Array.from(bytes) }); + } + + const mapping = [...components] + .map((component) => ({ name: component.name, path: component.path })) + .sort((left, right) => compareUtf8(left.name, right.name)); + + const bundleHash = yield* sourceBundleHash({ + entrypoint, + sources, + ...(mapping.length === 0 ? {} : { components: mapping }), + }); + + // Parsed rather than assembled: the descriptor this command hands to storage + // goes through the same closed parser storage reads one back through, so a + // candidate that is not canonical is refused here rather than retained. + const definition = parseSourceBundleDefinition({ + version: 2, + kind: "source-bundle", + hashAlgorithm: "sha256", + bundleHash, + entrypoint, + sources, + ...(targetPath === undefined ? {} : { targetPath }), + ...(mapping.length === 0 ? {} : { components: mapping }), + }); + if (!definition.ok) { + return definition; + } + return Ok({ definition: definition.value, sourceSnapshot: Object.freeze(sourceSnapshot) }); +} + +/** One declared component, read and parsed, with the bytes behind it. */ +export interface EstablishedComponent { + readonly name: string; + readonly path: string; + readonly sourceHash: string; + readonly content: string; + readonly bytes: Uint8Array; +} + +function sameBytes(left: Uint8Array, right: Uint8Array): boolean { + return left.byteLength === right.byteLength && left.every((byte, at) => byte === right[at]); +} + +const encoder = new TextEncoder(); + +/** Two strings in the UTF-8 byte order a source bundle is canonical in. */ +function compareUtf8(left: string, right: string): number { + const a = encoder.encode(left); + const b = encoder.encode(right); + const shared = Math.min(a.length, b.length); + for (let index = 0; index < shared; index++) { + const one = a[index]; + const other = b[index]; + if (one !== other && one !== undefined && other !== undefined) { + return one < other ? -1 : 1; + } + } + if (a.length === b.length) { + return 0; + } + return a.length < b.length ? -1 : 1; } /** - * The checkout a retained locator names, reauthorized before it is used. + * Where this run was started from, when that is cheap to learn. + * + * Provenance and nothing more: it is credential-free, excluded from identity + * and from compatible reuse, and never read back to find the source. A + * directory that is not a working tree simply has none, and a start there is an + * ordinary start — which is the whole point of the version. + */ +function* provenance(directory: string): Operation { + try { + return yield* inDirectory(directory, function* (): Operation { + const checkout = yield* repositoryRoot(); + const objectFormat = yield* gitObjectFormat(); + const commit = yield* revParse("HEAD^{commit}"); + return { + version: 1, + kind: RETRIEVAL_KIND, + checkout, + objectFormat, + commit: commit.toLowerCase(), + }; + }); + } catch { + // Not a repository, no commits yet, or Git is not installed. None of those + // is a reason a run cannot start: the bytes are already in hand. + return undefined; + } +} + +function describeCause(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * The checkout a retained version-1 locator names, reauthorized before use. * * A retained path is replaceable metadata rather than permission a host already * has, so it is checked against the repository it claims to be: a directory @@ -242,14 +478,18 @@ function parseRetrieval(metadata: Json | undefined): Result { } /** - * Load the exact object a retained run's definition names. + * Load the exact object a retained version-1 definition names. * * It never substitutes the current `HEAD` or a same-named file in the working * tree. A resume that could do either would silently continue a different * document under the same run id. + * + * Reached from one place only: the legacy source reader this host supplies to + * the Workflow lifecycle. Nothing else in the CLI loads a definition — a + * version-2 run's source comes out of the run. */ export function* loadRetainedDefinition( - definition: WorkflowDefinition, + definition: GitWorkflowDefinitionV1, metadata: Json | undefined, ): Operation> { const checkout = parseRetrieval(metadata); @@ -258,7 +498,7 @@ export function* loadRetainedDefinition( } try { - return yield* inRepository(checkout.value, function* (): Operation> { + return yield* inDirectory(checkout.value, function* (): Operation> { const root = yield* repositoryRoot(); if (resolve(root) !== resolve(checkout.value)) { return Err( @@ -294,22 +534,9 @@ export function* loadRetainedDefinition( } catch (error) { return Err( unavailable( - "this run's retained definition could not be loaded: " + - (error instanceof Error ? error.message : String(error)), + "this run's retained definition could not be loaded: " + describeCause(error), error, ), ); } } - -/** Whether this definition names a document this slice can execute. */ -export function supportedRootDocument(definition: WorkflowDefinition): Result { - if (definition.rootDocumentPath.endsWith(".md")) { - return Ok(undefined); - } - return Err( - unavailable( - "xmd workflow runs Markdown definitions. A function-component root is not supported yet.", - ), - ); -} diff --git a/packages/cli/src/workflow-fork.ts b/packages/cli/src/workflow-fork.ts index 2d8865bf2..e5c0ed09e 100644 --- a/packages/cli/src/workflow-fork.ts +++ b/packages/cli/src/workflow-fork.ts @@ -65,8 +65,8 @@ import { import type { ForkSelection, WorkflowRun } from "@executablemd/workflow"; import type { WorkflowRunDatabase } from "@executablemd/workflow"; import type { + SourceBundleWorkflowRunCreationV2, WorkflowExecutionTransitions, - WorkflowRunCreation, } from "@executablemd/workflow/deno"; import type { EstablishedDefinition } from "./workflow-definition.ts"; import type { WorkflowExecution } from "./workflow.ts"; @@ -84,7 +84,7 @@ export interface ForkRequest { readonly sourceRunId: string; readonly checkpointEventId: string; readonly established: EstablishedDefinition; - readonly creation: WorkflowRunCreation; + readonly creation: SourceBundleWorkflowRunCreationV2; } /** @@ -145,10 +145,14 @@ export function* preflightFork( } const selection = selected.value; + // The fork's own run value, in the shape its candidate's version declares. const run: WorkflowRun = { runId: request.runId, - base: request.creation.base, - pinnedCommit: request.creation.definition.objectId, + definitionVersion: 2, + bundleHash: request.creation.definition.bundleHash, + ...(request.creation.definition.targetPath === undefined + ? {} + : { targetPath: request.creation.definition.targetPath }), }; const imported = yield* captureRootImport(request, run, execute); if (!imported.ok) { @@ -312,7 +316,7 @@ function* replayPrefix( * One preflight execution of the candidate definition. * * The same shape both passes use: the fork's own identity, the candidate's - * pinned source and bundle, and a stream that answers with whatever that pass + * retained source and bundle, and a stream that answers with whatever that pass * is replaying. */ function execution( @@ -322,7 +326,11 @@ function execution( around: (operation: Operation) => Operation, ): WorkflowExecution { return { - root: retainedSource(request.creation.definition.rootDocumentPath, request.established.source), + root: retainedSource(request.creation.definition.entrypoint, request.established.source, { + ...(request.creation.definition.targetPath === undefined + ? {} + : { target: request.creation.definition.targetPath }), + }), props: request.creation.props, stream, installations: [ diff --git a/packages/cli/src/workflow-management.ts b/packages/cli/src/workflow-management.ts index 921a85665..554c2b150 100644 --- a/packages/cli/src/workflow-management.ts +++ b/packages/cli/src/workflow-management.ts @@ -42,7 +42,11 @@ import { exists, rm } from "@effectionx/fs"; import { linkSync, mkdtempSync, rmSync } from "node:fs"; import { rename } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; -import { WorkflowInputDelivery, WorkflowLifecycle } from "@executablemd/workflow"; +import { + isGitWorkflowRunRecord, + WorkflowInputDelivery, + WorkflowLifecycle, +} from "@executablemd/workflow"; import type { DefinitionRetrieval, WorkflowArtifactIdentity, @@ -227,7 +231,10 @@ function renderStatus( `run: ${record.runId}`, `status: ${record.status}`, `definition: ${describeDefinition(snapshot)}`, - `base: ${record.base}`, + // Only a Git run has one. A source-bundle run started from exact bytes + // rather than from a repository state, and printing a base for it would be + // naming a revision it never had. + ...(isGitWorkflowRunRecord(record) ? [`base: ${record.base}`] : []), `props: ${JSON.stringify(record.props)}`, `created: ${record.createdAt}`, `updated: ${record.updatedAt}`, @@ -375,9 +382,20 @@ function describeResult(event: DurableEvent): string { } } +/** + * What this run is a run of, in the terms its own version has. + * + * A Git run is an object and a path inside it. A source-bundle run is the + * entrypoint it retains and the hash of the bytes behind it — no commit, no + * repository-relative path, and nothing synthesized to fill the shape the other + * version has. + */ function describeDefinition(snapshot: WorkflowInspectionSnapshot): string { const { definition } = snapshot.record; const target = definition.targetPath === undefined ? "" : `#${definition.targetPath}`; + if (definition.kind === "source-bundle") { + return `${definition.bundleHash} ${definition.entrypoint}${target}`; + } return `${definition.objectId} ${definition.rootDocumentPath}${target}`; } diff --git a/packages/cli/src/workflow-source.ts b/packages/cli/src/workflow-source.ts index 06dc890c6..7ed59f336 100644 --- a/packages/cli/src/workflow-source.ts +++ b/packages/cli/src/workflow-source.ts @@ -1,49 +1,59 @@ /** - * Reading a retained definition's Markdown back, for an export to seal. + * This host's legacy version-1 source reader. * - * The one place this host turns retrieval metadata into bytes. It is installed - * into the lifecycle provider and captured there, so the only way to reach it - * is to be the provider — a request cannot carry a closure and no contextual - * name resolves to one. + * The one place the CLI turns a retained version-1 definition back into + * Markdown. It is handed to the Workflow lifecycle as a direct dependency and + * captured in its closure, so the only way to reach it is to be that provider — + * a request cannot carry a closure, and no contextual name resolves to one. * - * Authentication is `loadRetainedDefinition()`'s and is not repeated here: it - * reads the exact object the definition pins, never the working tree, and - * verifies every component against the hash the definition holds. What is added - * is the root's own blob identity, which a definition does not retain — a - * definition pins a commit and a path, so the identity of the document at that - * path is derived from the bytes that came back and compared with them again by - * the container. + * Version 2 never comes here. A source bundle's content is in the run's own + * store, so a host reaching a repository for it would be a second answer to a + * question storage has already answered. + * + * What this does is fetch. Whether what came back describes the definition it + * was asked about is Workflow's decision, not this adapter's: it recomputes + * every blob identity from the bytes returned and compares the root's own terms + * with the descriptor. An adapter that judged its own answer would be the only + * thing checking it. */ -import { Ok, type Operation, type Result } from "effection"; +import { Err, Ok, type Operation, type Result } from "effection"; import { gitBlobIdentity } from "@executablemd/workflow/deno"; -import type { XmdArtifactDefinitionClosure } from "@executablemd/workflow/deno"; -import type { WorkflowDefinition } from "@executablemd/workflow"; +import type { RetainedDefinitionSources } from "@executablemd/workflow/deno"; +import type { GitWorkflowDefinitionV1 } from "@executablemd/workflow"; +import { LegacyWorkflowSourceUnavailableError } from "@executablemd/workflow"; import type { Json } from "@executablemd/durable-streams"; import { loadRetainedDefinition } from "./workflow-definition.ts"; -export function* readDefinitionSource( - definition: WorkflowDefinition, +export function* readLegacyDefinitionSource( + definition: GitWorkflowDefinitionV1, retrieval: Json | undefined, -): Operation> { +): Operation> { const sources = yield* loadRetainedDefinition(definition, retrieval); if (!sources.ok) { - return sources; + // Mapped into the categorical failure Workflow declares for it: this host + // could not obtain the retained object, which is a different fact from the + // answer disagreeing with the descriptor. + return Err(new LegacyWorkflowSourceUnavailableError(sources.error.message)); } return Ok({ - root: { - objectFormat: definition.objectFormat, - pinnedCommit: definition.objectId, - rootDocumentPath: definition.rootDocumentPath, - ...(definition.targetPath === undefined ? {} : { targetPath: definition.targetPath }), - blobId: gitBlobIdentity(sources.value.source, definition.objectFormat), - content: sources.value.source, + definitionVersion: 1, + definition, + closure: { + root: { + objectFormat: definition.objectFormat, + pinnedCommit: definition.objectId, + rootDocumentPath: definition.rootDocumentPath, + ...(definition.targetPath === undefined ? {} : { targetPath: definition.targetPath }), + blobId: gitBlobIdentity(sources.value.source, definition.objectFormat), + content: sources.value.source, + }, + components: sources.value.components.map((component) => ({ + name: component.name, + path: component.path, + blobId: component.sourceHash, + content: component.content, + })), }, - components: sources.value.components.map((component) => ({ - name: component.name, - path: component.path, - blobId: component.sourceHash, - content: component.content, - })), }); } diff --git a/packages/cli/src/workflow.ts b/packages/cli/src/workflow.ts index 84fb5f056..c86217330 100644 --- a/packages/cli/src/workflow.ts +++ b/packages/cli/src/workflow.ts @@ -95,8 +95,12 @@ import type { SuspensionControllerOptions, SuspensionNotice } from "@executablem import { SUSPENSION_REQUEST } from "@executablemd/workflow"; import { describeError } from "./props.ts"; import { preflightFork } from "./workflow-fork.ts"; -import { loadRetainedDefinition, supportedRootDocument } from "./workflow-definition.ts"; import type { EstablishedDefinition, RetainedSources } from "./workflow-definition.ts"; +import type { RetainedDefinitionSources } from "@executablemd/workflow/deno"; +import type { WorkflowBundleComponent } from "@executablemd/core/host"; +import { decodeSourceText, isGitWorkflowRunRecord } from "@executablemd/workflow"; +import type { WorkflowRun, WorkflowRunRecord } from "@executablemd/workflow"; +import type { SourceBundleWorkflowRunCreationV2 } from "@executablemd/workflow/deno"; /** * What this module cannot do without knowing the host. @@ -880,7 +884,7 @@ export interface WorkflowStart { * status. * * `execute` is the shared CLI's own document machinery, handed everything this - * run decided: the pinned source, the retained props, the run's journal, the + * run decided: the retained source, the retained props, the run's journal, the * installations that belong inside the execution scope, and the attachment that * wraps it. * @@ -953,17 +957,6 @@ export function runWorkflow( } const { lock: executorLock } = acquired.value; - // A resumed run closed over a component bundle reconstructs it here: under - // the executor lock, from the retained commit, and before the execution - // record exists. A component that is gone, changed, or unreachable leaves - // the run's lifecycle records exactly as they are rather than adding an - // attempt that never began. - const reconstructed = yield* reconstructedSources(request, runId); - if (!reconstructed.ok) { - report(reconstructed.error.message); - return { exitCode: 1 }; - } - // One transaction: whatever the previous workflow executor left is reconciled, this // action is admitted against what that left behind, and the execution is // recorded — or none of it is. A fork's one transaction is its whole @@ -983,10 +976,12 @@ export function runWorkflow( } const { database, record, execution, replay } = begun.value; + // Only after the creation transaction committed. A run id reported before + // it would name something a failure could still leave absent. reportRun(record.runId); - // Only now, and only because execution or replay was admitted. - const source = yield* documentSource(start, database, reconstructed.value); + // What the transition authenticated, and nothing this command read. + const source = executableSources(begun.value.sources); if (!source.ok) { report(source.error.message); return { exitCode: 1 }; @@ -1042,7 +1037,14 @@ export function runWorkflow( const completed = yield* isCompleted(database.journal); const documentExecution: WorkflowExecution = { - root: retainedSource(record.definition.rootDocumentPath, source.value.source), + // The exact target the run retains, never a selector re-resolved now: a + // resumed run continues the section it started, and nothing asks the + // document layer that question a second time. + root: retainedSource(rootDocumentName(record), source.value.source, { + ...(record.definition.targetPath === undefined + ? {} + : { target: record.definition.targetPath }), + }), props: record.props, stream: database.journal, // The run already exists: the begin transition created or found it before @@ -1051,11 +1053,7 @@ export function runWorkflow( // beside it, through the same host-service slot `xmd run` fills with a // real adapter. installations: [ - retainedWorkflowInstallation({ - runId: record.runId, - base: record.base, - pinnedCommit: record.definition.objectId, - }), + retainedWorkflowInstallation(installedRun(record)), // The bundle this run is a run of, when it is a run of one. Both start // and resume install it, and a completed replay installs it too: the // retained history is held to the same components before its recorded @@ -1207,7 +1205,7 @@ function* forkInheritance( request: WorkflowRequest, runId: string, start: WorkflowStart | undefined, - creation: WorkflowRunCreation | undefined, + creation: SourceBundleWorkflowRunCreationV2 | undefined, host: WorkflowHost, transitions: WorkflowExecutionTransitions, execute: (execution: WorkflowExecution) => Operation>, @@ -1255,24 +1253,23 @@ function* startCreation( request: WorkflowRequest, start: WorkflowStart | undefined, inherited: Record | undefined, -): Operation> { +): Operation> { if (request.action === "resume") { return Ok(undefined); } if (start === undefined) { return Err(new Error(`xmd workflow ${request.action} has no definition to run`)); } - const supported = supportedRootDocument(start.established.definition); - if (!supported.ok) { - return supported; - } const props = yield* forkProps(start, inherited); if (!props.ok) { return props; } + // The descriptor and the bytes together. They were established from one read + // of each file before the lock was taken, and the transition copies them + // again before it validates — so what becomes durable is what was read. return Ok({ definition: start.established.definition, - base: start.established.base, + sourceSnapshot: start.established.sourceSnapshot, props: props.value, ...(start.established.retrieval === undefined ? {} @@ -1343,54 +1340,87 @@ function* inheritedProps( } /** - * The document this run executes. + * The document this run executes, as the lifecycle authenticated it. * - * A `start` already established it from Git to read what the pinned document - * declares. A resume fetches what the run retained, and only once the run has - * been admitted — a run that ended is not one to fetch a definition for. + * Not re-read, not re-fetched, and never the file the caller pointed at. The + * begin transition proved the source under the executor lock before it wrote + * anything, and what it answered with is the only thing that may import: a + * second read here would be a second source of truth about what the run is a + * run of, and the two could differ. */ -function* documentSource( - start: WorkflowStart | undefined, - database: WorkflowRunDatabase, - reconstructed: RetainedSources | undefined, -): Operation> { - if (start !== undefined) { - return Ok({ source: start.established.source, components: start.established.components }); +function executableSources(sources: RetainedDefinitionSources): Result { + if (sources.definitionVersion === 1) { + return Ok({ + source: sources.closure.root.content, + components: sources.closure.components.map((component) => ({ + name: component.name, + path: component.path, + sourceHash: component.blobId, + content: component.content, + })), + }); + } + + const byPath = new Map(sources.sources.map((source) => [source.path, source.bytes])); + const entry = byPath.get(sources.definition.entrypoint); + if (entry === undefined) { + return Err(new Error("this run's retained source holds no entrypoint")); } - if (reconstructed !== undefined) { - return Ok(reconstructed); + const source = decodeSourceText(entry); + if (!source.ok) { + return source; } - return yield* loadRetainedDefinition(database.record.definition, database.retrieval?.metadata); + + const components: WorkflowBundleComponent[] = []; + for (const declared of sources.definition.components ?? []) { + const bytes = byPath.get(declared.path); + const retained = sources.definition.sources.find((each) => each.path === declared.path); + if (bytes === undefined || retained === undefined) { + return Err(new Error(`this run's retained source holds no ${declared.name}`)); + } + const content = decodeSourceText(bytes); + if (!content.ok) { + return content; + } + components.push({ + name: declared.name, + path: declared.path, + sourceHash: retained.sourceHash, + content: content.value, + }); + } + return Ok({ source: source.value, components: Object.freeze(components) }); } /** - * The pinned sources a resumed run closed over a bundle needs before it begins. - * - * Answers with nothing for a `start`, which established its own bundle from Git - * before it asked storage for anything, and for a run whose definition names no - * components — that one keeps loading its root after the run has been admitted, - * because a run that ended is not one to fetch a definition for. + * The run value this execution is installed under. * - * A run this host cannot inspect answers with nothing too. What that run is, - * and whether this action may advance it, is the begin transition's to decide, - * and answering it here would report a different refusal for the same fact. + * Whichever version the record retains, spelled as that version spells it. A + * source-bundle run records its bundle hash and exact target; it invents no + * base or pinned commit for a repository it never had. */ -function* reconstructedSources( - request: WorkflowRequest, - runId: string, -): Operation> { - if (request.action !== "resume") { - return Ok(undefined); - } - const snapshot = yield* WorkflowLifecycle.operations.inspect(runId); - if (!snapshot.ok) { - return Ok(undefined); - } - const { definition } = snapshot.value.record; - if (definitionComponents(definition).length === 0) { - return Ok(undefined); +function installedRun(record: WorkflowRunRecord): WorkflowRun { + if (isGitWorkflowRunRecord(record)) { + return { + runId: record.runId, + base: record.base, + pinnedCommit: record.definition.objectId, + }; } - return yield* loadRetainedDefinition(definition, snapshot.value.retrieval?.metadata); + const { bundleHash, targetPath } = record.definition; + return { + runId: record.runId, + definitionVersion: 2, + bundleHash, + ...(targetPath === undefined ? {} : { targetPath }), + }; +} + +/** The logical path a run's own definition names its root document by. */ +function rootDocumentName(record: WorkflowRunRecord): string { + return isGitWorkflowRunRecord(record) + ? record.definition.rootDocumentPath + : record.definition.entrypoint; } /** diff --git a/packages/cli/tests/workflow-cli.test.ts b/packages/cli/tests/workflow-cli.test.ts index 422469f38..6ccf01527 100644 --- a/packages/cli/tests/workflow-cli.test.ts +++ b/packages/cli/tests/workflow-cli.test.ts @@ -21,6 +21,8 @@ import { tmpdir } from "node:os"; import { runCli } from "@executablemd/test-support/launch"; import { useWorkflowLifecycle, workflowRunPath } from "@executablemd/workflow/deno"; import { WorkflowLifecycle } from "@executablemd/workflow"; +import { DatabaseSync } from "node:sqlite"; +import { hashRunId } from "@executablemd/workflow/deno"; interface Fixture { /** The repository the definition lives in. */ @@ -31,6 +33,20 @@ interface Fixture { readonly home: string; } +/** Two sections, so one can be selected and the other seen not to run. */ +const SECTIONS = [ + "# Release", + "", + "## Publish", + "", + "publishing.", + "", + "## Announce", + "", + "announcing.", + "", +].join("\n"); + const RELEASE = [ "---", "props:", @@ -223,8 +239,11 @@ describe("Tier WFC — xmd workflow start and resume", () => { }); }); - it("WFC4: the definition is the committed object, not the working tree", function* () { + it("WFC4: the definition is the file's current bytes, committed or not", function* () { yield* useFixture({ "flows/release.md": RELEASE }, function* (fixture) { + // A tracked file, edited since its last commit. What runs is what it says + // now: the run retains these bytes, so recording them and running + // something else was the thing that could not be true. yield* writeTextFile( join(fixture.repository, "flows/release.md"), `${RELEASE}\nUNCOMMITTED\n`, @@ -239,7 +258,28 @@ describe("Tier WFC — xmd workflow start and resume", () => { expect(started.code).toBe(0); expect(started.stdout).toContain("Wrote: channel=stable"); - expect(started.stdout).not.toContain("UNCOMMITTED"); + expect(started.stdout).toContain("UNCOMMITTED"); + }); + }); + + it("WFC4b: an untracked file starts, and runs what it says", function* () { + yield* useFixture({ "flows/release.md": RELEASE }, function* (fixture) { + // Never added, never committed: there is no object for this document at + // all, and it starts anyway because the run retains the bytes. + yield* writeTextFile( + join(fixture.repository, "flows/untracked.md"), + `${RELEASE}\nUNTRACKED\n`, + ); + + const started = yield* xmd(fixture, [ + "workflow", + "start", + "--id=untracked-1", + "flows/untracked.md", + ]).join(); + + expect(started.code).toBe(0); + expect(started.stdout).toContain("UNTRACKED"); }); }); @@ -526,20 +566,28 @@ describe("Tier WFC — xmd workflow start and resume", () => { }); }); - it("WFC9: a definition outside a repository, and one that is not Markdown", function* () { - yield* useFixture( - { "flows/release.md": RELEASE, "flows/root.ts": "export default 1;\n" }, - function* (fixture) { - const notMarkdown = yield* xmd(fixture, ["workflow", "start", "flows/root.ts"]).join(); - expect(notMarkdown.code).toBe(1); - expect(notMarkdown.stderr).toMatch(/markdown/i); - expect(reportedRunId(notMarkdown.stderr)).toBeUndefined(); - - const outside = yield* xmd(fixture, ["workflow", "start", "../elsewhere.md"]).join(); - expect(outside.code).toBe(1); - expect(reportedRunId(outside.stderr)).toBeUndefined(); - }, - ); + it("WFC9: a file outside any repository starts, and one that is not there does not", function* () { + yield* useFixture({ "flows/release.md": RELEASE }, function* (fixture) { + // Outside the working tree entirely. No repository, no commit, no object + // — and a run, because the bytes are what the run is of. + const elsewhere = join(fixture.home, "elsewhere.md"); + yield* writeTextFile(elsewhere, `${RELEASE}\nOUTSIDE\n`); + + const outside = yield* xmd(fixture, [ + "workflow", + "start", + "--id=outside-1", + elsewhere, + ]).join(); + expect(outside.code).toBe(0); + expect(reportedRunId(outside.stderr)).toBe("outside-1"); + expect(outside.stdout).toContain("OUTSIDE"); + + // A path naming no file is still a refusal, and still reports no run. + const absent = yield* xmd(fixture, ["workflow", "start", "flows/absent.md"]).join(); + expect(absent.code).toBe(1); + expect(reportedRunId(absent.stderr)).toBeUndefined(); + }); }); it("WFC10: ordinary xmd run is unchanged by any of this", function* () { @@ -553,6 +601,70 @@ describe("Tier WFC — xmd workflow start and resume", () => { expect(yield* readTextFile(join(fixture.repository, "notes.md"))).toBe("channel=beta"); }); }); + + it("WFC9b: a reference selects one section, and the run is of that section", function* () { + yield* useFixture({ "flows/sections.md": SECTIONS }, function* (fixture) { + // A glob, deliberately. What the run retains has to be what core resolved + // it to, so a start that wrote the caller's selector down instead would + // be visible here rather than only where the selector happened to be + // spelled the same as its answer. + const started = yield* xmd(fixture, [ + "workflow", + "start", + "--id=section-1", + "flows/sections.md#Pub*", + ]).join(); + + expect(started.code).toBe(0); + expect(reportedRunId(started.stderr)).toBe("section-1"); + // One section ran, and the one beside it did not. + expect(started.stdout).toContain("publishing."); + expect(started.stdout).not.toContain("announcing."); + + // What the run retains is the exact canonical target, reported with the + // definition — never the glob the caller wrote. + const status = yield* xmd(fixture, ["workflow", "status", "section-1"]).join(); + expect(status.code).toBe(0); + expect(status.stdout).toContain("sections.md#Publish"); + expect(status.stdout).not.toContain("Pub*"); + + // The document is taken away, and the resume still runs that section out + // of what the run retained rather than resolving the selector again. + yield* rm(join(fixture.repository, "flows/sections.md"), { force: true }); + const resumed = yield* xmd(fixture, ["workflow", "resume", "section-1"]).join(); + expect(resumed.code).toBe(0); + expect(resumed.stdout).toContain("publishing."); + expect(resumed.stdout).not.toContain("announcing."); + }); + }); + + it("WFC9c: a whole-document run and a targeted one are two runs", function* () { + yield* useFixture({ "flows/sections.md": SECTIONS }, function* (fixture) { + yield* xmd(fixture, ["workflow", "start", "--id=whole-1", "flows/sections.md"]).expect(); + + // The same bytes under the same logical name, one section selected: a + // different definition, because the target is part of identity even + // though it is outside the bundle hash. + const targeted = yield* xmd(fixture, [ + "workflow", + "start", + "--id=whole-1", + "flows/sections.md#Publish", + ]).join(); + expect(targeted.code).toBe(1); + expect(targeted.stderr).toContain("definition"); + + // And a selector naming no section refuses before a run exists at all. + const absent = yield* xmd(fixture, [ + "workflow", + "start", + "--id=absent-1", + "flows/sections.md#Nowhere", + ]).join(); + expect(absent.code).toBe(1); + expect(reportedRunId(absent.stderr)).toBeUndefined(); + }); + }); }); /** @@ -594,10 +706,23 @@ const LOOP_FILES: Record = { "flows/Implementation.md": "implemented.\n", }; -/** Overwrite every checkout copy, so a run that reads one is visible. */ +/** + * Edit every committed component, leaving the root's declaration intact. + * + * The marker is added to what each component already said rather than replacing + * it, so the stage order stays observable while the edit is too: a run that + * read the committed objects would print the stages without the marker, and one + * that read the files beside the root prints both. + */ function* editCheckout(fixture: Fixture): Operation { - for (const name of Object.keys(LOOP_FILES)) { - yield* writeTextFile(join(fixture.repository, name), "EDITED IN THE WORKING TREE\n"); + for (const [name, content] of Object.entries(LOOP_FILES)) { + if (name === "flows/loop.md") { + continue; + } + yield* writeTextFile( + join(fixture.repository, name), + `${content}\nEDITED IN THE WORKING TREE\n`, + ); } } @@ -608,17 +733,18 @@ function* commitAgain(fixture: Fixture, message: string): Operation { } describe("Tier WFC — a workflow closed over a component bundle", () => { - it("WFC14: the five declared stages run from the commit, not from the checkout", function* () { + it("WFC14: the five declared stages run from the files beside the root", function* () { yield* useFixture(LOOP_FILES, function* (fixture) { // Every one of the six files says something else in the working tree by - // the time the run starts. + // the time the run starts, and that is what the run is of: the bundle is + // read from the directory the root lives in, and retained with it. yield* editCheckout(fixture); const started = yield* xmd(fixture, ["workflow", "start", "flows/loop.md"]).join(); expect(started.code).toBe(0); expect(reportedStatus(started.stderr)).toBe("completed"); - expect(started.stdout).not.toContain("EDITED IN THE WORKING TREE"); + expect(started.stdout).toContain("EDITED IN THE WORKING TREE"); const order = [ "discovered.", @@ -762,7 +888,7 @@ describe("Tier WFC — a workflow closed over a component bundle", () => { }); }); - it("WFC19: a resume whose pinned components are unreachable is refused whole", function* () { + it("WFC19: a resume needs no repository, because the run retains its bundle", function* () { yield* useFixture(LOOP_FILES, function* (fixture) { const started = yield* xmd(fixture, [ "workflow", @@ -772,21 +898,26 @@ describe("Tier WFC — a workflow closed over a component bundle", () => { ]).join(); expect(started.code).toBe(0); - const before = yield* xmd(fixture, ["workflow", "history", "loop-4", "--json"]).join(); - - // The repository this run retains is no longer a repository. + // Everything the run was started from is taken away: the repository it + // was started in, and every file it was read from. None of it was the + // run's source — the run's source is in the run. yield* rm(join(fixture.repository, ".git"), { recursive: true, force: true }); + yield* rm(join(fixture.repository, "flows"), { recursive: true, force: true }); const resumed = yield* xmd(fixture, ["workflow", "resume", "loop-4"]).join(); - expect(resumed.code).toBe(1); - expect(reportedStatus(resumed.stderr)).toBeUndefined(); - - // Its lifecycle records are exactly what they were: the refusal happened - // before an execution was recorded. - yield* git(fixture.repository, ["init", "-q", "--initial-branch=main", "."]); - const after = yield* xmd(fixture, ["workflow", "history", "loop-4", "--json"]).join(); - expect(after.stdout).toBe(before.stdout); + expect(resumed.code).toBe(0); + expect(reportedStatus(resumed.stderr)).toBe("completed"); + // And it replayed the same five stages, in the same order. + const order = [ + "discovered.", + "instruction files listed.", + "checkpoint reached.", + "planned.", + "implemented.", + ].map((text) => resumed.stdout.indexOf(text)); + expect(order.every((at) => at >= 0)).toBe(true); + expect([...order].sort((left, right) => left - right)).toEqual(order); }); }); }); @@ -953,7 +1084,7 @@ describe("Tier WFX — what xmd workflow export refuses", () => { }); }); - it("WFX5: refuses when the run's own source cannot be read back", function* () { + it("WFX5: exports after everything the run was started from is gone", function* () { yield* useFixture({ "flows/release.md": RELEASE }, function* (fixture) { const started = yield* xmd(fixture, [ "workflow", @@ -963,89 +1094,69 @@ describe("Tier WFX — what xmd workflow export refuses", () => { ]).join(); expect(started.code).toBe(0); - // The repository the run retains stops being one. Nothing about the run - // changes: what is gone is the only place its definition's bytes were. + // The repository stops being one, and the document stops existing. The + // run is untouched by either: its source is what it retains, and that is + // what an artifact seals. yield* rm(join(fixture.repository, ".git"), { recursive: true, force: true }); + yield* rm(join(fixture.repository, "flows"), { recursive: true, force: true }); - const target = join(fixture.repository, "unreadable.xmd"); - const refused = yield* xmd(fixture, [ + const target = join(fixture.home, "detached.xmd"); + const sealed = yield* xmd(fixture, [ "workflow", "export", "release-1", `--output=${target}`, ]).join(); - expect(refused.code).toBe(1); - // Named, because "exited 1" is what a run this command never found would - // also say: what refused is the reading of this run's own definition. - expect(refused.stderr).toContain("this run's retained definition could not be loaded"); - yield* expectNothingPublished(fixture, target); + expect(sealed.code).toBe(0); + expect(yield* exists(target)).toBe(true); }); }); - it("WFX6: refuses source the repository hands back that is not this run's", function* () { - yield* useFixture(BUNDLE_FILES, function* (fixture) { + it("WFX6: refuses when the run's own retained source no longer describes itself", function* () { + yield* useFixture({ "flows/release.md": RELEASE }, function* (fixture) { const started = yield* xmd(fixture, [ "workflow", "start", - "--id=bundle-1", - "flows/bundle.md", + "--id=corrupt-1", + "flows/release.md", ]).join(); expect(started.code).toBe(0); - const pinned = yield* revision(fixture, "HEAD"); - - // A second commit that says something else at the component's path, put - // in front of the pinned one. The definition still names the commit it - // named; the repository now answers for it with another object, which is - // exactly the case a host's own reading cannot notice. - yield* writeTextFile(join(fixture.repository, "flows/Stage.md"), "replaced.\n"); - yield* commitAgain(fixture, "replacement"); - const replacement = yield* revision(fixture, "HEAD"); - yield* git(fixture.repository, ["replace", pinned, replacement]); - - const target = join(fixture.repository, "mismatched.xmd"); + + // The same length, different bytes. Every constraint the table declares + // still holds; what fails is recomputing the hash the descriptor names, + // which is the one check a container cannot do for itself. + const store = join(fixture.runs, `${hashRunId("corrupt-1")}.sqlite`); + const database = new DatabaseSync(store); + try { + const current = database.prepare("SELECT content FROM workflow_definition_blob").get(); + const bytes = current?.["content"]; + if (!(bytes instanceof Uint8Array)) { + throw new Error("the run retains no source bytes"); + } + const altered = Uint8Array.from(bytes); + altered[0] = altered[0] === 0x23 ? 0x2a : 0x23; + database.prepare("UPDATE workflow_definition_blob SET content = ?").run(altered); + } finally { + database.close(); + } + + const target = join(fixture.home, "corrupt.xmd"); const refused = yield* xmd(fixture, [ "workflow", "export", - "bundle-1", + "corrupt-1", `--output=${target}`, ]).join(); expect(refused.code).toBe(1); - expect(refused.stderr).toContain("no longer the object this run's definition names"); + expect(refused.stderr).toContain("disagrees with its own descriptor"); yield* expectNothingPublished(fixture, target); }); }); }); -/** A root closed over one component, so a definition has an object to disagree about. */ -const BUNDLE_FILES: Record = { - "flows/bundle.md": [ - "---", - "workflow:", - " components:", - " Stage: ./Stage.md", - "---", - "", - "# Bundle", - "", - "", - "", - ].join("\n"), - "flows/Stage.md": "staged.\n", -}; - /** What one revision resolves to in the fixture's repository right now. */ -function* revision(fixture: Fixture, name: string): Operation { - const result = yield* exec("git", { - arguments: ["rev-parse", "--verify", name], - cwd: fixture.repository, - }).expect(); - if (result.code !== 0) { - throw new Error(`git rev-parse ${name} failed: ${result.stderr}`); - } - return result.stdout.trim(); -} /** * That a refusal left the destination alone, and no staging beside it. diff --git a/packages/cli/tests/workflow-declaration.test.ts b/packages/cli/tests/workflow-declaration.test.ts index d50d6bcc7..69955971c 100644 --- a/packages/cli/tests/workflow-declaration.test.ts +++ b/packages/cli/tests/workflow-declaration.test.ts @@ -25,6 +25,15 @@ import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { declaredBundle } from "../src/workflow-bundle.ts"; import type { DeclaredComponent } from "../src/workflow-bundle.ts"; +import { ensure, scoped, until } from "effection"; +import type { Operation } from "effection"; +import { ensureDir, rm, writeTextFile } from "@effectionx/fs"; +import { writeFile } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { sourceContentHash } from "@executablemd/workflow"; +import { establishDefinition } from "../src/workflow-definition.ts"; const ROOT = "workflows/loop.md"; @@ -203,3 +212,238 @@ describe("Tier WFD — reading a workflow's component declaration", () => { ]); }); }); + +/** + * Tier WFD — what a declaration becomes once it is read from disk. + * + * The declaration is normalization; this is establishment. A root's own name is + * the bundle's entrypoint, each declared path is a logical path beside it, and + * every source's identity is the hash of the bytes that were actually read — + * so what the descriptor names and what executes come from one read of one + * file. A component that is not there refuses the start, before any storage + * exists to refuse it later. + */ +describe("Tier WFD — establishing a declared bundle", () => { + const PLAIN = "# Plain\n\nno components at all\n"; + + function useDirectory( + files: Record, + body: (directory: string) => Operation, + ): Operation { + return scoped(function* () { + const directory = join(tmpdir(), `xmd-wfd-${randomUUID()}`); + yield* ensure(() => rm(directory, { recursive: true, force: true })); + for (const [name, content] of Object.entries(files)) { + const at = join(directory, name); + yield* ensureDir(join(at, "..")); + yield* writeTextFile(at, content); + } + return yield* body(directory); + }); + } + + function* established(directory: string, name: string) { + return yield* establishDefinition(join(directory, name)); + } + + it("WFD40: a root with no declaration retains exactly one source", function* () { + yield* useDirectory({ "plain.md": PLAIN }, function* (directory) { + const result = yield* established(directory, "plain.md"); + if (!result.ok) { + throw result.error; + } + const { definition, sourceSnapshot } = result.value; + + expect(definition.entrypoint).toBe("plain.md"); + expect(definition.sources.map((source) => source.path)).toEqual(["plain.md"]); + expect("components" in definition).toBe(false); + // The identity is the bytes that were read, and the snapshot is those + // bytes: one read, two views of it. + expect(definition.sources[0]?.byteLength).toBe(new TextEncoder().encode(PLAIN).byteLength); + expect(new TextDecoder().decode(sourceSnapshot[0]?.bytes)).toBe(PLAIN); + expect(definition.sources[0]?.sourceHash).toBe( + yield* sourceContentHash(new TextEncoder().encode(PLAIN)), + ); + }); + }); + + it("WFD41: a declared component becomes a logical path beside the root", function* () { + const root = [ + "---", + "workflow:", + " components:", + " Stage: ./stages/Stage.md", + "---", + "", + "# Root", + "", + ].join("\n"); + const stage = "# Stage\n\nthe stage says this\n"; + + yield* useDirectory({ "root.md": root, "stages/Stage.md": stage }, function* (directory) { + const result = yield* established(directory, "root.md"); + if (!result.ok) { + throw result.error; + } + const { definition, sourceSnapshot, components } = result.value; + + // Bundle-relative, not the absolute path this machine found it at, and + // in the canonical UTF-8 order the manifest is required to be in. + expect(definition.sources.map((source) => source.path)).toEqual([ + "root.md", + "stages/Stage.md", + ]); + expect(definition.components).toEqual([{ name: "Stage", path: "stages/Stage.md" }]); + expect(sourceSnapshot.map((entry) => entry.path)).toEqual(["root.md", "stages/Stage.md"]); + + // The bytes retained for the component are the component's, and the + // execution view carries the same identity the descriptor does. + const retained = sourceSnapshot.find((entry) => entry.path === "stages/Stage.md"); + expect(new TextDecoder().decode(retained?.bytes)).toBe(stage); + expect(components.map((component) => component.name)).toEqual(["Stage"]); + expect(components[0]?.content).toBe(stage); + expect(components[0]?.sourceHash).toBe( + definition.sources.find((source) => source.path === "stages/Stage.md")?.sourceHash, + ); + }); + }); + + it("WFD42: a declared component that is not beside the root refuses the start", function* () { + const root = [ + "---", + "workflow:", + " components:", + " Absent: ./Absent.md", + "---", + "", + "# Root", + "", + ].join("\n"); + + yield* useDirectory({ "root.md": root }, function* (directory) { + const result = yield* established(directory, "root.md"); + expect(result.ok).toBe(false); + expect(result.ok ? "" : result.error.message).toContain("Absent"); + }); + }); + + it("WFD43: a root that is not well-formed UTF-8 refuses before anything is read", function* () { + const directory = join(tmpdir(), `xmd-wfd-${randomUUID()}`); + yield* ensure(() => rm(directory, { recursive: true, force: true })); + yield* ensureDir(directory); + // A lone continuation byte: no decoder produces text from it, and + // producing replacement characters would parse a document nobody wrote. + yield* until(writeFile(join(directory, "broken.md"), new Uint8Array([0x23, 0x20, 0x80]))); + + const result = yield* establishDefinition(join(directory, "broken.md")); + expect(result.ok).toBe(false); + expect(result.ok ? "" : result.error.message).toContain("UTF-8"); + }); +}); + +/** + * Tier WFD — the one exact target a reference selects. + * + * The argument is a document reference, so `notes.md#Release/Publish` names a + * section rather than a file whose name ends in `#Release/Publish`. What the + * descriptor keeps is the canonical target core resolved against the bytes that + * are about to be retained — never the selector the caller wrote, because two + * spellings of one request are one run and a glob re-resolved against different + * bytes would name a different section. + */ +describe("Tier WFD — the target a reference resolves", () => { + const SECTIONS = [ + "# Release", + "", + "## Publish", + "", + "publishing.", + "", + "## Announce", + "", + "announcing.", + "", + ].join("\n"); + + function useDocument( + files: Record, + body: (directory: string) => Operation, + ): Operation { + return scoped(function* () { + const directory = join(tmpdir(), `xmd-wfdt-${randomUUID()}`); + yield* ensure(() => rm(directory, { recursive: true, force: true })); + for (const [name, content] of Object.entries(files)) { + const at = join(directory, name); + yield* ensureDir(join(at, "..")); + yield* writeTextFile(at, content); + } + return yield* body(directory); + }); + } + + it("WFD44: a selector resolves to one exact target before storage exists", function* () { + yield* useDocument({ "sections.md": SECTIONS }, function* (directory) { + const result = yield* establishDefinition(`${join(directory, "sections.md")}#Publish`); + if (!result.ok) { + throw result.error; + } + const { definition } = result.value; + + // The path is the file, and the fragment never reached it. + expect(definition.entrypoint).toBe("sections.md"); + expect(definition.sources.map((source) => source.path)).toEqual(["sections.md"]); + expect(definition.targetPath).toBe("Publish"); + + // And the target is outside the bundle hash: the same document without a + // selector retains the same bytes under the same hash. + const whole = yield* establishDefinition(join(directory, "sections.md")); + if (!whole.ok) { + throw whole.error; + } + expect("targetPath" in whole.value.definition).toBe(false); + expect(whole.value.definition.bundleHash).toBe(definition.bundleHash); + }); + }); + + it("WFD45: a glob resolves to the canonical target, not to itself", function* () { + yield* useDocument({ "sections.md": SECTIONS }, function* (directory) { + const result = yield* establishDefinition(`${join(directory, "sections.md")}#Pub*`); + if (!result.ok) { + throw result.error; + } + // What is retained is what core resolved, so a resume continues the + // section this start selected rather than re-answering the pattern. + expect(result.value.definition.targetPath).toBe("Publish"); + }); + }); + + it("WFD46: a selector naming no section refuses, before anything is retained", function* () { + yield* useDocument({ "sections.md": SECTIONS }, function* (directory) { + const absent = yield* establishDefinition(`${join(directory, "sections.md")}#Nowhere`); + expect(absent.ok).toBe(false); + expect(absent.ok ? "" : absent.error.message).toContain("Nowhere"); + }); + }); + + it("WFD47: a filename holding a # is addressable, and is not a logical path", function* () { + yield* useDocument( + { "od#d.md": "# Odd\n\nthis file really has a hash\n" }, + function* (directory) { + // `%23` is the one way a reference names it, and it is read as the path + // rather than split into a selector — which is the whole reason the + // argument is parsed as a reference before it is resolved as a path. + const escaped = yield* establishDefinition(`${join(directory, "od%23d.md")}`); + expect(escaped.ok).toBe(false); + // Refused for what it is: a logical path inside a bundle admits no `#`, + // because a retained path holding one could not be told from a path and + // a target. The refusal names the grammar, not the selector parser. + expect(escaped.ok ? "" : escaped.error.message).toContain('without a "#"'); + + // Read as a plain path, the same argument names nothing at all: the + // text after the `#` was never part of the filename. + const unescaped = yield* establishDefinition(`${join(directory, "od#d.md")}`); + expect(unescaped.ok).toBe(false); + }, + ); + }); +}); diff --git a/packages/cli/tests/workflow-export-publication.test.ts b/packages/cli/tests/workflow-export-publication.test.ts index 844207ca1..ca049a7e1 100644 --- a/packages/cli/tests/workflow-export-publication.test.ts +++ b/packages/cli/tests/workflow-export-publication.test.ts @@ -42,7 +42,7 @@ import { tmpdir } from "node:os"; import { runCli } from "@executablemd/test-support/launch"; import { useWorkflowLifecycle, useWorkflowRunStorage } from "@executablemd/workflow/deno"; import { exportArtifact, type ExportFilesystem } from "../src/workflow-management.ts"; -import { readDefinitionSource } from "../src/workflow-source.ts"; +import { readLegacyDefinitionSource } from "../src/workflow-source.ts"; const RELEASE = ["# Release", "", "nothing to see here", ""].join("\n"); @@ -127,7 +127,7 @@ function withHost(fixture: Fixture, body: () => Operation): Operation { process.chdir(previous); }); yield* useWorkflowRunStorage({ root: fixture.runs }); - yield* useWorkflowLifecycle({ root: fixture.runs, definitionSource: readDefinitionSource }); + yield* useWorkflowLifecycle({ root: fixture.runs, legacySource: readLegacyDefinitionSource }); return yield* body(); }); } diff --git a/packages/cli/tests/workflow-fork.test.ts b/packages/cli/tests/workflow-fork.test.ts index ccf7fac7b..08b7dc3ea 100644 --- a/packages/cli/tests/workflow-fork.test.ts +++ b/packages/cli/tests/workflow-fork.test.ts @@ -511,7 +511,7 @@ describe("Tier WFF — xmd workflow fork", () => { }); }); - it("WFF3: the fork owns its inherited prefix once the source is deleted", function* () { + it("WFF3: the fork owns its inherited prefix and its own retained source", function* () { yield* useFixture({ [DEFINITION]: SOURCE }, function* (fixture) { { yield* xmd(fixture, ["workflow", "start", "--id=source-1", DEFINITION]).expect(); @@ -533,6 +533,13 @@ describe("Tier WFF — xmd workflow fork", () => { const gone = yield* xmd(fixture, ["workflow", "status", "source-1"]).join(); expect(gone.code).toBe(1); + // Everything the fork's own candidate was established from goes too: + // the file it was read from, and the repository it sat in. A resume + // that reloaded either would be running bytes this fork is not a fork + // of, and deleting only the source run would leave that reload working. + yield* rm(join(fixture.repository, DEFINITION), { force: true }); + yield* rm(join(fixture.repository, ".git"), { recursive: true, force: true }); + // The fork still reads, still holds the inherited row, and still names // the Workspace root that row was written against. const entries = yield* history(fixture, "fork-1"); @@ -541,10 +548,18 @@ describe("Tier WFF — xmd workflow fork", () => { expect(inherited.workspaceRootId).toBe(checkpoint.workspaceRootId); expect(inherited.inherited?.sourceRunId).toBe("source-1"); - // And continuing it consults nothing that is gone. + // And continuing it consults nothing that is gone: the corrected + // document the fork was created from is what replays, out of the + // fork's own store. const resumed = yield* xmd(fixture, ["workflow", "resume", "fork-1"]).join(); expect(resumed.code).toBe(0); expect(resumed.stderr).toContain("workflow status: completed"); + // The corrected document is what the fork retains, and the source's is + // not. Read from the fork's own journal rather than from what was + // rendered, because a command's output is retained history rather than + // displayed text — and it is the retained history a reload of either + // deleted file would have changed. + expect(committedExec(workflowRunPath(fixture.runs, "fork-1")).stdout).toBe("corrected\n"); } }); }); @@ -978,3 +993,22 @@ describe("Tier WFF — xmd workflow fork", () => { }); }); }); + +/** The exec record a run committed, as a second connection reads it. */ +function committedExec(path: string): { stdout?: string } { + const database = new DatabaseSync(path, { readOnly: true }); + try { + for (const row of database + .prepare("SELECT record FROM journal_events ORDER BY sequence") + .all()) { + const record = typeof row["record"] === "string" ? row["record"] : ""; + const parsed = JSON.parse(record); + if (parsed?.description?.type === "exec" && parsed?.result?.status === "ok") { + return parsed.result.value; + } + } + throw new Error("no committed exec record"); + } finally { + database.close(); + } +} diff --git a/packages/cli/tests/workflow-inspection.test.ts b/packages/cli/tests/workflow-inspection.test.ts index 9bc85409e..57fbc5268 100644 --- a/packages/cli/tests/workflow-inspection.test.ts +++ b/packages/cli/tests/workflow-inspection.test.ts @@ -232,7 +232,10 @@ describe("Tier WFI — xmd workflow status, list and history", () => { expect(human.code).toBe(0); expect(human.stdout).toContain("run: release-1"); expect(human.stdout).toContain("status: completed"); - expect(human.stdout).toContain("flows/release.md"); + // The logical entrypoint, which is the file's own name. Where the file + // sat is this machine's arrangement and is not part of what the run is. + expect(human.stdout).toContain("release.md"); + expect(human.stdout).not.toContain("flows/release.md"); expect(human.stdout).toContain("executions: 1"); const structured = yield* xmd(fixture, ["workflow", "status", "release-1", "--json"]).join(); @@ -361,7 +364,9 @@ describe("Tier WFI — xmd workflow status, list and history", () => { // history says so from what the event retained. const command = entries.find((entry) => entry.event.description?.type === "exec"); expect(command?.source).toEqual({ - path: "flows/release.md", + // The authored position is inside the retained source, so it is named + // by the logical path that source has in the bundle. + path: "release.md", offset: RELEASE.indexOf("```bash exec"), line: 5, column: 1, @@ -370,7 +375,7 @@ describe("Tier WFI — xmd workflow status, list and history", () => { // A Workspace file effect is authored the same way, and says where. const file = entries.find((entry) => entry.event.description?.type === "workspace_file"); expect(file?.source).toEqual({ - path: "flows/release.md", + path: "release.md", offset: RELEASE.indexOf(" { const human = yield* xmd(fixture, ["workflow", "history", "release-1"]).join(); expect(human.code).toBe(0); expect(human.stdout).toContain("EVENT"); - expect(human.stdout).toContain("flows/release.md:5:1"); + expect(human.stdout).toContain("release.md:5:1"); // The root Close is the outcome footer rather than one more operation row. expect(human.stdout).toContain("Outcome: completed at"); }); diff --git a/packages/cli/tests/workflow-installation.test.ts b/packages/cli/tests/workflow-installation.test.ts index 2ff961be5..fe68b0589 100644 --- a/packages/cli/tests/workflow-installation.test.ts +++ b/packages/cli/tests/workflow-installation.test.ts @@ -29,9 +29,9 @@ import { import type { WorkflowExecutionTransitions } from "@executablemd/workflow/deno"; import { Git, WorkflowLifecycle, WorkflowRunStorage } from "@executablemd/workflow"; import type { WorkflowRunDatabase, WorkflowRunStatus } from "@executablemd/workflow"; -import type { Json } from "@executablemd/core"; import { runWorkflow } from "../src/workflow.ts"; import type { WorkflowExecution, WorkflowHost, WorkflowRequest } from "../src/workflow.ts"; +import { readLegacyDefinitionSource } from "../src/workflow-source.ts"; /** * The fixture repository, answered through the Git Api itself. @@ -114,10 +114,10 @@ function useRunStore(): Operation { function recordingHost(root: string, attached: string[]): WorkflowHost { return { useRunHost(): Operation { - return useWorkflowRunHost({ root }); + return useWorkflowRunHost({ root, legacySource: readLegacyDefinitionSource }); }, useLifecycle(): Operation { - return useWorkflowLifecycle({ root }); + return useWorkflowLifecycle({ root, legacySource: readLegacyDefinitionSource }); }, useDelivery(): Operation { return useWorkflowInputDelivery({ root }); @@ -164,7 +164,7 @@ function refusingHost(root: string, refuse: "settle" | "none", attempted: string /** Record a root terminal, so the next pass over this journal is a replay. */ function* closeRoot(root: string, runId: string): Operation { yield* scoped(function* () { - yield* useWorkflowRunHost({ root }); + yield* useWorkflowRunHost({ root, legacySource: readLegacyDefinitionSource }); const found = yield* WorkflowRunStorage.operations.lookup(runId); if (!found.ok) { throw found.error; @@ -180,7 +180,10 @@ function* closeRoot(root: string, runId: string): Operation { /** Put a run into the state a previous invocation would have left it in. */ function* endRun(root: string, runId: string, status: WorkflowRunStatus): Operation { yield* scoped(function* () { - const transitions = yield* useWorkflowRunHost({ root }); + const transitions = yield* useWorkflowRunHost({ + root, + legacySource: readLegacyDefinitionSource, + }); const acquired = yield* WorkflowLifecycle.operations.acquireExecutor(runId); if (!acquired.ok) { throw acquired.error; @@ -214,7 +217,7 @@ function* endRun(root: string, runId: string, status: WorkflowRunStatus): Operat */ function* runSnapshot(root: string, runId: string): Operation { return yield* scoped(function* () { - yield* useWorkflowRunHost({ root }); + yield* useWorkflowRunHost({ root, legacySource: readLegacyDefinitionSource }); const found = yield* WorkflowRunStorage.operations.lookup(runId); if (!found.ok) { throw found.error; @@ -408,9 +411,16 @@ describe("Tier WFI — what a run hands to canonical core", () => { }); expect(outcome.result.exitCode).toEqual(1); - // Nothing was fetched, attached or run — the definition in particular was - // never read out of Git. - expect(asked).toEqual([]); + // The source is authenticated first, because that is the order the + // contract fixes: every version-1 source check happens before recovery + // and before admission, so an unobtainable source wins over any lifecycle + // decision that would otherwise have been reached. + expect(asked).toContain("repositoryRoot"); + expect(asked.some((call) => call.startsWith("readObject:"))).toBe(true); + // And nothing past it happened. The run was refused for what it is, with + // no Workspace attached, no document executed and no lifecycle state + // moved — which is what the refusal has to leave behind whether or not a + // source was proved on the way to it. expect(attached).toEqual([]); expect(executions).toEqual(0); // No status was published for a run whose status did not change. @@ -612,7 +622,14 @@ function* startedRun(root: string): Operation { const objectFormat = (yield* git(repository, ["rev-parse", "--show-object-format"])).trim(); return yield* scoped(function* () { - const transitions = yield* useWorkflowRunHost({ root }); + // The same substituted boundary the cases install. Beginning a version-1 + // run obtains its source now, so a fixture that created one against real + // Git would be creating it under a definition these cases never describe. + yield* useGit(repository, objectId, contents); + const transitions = yield* useWorkflowRunHost({ + root, + legacySource: readLegacyDefinitionSource, + }); const runId = crypto.randomUUID(); const acquired = yield* WorkflowLifecycle.operations.acquireExecutor(runId); if (!acquired.ok) { diff --git a/packages/cli/tests/workflow-retention.test.ts b/packages/cli/tests/workflow-retention.test.ts index fe40edb20..226cfbd47 100644 --- a/packages/cli/tests/workflow-retention.test.ts +++ b/packages/cli/tests/workflow-retention.test.ts @@ -12,7 +12,7 @@ import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { ensure, scoped } from "effection"; import type { Operation } from "effection"; -import { ensureDir, rm, writeTextFile } from "@effectionx/fs"; +import { ensureDir, readTextFile, rm, writeTextFile } from "@effectionx/fs"; import { exec } from "@effectionx/process"; import { randomUUID } from "node:crypto"; import { join } from "node:path"; @@ -119,3 +119,155 @@ describe("Tier FG — workflow retention", () => { }); }); }); + +/** + * Tier FG — the source a run retains, after the file it came from is gone. + * + * The bytes are the run. So the cases here take the original away in each of + * the three ways a caller can — edit it, move it, delete it — and then ask the + * run to continue. It continues from what it kept, every time, and a second + * start of the same bytes under the same logical name is the same run wherever + * on this machine those bytes now happen to sit. + */ +describe("Tier FG — retained source", () => { + /** A line only the retained document says, so a replay of it is visible. */ + const RETAINED_LINE = "this line is the retained source"; + const RETAINED = [ + "# Retained", + "", + RETAINED_LINE, + "", + "```bash exec", + `printf 'retained-output'`, + "```", + "", + ].join("\n"); + + function* startRetained(fixture: Fixture, id: string, at: string): Operation { + const started = yield* runCli(["workflow", "start", `--id=${id}`, at], { + cwd: fixture.repository, + env: { HOME: fixture.home, XMD_WORKFLOW_RUNS: fixture.runs }, + }).join(); + expect(started.code).toBe(0); + } + + function resume(fixture: Fixture, id: string) { + return runCli(["workflow", "resume", id], { + cwd: fixture.repository, + env: { HOME: fixture.home, XMD_WORKFLOW_RUNS: fixture.runs }, + }).join(); + } + + /** One way a caller can take the original away, and what it is called. */ + interface Disturbance { + readonly name: string; + disturb(fixture: Fixture, at: string): Operation; + } + + const DISTURBANCES: readonly Disturbance[] = [ + { + name: "edited", + *disturb(_fixture: Fixture, at: string): Operation { + yield* writeTextFile(at, "# Something else entirely\n"); + }, + }, + { + name: "moved", + *disturb(fixture: Fixture, at: string): Operation { + yield* writeTextFile(join(fixture.repository, "flows/moved.md"), RETAINED); + yield* rm(at, { force: true }); + }, + }, + { + name: "deleted", + *disturb(_fixture: Fixture, at: string): Operation { + yield* rm(at, { force: true }); + }, + }, + ]; + + it("FG40: an edited, moved or deleted original changes nothing about the run", function* () { + for (const { name, disturb } of DISTURBANCES) { + yield* useFixture(function* (fixture) { + const at = join(fixture.repository, "flows/retained.md"); + yield* writeTextFile(at, RETAINED); + yield* startRetained(fixture, `retained-${name}`, at); + + yield* disturb(fixture, at); + + const resumed = yield* resume(fixture, `retained-${name}`); + expect({ name, code: resumed.code }).toEqual({ name, code: 0 }); + // The retained document, rendered from the run rather than from a file + // that no longer says it — or no longer exists at all. + expect(resumed.stdout).toContain(RETAINED_LINE); + expect(resumed.stdout).not.toContain("Something else entirely"); + // And the command's own retained result, which is what a replay reads + // back instead of running it again. + expect(committedExec(workflowRunPath(fixture.runs, `retained-${name}`)).stdout).toBe( + "retained-output", + ); + }); + } + }); + + it("FG43: a damaged retained source refuses, and never falls back to the file", function* () { + yield* useFixture(function* (fixture) { + const at = join(fixture.repository, "flows/fallback.md"); + yield* writeTextFile(at, RETAINED); + yield* startRetained(fixture, "fallback-1", at); + + // The run's own retained content stops describing itself. The file it was + // started from is untouched and still says exactly what it always said — + // which is the whole point: a resume that read it would succeed, and a + // resume that must not read it refuses. + const store = workflowRunPath(fixture.runs, "fallback-1"); + const database = new DatabaseSync(store); + try { + const row = database.prepare("SELECT content FROM workflow_definition_blob").get(); + const bytes = row?.["content"]; + if (!(bytes instanceof Uint8Array)) { + throw new Error("the run retains no source bytes"); + } + const altered = Uint8Array.from(bytes); + altered[0] = altered[0] === 0x23 ? 0x2a : 0x23; + database.prepare("UPDATE workflow_definition_blob SET content = ?").run(altered); + } finally { + database.close(); + } + expect(yield* readTextFile(at)).toBe(RETAINED); + + const resumed = yield* resume(fixture, "fallback-1"); + expect(resumed.code).toBe(1); + expect(resumed.stderr).toContain("disagrees with its own descriptor"); + expect(resumed.stdout).not.toContain(RETAINED_LINE); + }); + }); + + it("FG42: another entrypoint, or other bytes, is another run and conflicts", function* () { + yield* useFixture(function* (fixture) { + yield* writeTextFile(join(fixture.repository, "flows/named.md"), RETAINED); + yield* startRetained(fixture, "named-1", join(fixture.repository, "flows/named.md")); + + const environment = { HOME: fixture.home, XMD_WORKFLOW_RUNS: fixture.runs }; + + // The same bytes under a different logical name: a different definition, + // because source positions and later relative references use the name. + yield* writeTextFile(join(fixture.repository, "flows/renamed.md"), RETAINED); + const renamed = yield* runCli( + ["workflow", "start", "--id=named-1", join(fixture.repository, "flows/renamed.md")], + { cwd: fixture.repository, env: environment }, + ).join(); + expect(renamed.code).toBe(1); + expect(renamed.stderr).toContain("definition"); + + // The same name over different bytes: also a different definition. + yield* writeTextFile(join(fixture.home, "named.md"), `${RETAINED}\nand one more line\n`); + const changed = yield* runCli( + ["workflow", "start", "--id=named-1", join(fixture.home, "named.md")], + { cwd: fixture.repository, env: environment }, + ).join(); + expect(changed.code).toBe(1); + expect(changed.stderr).toContain("definition"); + }); + }); +}); diff --git a/scripts/tests/plugin-compiled.test.ts b/scripts/tests/plugin-compiled.test.ts index b848e87b5..6dc9f5044 100644 --- a/scripts/tests/plugin-compiled.test.ts +++ b/scripts/tests/plugin-compiled.test.ts @@ -18,8 +18,8 @@ import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; -import { ensure, until } from "effection"; -import { exists, rm, writeTextFile } from "@effectionx/fs"; +import { ensure, scoped, until } from "effection"; +import { ensureDir, exists, rm, writeTextFile } from "@effectionx/fs"; import { exec } from "@effectionx/process"; import { timebox } from "@effectionx/timebox"; import type { ProcessResult } from "@effectionx/process"; @@ -164,3 +164,72 @@ describe("compiled xmd", { sanitizeOps: false, sanitizeResources: false }, () => }); }); }); + +/** + * The compiled binary starting and resuming a file outside any repository. + * + * The behaviour #443 exists for, through the artifact that ships. The compiled + * host is Deno too, so what this proves is not a second implementation — it is + * that nothing the compile step does to module resolution, to the bundled + * SQLite, or to the Git capability's absence stops a run whose source is its + * own bytes. + * + * Deliberately not in the checkout and deliberately not a repository: the + * temporary directory has no `.git` anywhere above it that this run may use, + * and the document is removed before the resume so nothing on disk could + * answer for it. + */ +describe( + "compiled workflow source bundles", + { sanitizeOps: false, sanitizeResources: false }, + () => { + it("starts and resumes a document outside any repository", function* () { + if (!(yield* exists(BINARY))) { + throw new Error(`${BINARY} is missing — run \`deno task build\` before this case`); + } + + yield* scoped(function* () { + const dir = yield* until(mkdtemp(path.join(tmpdir(), "xmd-compiled-bundle-"))); + yield* ensure(() => rm(dir, { recursive: true, force: true })); + + const runs = path.join(dir, "runs"); + const home = path.join(dir, "home"); + yield* ensureDir(runs); + yield* ensureDir(home); + + const document = path.join(dir, "release.md"); + yield* writeTextFile(document, "# Release\n\nretained by the run\n"); + + const environment = { HOME: home, XMD_WORKFLOW_RUNS: runs }; + const started = yield* timebox(TIMEOUT, function* () { + return yield* exec(BINARY, { + arguments: ["workflow", "start", "--id=compiled-1", document], + cwd: dir, + env: environment, + }).join(); + }); + if (started.timeout) { + throw new Error("the compiled binary timed out starting a source-bundle run"); + } + expect(started.value.code).toBe(0); + expect(started.value.stdout).toContain("retained by the run"); + + // The document is gone. The run is not. + yield* rm(document, { force: true }); + + const resumed = yield* timebox(TIMEOUT, function* () { + return yield* exec(BINARY, { + arguments: ["workflow", "resume", "compiled-1"], + cwd: dir, + env: environment, + }).join(); + }); + if (resumed.timeout) { + throw new Error("the compiled binary timed out resuming a source-bundle run"); + } + expect(resumed.value.code).toBe(0); + expect(resumed.value.stdout).toContain("retained by the run"); + }); + }); + }, +); From 0bcb87c1d0cdf53cf791bf8a61e26f4aade23a62 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Thu, 17 Sep 2026 03:00:02 -0400 Subject: [PATCH 4/8] =?UTF-8?q?=F0=9F=93=9D=20Specify=20source-bundle=20wo?= =?UTF-8?q?rkflow=20durability?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A workflow definition is two versions now, and the five authoritative documents say so. Version 2 is a source bundle: the exact bytes themselves, addressed by logical paths and retained with the run, so a file outside a repository, an untracked file and a file edited since its last commit all start and all stay runnable after the original is edited, moved or deleted. Version 1 is unchanged in shape, identity, behavior and bytes, and nothing migrates between them. `specs/workflow-spec.md` states the closed descriptor union and its dispatch by kind, the logical-path grammar, the domain-separated source and bundle hashes and what each commits to, the target's exclusion from the bundle hash and its place in identity, per-version compatible reuse, the two immutable live schema versions and their recognition, what a retained source has to prove, the creation union with its owned copies and one atomic transaction, the order that puts every source check before recovery and admission, the legacy source reader and its validation, and the five source refusals. `specs/xmd-artifact-spec.md` keeps the physical container at schema version 1 and splits the semantic format: format 1 retains its manifest, identity domain and Git closure byte for byte, and format 2 declares its own manifest version, `xmd-artifact\0v2\0` domain, closed inventory and one entry/content pair per logical source. The reader selects the verifier from the header and never reads one version's closure as the other's. `specs/workflow-workspace-spec.md` states the start ordering, the boundary that proves source before recovery, execution records, Workspace attachment or replay, the reference grammar `start` accepts, the fork candidate's independence from the source run and from every file it was read from, and the status shapes that carry no fabricated Git fields. `architecture.md` records the terminology, the two live schema versions, the CLI candidate against the authoritative retained source, and the inventory rows for the definition, the reader and both artifact formats. Two things are stated as undelivered rather than described as done: the artifact-backed fork, which this revision supplies the authenticated source input for, and #822's extraction of `workflowInstallation({ base })` — still the one place this package reaches Git to establish a run. Refs #443 --- architecture.md | 169 +++++++++---- specs/executable-mdx-spec.md | 17 +- specs/workflow-spec.md | 420 +++++++++++++++++++++++++++---- specs/workflow-workspace-spec.md | 224 ++++++++++++----- specs/xmd-artifact-spec.md | 125 ++++++--- 5 files changed, 767 insertions(+), 188 deletions(-) diff --git a/architecture.md b/architecture.md index 72e8df0a3..1ac442b8a 100644 --- a/architecture.md +++ b/architecture.md @@ -34,14 +34,17 @@ and categorization rather than clarify them, so both stay exactly as written. | middleware | applied by the lexical structure, used by runtime execution | | workflow run | a workflow being carried out with its progress and outcome recorded durably; document executions perform its work, while ongoing effects remain scoped to the document execution in which they run | | document execution | one evaluation of a root document initiated through `execute()`, producing one output stream and one completion result while reading and appending a durable journal; its ongoing effects belong to the Effection scope in which the evaluation runs | -| workflow definition | what a workflow run is a run of: a versioned descriptor naming an immutable object — its format and object ID — together with the repository-relative path of the root document inside it, the exact canonical document target when one is selected, and the component bundle it is closed over when it declares one. A repository locator is not part of it, and it is distinct from every Repository created inside the run's Workspace | -| workflow component bundle | the closed set of authored Markdown components a workflow root declares, each named, located by a canonical repository-relative path inside the pinned commit, and identified by that blob's object ID. It is both definition identity and canonical resolution: the names it declares resolve to its exact pinned sources and to nothing else | +| workflow definition | what a workflow run is a run of: a closed versioned descriptor, in one of two versions. Version 1 names an immutable Git object — its format and object ID — together with the repository-relative path of the root document inside it. Version 2 is a **source bundle**: the exact bytes themselves, addressed by logical paths and retained with the run. Both optionally carry the exact canonical document target and the component bundle the root declares. A repository locator is not part of either, and neither is distinct from every Repository created inside the run's Workspace | +| source bundle | a version-2 workflow definition: an entrypoint, a canonically ordered manifest of logical paths with each source's SHA-256 and byte length, and a bundle hash over that manifest. The bytes behind it are retained with the run, so a file outside a repository, an untracked file and a file edited since its last commit are all ordinary starts, and editing, moving or deleting the original afterwards changes nothing about the run | +| logical path | a portable identity inside a source bundle, never a host filesystem path: NFC Unicode scalar values, `/`-separated, without leading or trailing separator, NUL, C0 control, DEL, backslash or `#`, and without an empty, `.` or `..` segment, compared case-sensitively by its UTF-8 bytes. The containing directory, the absolute path and the invocation working directory are retrieval facts and never identity | +| workflow component bundle | the closed set of authored Markdown components a workflow root declares. A version-1 bundle locates each by a canonical repository-relative path inside the pinned commit and identifies it by that blob's object ID; a version-2 bundle maps each name onto a logical path the same source bundle retains. It is both definition identity and canonical resolution: the names it declares resolve to its exact retained sources and to nothing else | | declared Markdown component | exact first-party Markdown a trusted host declares to one document execution, read once and held by the invocation — schemas copied, not referenced — before any installation runs, as immutable data on an `ExecutionInstallation`: the public name, the reported origin, the bytes, the SHA-256 of those bytes, the accepted forms, the contract parsed from them, and any private component closure only those bytes may write. The host claims the name rather than offering a default for it, so it resolves ahead of a repository file, a workflow component bundle and every registration; a second claim on one name is a configuration failure. It is a composition mechanism and not a policy loader: no caller-facing option selects one, adds one, or names its source | | private component closure | the components one declared Markdown component carries that only its own bytes may write. Each is minted like any other invocation-identity component and registered nowhere, so it resolves while canonical core is expanding that declaration's body and nowhere else — not from the caller's root, the content the caller projected through it, a sibling declaration, an imported component, middleware, or an implementation kept past teardown. Eligibility is the authored occurrence rather than the name: what an import may invoke is what canonical core produced inside that exact ask; an implementation the closure built is refused for every other name, copy, later site and later execution, because a run that has ended admits nothing; and a private name written anywhere else resolves to nothing before any other tier can answer. It is lexical availability, not permission: every private component still takes its operation from the invocation that carries it | | retrieval metadata | replaceable, credential-free information about where a workflow definition can be fetched from now; it takes no part in run identity and is reauthorized by the host before use | | stop reason | why a workflow run or a document execution stopped: a categorical host code, or a reference to an already-filtered journal event | | run ID | an opaque stable public identifier generated by the host or selected by an authorized caller; it associates the run's durable records and effects, remains unchanged for the life of the run, and has no semantics beyond equality and lifecycle addressing | -| definition base | the Git revision supplied to choose a workflow definition's pinned commit | +| definition base | the Git revision supplied to choose a version-1 workflow definition's pinned commit. A source-bundle run has none | +| legacy workflow source reader | the direct, non-contextual capability a trusted host supplies so the retained lifecycle can obtain a version-1 definition's Markdown. It is captured before document code runs and reachable through no Context, Api, component, Plugin installation result or authored value; Workflow validates everything it returns against the descriptor it asked about | | Repository base | the optional Git revision from which one named Repository initializes its primary checkout | | Repository selection | plain structural composition data naming the repository one component invocation acts on: an opaque provider-minted selection identifier, the display name, the credential-free repository identity, and the selected checkout path. It carries no credential, provider handle, lock, database, run ID or permission — the installed provider authenticates every selection against private state before it touches Git or a service, so a copied, replaced or rebuilt one can misname a target and be refused but can never reach one | | ambient Repository | the repository an ordinary `xmd run` was started inside, discovered once before root expansion from the invocation's starting directory. Its identity is the canonical common Git directory and its selected checkout is the canonical checkout root, so starting in a linked worktree names the same repository as starting in the primary checkout while Git operations still act on the worktree. A workflow run has none | @@ -84,7 +87,7 @@ and categorization rather than clarify them, so both stay exactly as written. | recovered inspection snapshot | an immutable lifecycle snapshot read from a private scratch copy after SQLite rolls that copy's hot journal back to the retained store's last committed state; the retained database and journal are not changed | | XMD artifact | one immutable portable evidence file with the `.xmd` extension, containing one workflow run's committed retained state at one artifact frontier together with its workflow definition source closure; it is neither a live run store nor permission to continue the source run | | artifact frontier | the exact committed lifecycle snapshot, journal boundary and current Workspace root an XMD artifact records | -| workflow definition source closure | the exact root Markdown source and every source in its declared workflow component bundle, each authenticated against the immutable object identity in the workflow definition; retrieval metadata and a repository checkout are not part of the closure | +| workflow definition source closure | the exact root Markdown source and every source in its declared workflow component bundle, in the form the run's own definition version retains: a version-1 closure is authenticated against the immutable object identity in the definition, and a version-2 closure is the retained bytes re-derived from the run's own store. Retrieval metadata, provenance and a repository checkout are not part of either | | artifact manifest | the canonical versioned inventory of every semantic record and retained byte in one XMD artifact, excluding its own derived identity and physical container layout | | XMD artifact identity | the lowercase SHA-256 of an XMD artifact's artifact manifest; it identifies the evidence independently of its host path and physical encoding and is retained as the source identity of a history fork made from that artifact | | Agent session portability evidence | what one XMD artifact states about every logical workflow Agent session that contributed a retained Prompt: either complete portable evidence — the session and its compatibility attributes, the provider, the bundle kind and compatibility identifier, the source and bundled provider-session identities, the identity-allocation mode, the ordered provider checkpoint tokens, and the Agent session bundle's length, hash and bytes — or an explicit unavailable marker naming why. A session that retained no Prompt has none. It is detached retained evidence about a conversation, never a live provider session and never control over the host that produced it | @@ -192,13 +195,16 @@ The workflow run exists once that value is durably recorded. A document failure or cancellation after that point does not erase it. Failure before that point creates no workflow run and the root document does not expand. -`WorkflowRun.base` and `WorkflowRun.pinnedCommit` identify the source repository -state containing the workflow definition. They do not create or seed a -Workspace Repository. Each `` inside the definition resolves and -pins its own optional Repository base, so one workflow may compose repositories -unrelated to the definition repository or to one another. +`WorkflowRun` is a closed union. A version-1 run carries `base` and +`pinnedCommit`, which identify the source repository state containing the +workflow definition; a version-2 run carries `definitionVersion`, `bundleHash` +and the exact target when it has one, and invents neither. Neither creates or +seeds a Workspace Repository. Each `` inside the definition resolves +and pins its own optional Repository base, so one workflow may compose +repositories unrelated to the definition repository or to one another. -Base resolution goes through the contextual Git capability: +Base resolution — for a version-1 run only — goes through the contextual Git +capability: ```ts interface GitApi { @@ -209,11 +215,21 @@ interface GitApi { `Git.revParse(revision)` has the semantics of `git rev-parse --verify --end-of-options ` in the contextual working directory. Its default provider invokes the Git CLI; another provider may -replace it lexically. Workflow initialization calls it with +replace it lexically. `workflowInstallation({ base })` calls it with `${base}^{commit}`, which verifies that the result is a commit and returns its -full object ID. Starting a workflow run fails before root expansion when Git +full object ID, and it is the only caller of that capability left in the +workflow package. Starting such a run fails before root expansion when Git cannot be invoked, the working directory is not a Git repository, or the base -does not resolve to a commit. Ordinary `execute()` remains Git-independent. +does not resolve to a commit. Ordinary `execute()` remains Git-independent, and +so does starting, resuming, forking or exporting a source-bundle run. + +**Retained workflow identity and retained source retrieval require no Git.** The +CLI establishes a version-2 candidate from filesystem bytes, and the retained +lifecycle re-derives a version-2 source from the run's own store. Version-1 +source crosses the host-supplied legacy reader, which is a direct dependency +rather than a package-level import. The one remaining exception is +`workflowInstallation({ base })` itself, which issue #822 moves into the bundled +Git Plugin; nothing else in the retained path names Git. Replay restores the recorded `WorkflowRun` without allocating another run ID or invoking Git. The supplied base must equal the recorded base. Git is not @@ -225,9 +241,11 @@ A host that has already created the run's storage record passes Nothing is left for the execution to decide: it records that value through the same `workflow_run` durable operation, allocates no identifier and resolves no base, and every journal state — live, truncated and completed — requires the -recorded run to agree with the supplied one in run ID, base and pinned commit. A -journal that disagrees in any of them is not this run's journal, and the refusal -names the fields rather than their values. +recorded run to agree with the supplied one in every member of its own version: +run ID, base and pinned commit for version 1; run ID, bundle hash and exact +target for version 2. A journal that disagrees in any of them, or that records +the other version, is not this run's journal, and the refusal names the fields +rather than their values. `getWorkflowRun()` returns the frozen `WorkflowRun` for the current document execution. Every call in one live execution returns the same object. It throws @@ -270,8 +288,12 @@ they can reach SQLite. ### Identity is separate from retrieval Identity is the run ID, the whole workflow definition including its descriptor -version, the definition base, and the normalized props. Values are compared -canonically, so props differing only in key order are the same props. +version and kind, the definition base for a version-1 run, and the normalized +props. Values are compared canonically, so props differing only in key order are +the same props. Each version is compared by its own complete identity — a +version-2 comparison covers the bundle hash, the entrypoint, the whole canonical +source manifest, the component mapping and the exact target — and two +descriptors of different versions are never one run. Everything a run accumulates is excluded from that comparison: status, stop reason, retrieval metadata, timestamps, document executions and journal @@ -318,6 +340,32 @@ content-addressed root manifest describing only the root directory, empty retained manifest and blob reference sets, and the corresponding root-only DOFS frontier. +**Schema version 2 is the second immutable inventory, not an amendment of the +first.** It keeps every version-1 object byte for byte except `workflow_run`, +replaces that table with one that has no `base` column and a `definition` CHECK +pinning version 2 and kind `source-bundle`, and adds exactly two tables holding +the run's own source: content keyed by its SHA-256 with its byte length, and one +manifest row per descriptor entry mapping a logical path to that hash. Content +is retained as BLOB bytes rather than as text or re-encoded JSON, and two paths +holding identical bytes reference one blob. + +A new Git run is written at `user_version = 1` and a new source-bundle run at +`user_version = 2`; initialization selects the version from the candidate +definition already parsed and never upgrades an existing database. Recognition +reads the application ID first, then dispatches on those two versions and holds +the whole inventory to that version's declaration. Version 0 under either +declared inventory is corruption, any other version is unsupported, and a +hybrid, an altered object or an extra object is corruption rather than a +migration candidate. + +Before a version-2 record is exposed, its retained source is re-derived in full: +the manifest equals the descriptor's `sources`, every referenced blob exists, +both retained lengths equal the content's own, recomputing each blob's hash +produces its key, no blob is unreferenced, and recomputing the bundle hash from +the retained manifest and mapping produces `bundleHash`. No reader returns a +partial bundle, and nothing falls back to the original file, the provenance or +the legacy reader. + A Workspace root is a complete, immutable filesystem checkpoint. Its canonical format-1 JSON includes `/` and every reachable absolute POSIX path in UTF-8 byte order, with kind, mode, observable mtime, file size and DOFS manifest identity, @@ -451,30 +499,47 @@ incomplete workflow for a finished one. A request the command refuses — bad grammar, a missing run, an incompatible reuse, damaged storage, an unsupported host — exits 1. -`start` establishes an immutable definition from Git rather than identifying a -working-tree file. It locates the repository containing the supplied path, -resolves `HEAD^{commit}` once because the command has no base option, reads the -repository's object format, and stores version 1 of the descriptor with that -format, the lowercase commit ID and the normalized repository-relative POSIX -path. **The bytes that execute are the ones that commit holds**, so a working -tree with uncommitted edits runs the committed document. Where the repository is -checked out is retrieval metadata: replaceable, credential-free, excluded from -identity, and reauthorized before it is used again. `resume` loads exactly the -retained object through that locator and never substitutes the current `HEAD` or -a same-named working-tree file. A missing object, missing or unreadable -retrieval metadata, a path outside the repository, or a root that is not -Markdown fails explicitly; none of them creates a replacement run or an empty -definition. A workflow definition is one immutable object, so the component -search path is empty and a repository component fails to resolve rather than -resolving to content beside the definition in a mutable checkout. +`start` establishes an immutable definition from **the bytes the caller named**. +Its argument is a document reference — a path, optionally followed by `#` and one +target selector — so the reference is taken apart before anything is resolved as +a path. It reads the supplied file once, derives the logical entrypoint from that +file's own NFC-normalized final segment, decodes it strictly, parses it, resolves +any selector through core against those exact bytes to the one canonical target, +reads the declared bundle from the root's own directory, and builds and verifies +version 2 of the descriptor — all before storage exists. + +**The bytes that execute are the bytes the file held**, so a file outside a +repository starts, an untracked file starts, and a file edited since its last +commit runs what it says. Where the file sat is never identity: the containing +directory, the absolute path and the invocation working directory are retrieval +facts. Git is optional provenance, recorded as replaceable metadata when it is +cheaply available; failing to obtain it cannot fail a start, and nothing reads it +back to find the source. + +`resume` executes the closure the lifecycle authenticated from the run's own +store and never rereads the candidate path. A file that is unreadable, not +well-formed UTF-8, not Markdown, whose selector resolves to nothing, or that +declares a component this command cannot read fails explicitly and before +storage exists; none of them creates a replacement run or an empty definition. A +workflow definition is one closed set of retained bytes, so the component search +path is empty and a repository component fails to resolve rather than resolving +to content beside the definition in a mutable checkout. + +A version-1 run remains resumable, replayable and exportable exactly as it was. +Its Markdown crosses the host-supplied legacy source reader, and Workflow +recomputes every returned blob identity from the bytes that came back before any +of it executes. A root may close itself over a **component bundle** by declaring one in its own frontmatter — `workflow.components`, a non-empty mapping from a component name to a relative POSIX Markdown path beside the root. `start` normalizes that -declaration against the root's own directory, reads every member from the same -pinned commit through `git cat-file`, takes each blob's own object ID as its -source hash, parses each component before the run exists, and stores the bundle -on the definition as a `components` array sorted by name. The bundle is a member +declaration against the root's own directory, reads every member from beside the +root as exact bytes, takes each source's own domain-separated content hash as +its source hash, +parses each component before the run exists, and stores the bundle on the +definition as a `components` array in canonical order — so each declared path is +both the logical path the run retains and the file this host happened to find it +at. The bundle is a member of the one descriptor rather than a version past it: a root that declares nothing writes no `components` member at all, so a definition retained before bundles existed still reads and still means "closed over no components". An @@ -486,11 +551,15 @@ only once the name has passed the grammar a document writes. The bundle is identity, so compatible reuse compares the whole array: a changed name, canonical path, source hash, or component set conflicts as `definition`, and a run closed over a bundle is never the same run as one closed over none. -`resume` reconstructs the -bundle under the executor lock, from the retained commit, verifying every blob -against the retained hash before the lifecycle advances — so a component that -changed, went missing, or became unreachable leaves the run's records exactly as -they are. +`resume` obtains the bundle under the executor lock and before the lifecycle +advances, by the version the run retains. A version-2 run re-derives it from its +own retained BLOBs, recomputing every source hash and the bundle hash from the +bytes it holds; nothing is read from a repository or from the files the run was +started beside. A version-1 run reconstructs it from the pinned commit through +the host-supplied legacy source reader, and every returned blob identity is +recomputed from the bytes that came back. Either way a component that changed, +went missing, or became unreachable leaves the run's records exactly as they +are. The bundle is also canonical resolution. It reaches canonical core as plain immutable data on the `ExecutionInstallation` that already carries the run's @@ -2554,6 +2623,12 @@ selector while inspecting the document, then asks execution for the exact target that resolved — so a file replaced between the two reads fails on the target the run chose, rather than silently running whatever the glob would name now. +`xmd workflow start` is the second, and it is stricter because its answer is +retained. It resolves the selector once, against the exact bytes it is about to +retain, and stores only what core answered. A resume re-enters that section +without resolving anything again: the glob is gone, and the bytes it was +resolved against are the run's. + The workflow definition is the second. It optionally carries that exact target, compares it with the rest of the descriptor, and validates it through core's own canonical-target predicate rather than a rule the workflow package restates — @@ -5359,10 +5434,12 @@ Status is measured against main. | document-aware `xmd run … --help` | describes what one document declares and every target it addresses, each as a full document reference with the description its section states, by inspection alone | built on the #463 stack | | standard-input root documents | `xmd run -` and `xmd run -- -` read the whole root document from standard input, once, to end of file, and run it through the ordinary run profile. Fixed grammar selects it — the explicit `run` command form plus a document argument that is exactly `-`, read from the parser's own unconsumed remainder so a `-` another option took as its value is not one — and every other spelling keeps the meaning it had: the shorthand `xmd -` executes the file named `-`, `xmd run -#Section` executes that file's `Section`, another command's `-` is that command's, and `--eval -` keeps its refusal. `-` is the one filename the option grammar leaves unwritable, so the reference grammar reaches it and nothing else beginning with `-` is read as a document. The parsed path and every recovered reference stay separate facts until the grammar is settled, so a command line naming two roots refuses in either order, before the read and before either candidate is inspected. The reader is a value each runtime-named entrypoint supplies and the shared CLI never reaches a stdin global; what comes back is `retainedSource("", source)`, adding no root-source variant, constructor, digest member or public API. The complete input is acquired before inspection, provider setup, the secret-detection announcement, journal creation, root admission and execution, inside the run's existing deadline; a failed read is one fixed sentence carrying no host error, input or path, and cancellation tears the reader down without becoming one | built on this stack | | targeted `xmd run` | reads a file argument as a document reference and executes the one exact target its selector resolved to, replacing the selector before execution rereads the file | built on the #412 stack | -| targeted workflow definition | the V1 workflow definition optionally carries the exact canonical document target, which takes part in definition identity and in compatible reuse | built on the #412 stack; the workflow CLI does not supply one yet | +| targeted workflow definition | a workflow definition of either version optionally carries the exact canonical document target, which takes part in definition identity and in compatible reuse but not in the source-bundle hash | built on the #412 stack; `xmd workflow start` takes a document reference and resolves any selector through core against the bytes it is about to retain | +| source-bundle workflow definition | version 2 of the workflow definition is the exact bytes themselves: an entrypoint, a canonically ordered manifest of logical paths with each source's domain-separated SHA-256 and byte length, a bundle hash over that manifest and mapping, and the optional exact target and component mapping. The bytes are retained with the run in live schema version 2, so starting, resuming, replaying, forking and exporting need no repository and survive the original being edited, moved or deleted | built on the #443 stack; version 1 is unchanged, nothing migrates between them, and the artifact-backed fork remains unbuilt | +| legacy workflow source reader | a version-1 run's Markdown is obtained through a direct capability a trusted host captures before document code runs, reachable through no Context, Api, component, Plugin result or authored value. Workflow validates the returned closure against the descriptor it asked about, recomputing every blob identity from the bytes that came back | built on the #443 stack; the Deno and compiled hosts supply the Git-object adapter, and #822 moves it into the bundled Git Plugin | | installed structural syntax | a trusted host declares a structural construct and the regions written directly inside it, as the second arm of the same `declarations` list: name, origin, accepted forms, props schema, syntax examples, description, what the content means, and `parent` — `null` for the construct, the construct's name for one of its regions. The accepted regions are derived from the regions that named the construct rather than restated by it. The installation that declares them supplies the one `expand` handler, read once and bound at capture, and held on the admitted catalog entry rather than in any registry, context or second dispatch protocol. Admission refuses an invalid name, an empty origin, absent or non-canonical forms, an uncompilable schema, missing syntax examples or description, a context that is neither prose nor `null`, a name the engine's structural table or the canonical protected tier owns, a name a reserved registration claims, a duplicate across either arm or across installations, a collision with a private closure name, an orphan region, a region of a region, a pair split across two installations, a construct with no region, declarations with no handler, and a handler with no declarations — all of it before the root document is read. Resolution places a declared construct in the host tier beside declared Markdown; `inspectComponent()`, `inspectSyntax()` and document validation describe and check it from the same catalog, with the structural entries sorted by code point beside the engine's own and the symbols unchanged at version 2. Expansion asks the catalog after every engine branch and before component import, settles placement, forms, `as`/`slot` and every construct and region prop, and then calls the captured handler once inside its own scope with a frozen request. Each region is an operation whose subscription owns one demand-driven producer: a chunk is delivered only to a waiting read, authored work waits for the read that permits it, output never reaches `DocumentOutput` or the journal, and leaving the handler's scope halts and joins every producer it entered | built on the #806 stack | | declared Markdown component | a trusted host declares exact first-party Markdown to one execution as immutable data on an `ExecutionInstallation`, built with `Markdown({…})`: the host states the name, origin, source, its SHA-256, the accepted forms, an optional statement of the props and return that must agree with the parsed source, and an optional private component closure, and the constructor returns a fresh declaration carrying the required `kind: "component"` written after that description. It validates, hashes, copies and freezes nothing. Admission reads the kind before any other member and refuses a missing, unknown or superseded one — including the `"markdown"` earlier versions stated — rather than reading the value as Markdown, then parses the bytes and refuses a mismatched digest or schema, a non-canonical form, a name that is not a component name or is structural, a duplicate, a reserved-registration collision and a private name a registration also claims. Resolution places it in the protected tier with reserved registrations, above the workflow component bundle, repository files and every registered default. Live import and retained history are held to the declared origin, digest and bytes, private names resolve only while canonical core expands the declaring bytes' own body — by the authored occurrence rather than by the name, so an answer kept from a legitimate private import authorizes no later site, no alias, no copy of the definition, no invocation that is over and no later execution — including one that declares no Markdown at all — while a private name written anywhere else resolves to nothing before the bundle, the repository or a registration can answer for it — and `xmd syntax` and document validation describe the declared contract from the same declaration without describing the closure. Closure is per name: only the declared component and its private closure become canonical imports, and every other name in the execution stays the ordinary open import middleware may still answer | built on the #660 stack; no public component uses it yet (#660 PR 2) | -| workflow component bundle | a workflow root declares a closed set of authored Markdown components; the V1 workflow definition optionally carries them as one array sorted by component name, each entry holding the name, its canonical repository-relative path inside the pinned commit and that blob's object ID, and an absent member identifies a run closed over no components — so a definition retained before the member existed reads unchanged. `start` and `resume` read every component from the definition's own pinned commit; the array takes part in definition identity and is compared as part of the same V1 descriptor in compatible reuse; and canonical core resolves those names and holds both live import and retained history to that exact bundle | built on the #301 stack; the full adversarial implementation loop and its scheduling remain unbuilt (#300), and generated XMD admits no bundled Markdown component (#369) | +| workflow component bundle | a workflow root declares a closed set of authored Markdown components; the workflow definition optionally carries them as one canonically ordered array, and an absent member identifies a run closed over no components — so a definition retained before the member existed reads unchanged. A version-1 entry holds the name, its canonical repository-relative path inside the pinned commit and that blob's object ID; a version-2 entry maps the name onto a logical path the same source bundle retains. `start` reads every component from beside the root and retains it, and `resume` executes the closure the lifecycle authenticated; the array takes part in definition identity and is compared as part of the same descriptor in compatible reuse; and canonical core resolves those names and holds both live import and retained history to that exact bundle | built on the #301 stack; the full adversarial implementation loop and its scheduling remain unbuilt (#300), and generated XMD admits no bundled Markdown component (#369) | | `workflowInstallation()` / `getWorkflowRun()` | associates one document execution with a workflow run, through an `ExecutionInstallation` the trusted host passes to `executeInstalled()` | built on the #366 stack | | `retainedWorkflowInstallation()` | associates one document execution with a run storage already created, requiring exact journal agreement | built on the #366 stack | | `Git.revParse()` | verifies and resolves one Git revision expression contextually | built on main | @@ -5404,7 +5481,7 @@ Status is measured against main. | ordinary repository provider assembly | the Deno source entrypoint and the compiled binary install the live provider for `xmd run`, parameterized by the same credential-helper assembly the workflow host uses and by the two existing host configurations, `XMD_WORKFLOW_GITHUB_ISSUES` and `XMD_WORKFLOW_GITHUB_PULL_REQUESTS`, both read and validated before a document runs. A nested `` child receives a fresh instance — its own invocation identity, leases and Push evidence — so nothing it publishes authorizes its parent or a sibling. The outer `xmd test` command and a workflow execution install none. Node and Bun register the vocabulary and install no operational provider, so every repository operation reports an absent provider before a lock, a credential, a subprocess or a request exists | built on the #643 stack | | `` under a workflow run | asks one of two questions, decided by its own shape, through a boundary of its own rather than the Git host's. Self-closing with `url` reads that issue and binds `{ url, title, description, tags, assignee }`; paired with `title` upserts and binds exactly `{ url }`, its rendered content being the description. There is no `description` prop. Props are exactly `url`, `title`, optional `tags`, optional `assignee` and — on a read only — optional `provider`; no repository/token/label/milestone/project/comment/close or approval prop. Both forms render nothing. The form is decided before the tracker is read, before any provider is asked and before an `issue_effect` record exists, and that is where a mixed `url`+`title`, a read carrying content or `tags`/`assignee`, an upsert with no content, an upsert naming a `provider`, and an element that is neither are all refused. A read needs no tracker — its URL is the identity; an upsert requires the nearest lexical `` and takes its discriminator only from there. The tracker carries a credential-free `url` and an optional `provider`; the URL is canonicalized — a credential, a query and a fragment are refused rather than stripped — and a nested tracker replaces the whole value for its descendants, never merging members, with the enclosing one restored on leaving. It is composition data, not permission: the provider holds an adapter-private ceiling beside its credentials, admitted before it connects, so a target outside it sends nothing. One stable contextual operation, `executablemd.workflow.issue`, with `read(url, options)` and `upsert(issue, options)`; a provider is ordinary middleware around it, matching its own URLs without a discriminator and only its own name with one, independently per member, with no host-side resolution. Once middleware matches it owns the answer — it never delegates afterwards, and nothing catches its refusal to try somebody else — and a request everyone delegated reaches `NoIssueProvider` unchanged. `issue_effect` records an operation discriminator with the normalized request and result; both forms replay without reaching `IssueApi` and therefore without network access; only an upsert derives an idempotency key, from the operation, the canonical target and the run's own effect identity. Retention excludes credentials, endpoints, payloads, provider identities, origin markers and host paths. Observing, adopting, creating once and recovering an interrupted creation are the provider's, because they are knowledge about what a service can prove; title is never identity, and tags are a code-point-sorted set. The Deno workflow host installs configured GitHub middleware and installs none otherwise, so absence of configuration is fail-closed | built on the #296 stack; GitHub middleware, Deno host | | workflow lifecycle inspection and control | reads status/list/history without advancing a run, recovering a private copy when a crashed source needs rollback; enforces the executor lock, refuses live cancellation, cancels non-live runs under that lock and deletes retained state | direct read-only inspection and control built on the #367 stack; coordinated recovered inspection built on the #513 stack, Deno provider only | -| XMD artifact export, inspection and fork source | seals one run's committed retained state, Workspace roots and workflow definition source closure into one immutable `.xmd` evidence file; opens that file read-only for status/history and admits continuation only by creating a new history fork whose lineage names the artifact identity | specified by `specs/xmd-artifact-spec.md`; the version-1 sealed container, its total read-only verifier, `xmd workflow export` and artifact `status`/`history` are built, Deno provider only — the artifact-source fork remains unbuilt. Inspection is two sibling lifecycle operations, `inspectArtifact()` and `historyArtifact()`, taking a path rather than a run id: a run id names live lifecycle ownership and a path names immutable evidence, so neither is a mode of the other. They reach no run store, lock, Workspace, definition reader or external provider, and the artifact path never enters the structural answer | +| XMD artifact export, inspection and fork source | seals one run's committed retained state, Workspace roots and workflow definition source closure into one immutable `.xmd` evidence file; opens that file read-only for status/history and admits continuation only by creating a new history fork whose lineage names the artifact identity. The physical container stays at schema version 1 while the semantic format follows the run: format 1 carries a Git closure, format 2 carries a source bundle as one entry/content pair per logical path under its own manifest version and `xmd-artifact\0v2\0` identity domain, and the reader selects the closed inventory and verifier from the header rather than reading either as the other | specified by `specs/xmd-artifact-spec.md`; the sealed container, its total read-only verifier, both semantic formats, `xmd workflow export` and artifact `status`/`history` are built, Deno provider only — the artifact-source fork remains unbuilt. Inspection is two sibling lifecycle operations, `inspectArtifact()` and `historyArtifact()`, taking a path rather than a run id: a run id names live lifecycle ownership and a path names immutable evidence, so neither is a mode of the other. They reach no run store, lock, Workspace, definition reader or external provider, and the artifact path never enters the structural answer | | Agent session portability evidence in an XMD artifact | classifies every logical Agent session that contributed a retained Prompt as portable — with ordered provider checkpoint tokens and an opaque Agent session bundle — or as explicitly unavailable, as two content kinds inside the existing version-1 manifest and identity | specified by `specs/xmd-artifact-spec.md` §2.5; the closed union, both content kinds and the complete post-identity profile verifier are built on the #621 stack, Deno provider only. Provider bundle capture, Agent-aware export, intrinsic Agent-aware inspection and artifact-backed fork are unbuilt | | historical authored source | retains an authored durable operation's normalized `SourcePosition` beside its identity, and history parses it or refuses the entry | built on the #367 stack | | history fork | creates a new run from one compatible checkpoint and retained Workspace root, under a new immutable definition and normalized props | built on the #368 stack, Deno provider only | diff --git a/specs/executable-mdx-spec.md b/specs/executable-mdx-spec.md index bbf9eccc4..1a052f929 100644 --- a/specs/executable-mdx-spec.md +++ b/specs/executable-mdx-spec.md @@ -3937,7 +3937,11 @@ function* durableImportComponent( "reserved": false } } ``` -A component the workflow definition is closed over records its own shape: +A component the workflow definition is closed over records its own shape. The +path it records is the one that definition retains it under — a +repository-relative path inside the pinned commit for a Git definition, a +logical path inside the bundle for a source bundle — so an authored position +inside a retained component names it by the same path the run does: ```json { "type": "import_component", "name": "Discovery" } @@ -10305,9 +10309,14 @@ completed is restored from its retained record rather than performed again. An installation may also carry a `bundle`: the closed set of authored Markdown components one workflow execution is closed over, as plain immutable data — -each entry's name, its canonical repository-relative path inside the pinned -commit, that blob's object ID, and the exact source read from it. It is read -once and copied entry by entry before any `install()` runs, on the same terms as +each entry's name, the canonical path its definition retains it under, that +source's own identity hash, and the exact source behind it. What that path and +hash mean belongs to the workflow definition's version and not to core: a +version-1 definition retains a repository-relative path inside a pinned commit +and the blob's object ID, and a version-2 source bundle retains a logical path +and the source's own content hash. Either way core receives the same immutable +entries. It is read once and copied entry by entry before any `install()` runs, +on the same terms as the admissions and preparations beside it, so what a name resolves to and which answers a document may invoke are fixed before anything can observe or replace them. One execution runs under one bundle: two installations supplying one is diff --git a/specs/workflow-spec.md b/specs/workflow-spec.md index 04bce297e..66e40bb5f 100644 --- a/specs/workflow-spec.md +++ b/specs/workflow-spec.md @@ -2,9 +2,9 @@ * **Status:** Current * **Scope:** `@executablemd/workflow` — associating a document execution with a - workflow run whose starting repository state is pinned once, retaining that - run so another process can find it, and giving that run's document its own - transactional filesystem. + workflow run whose definition is fixed once, retaining that run and the exact + source it executes so another process can find both, and giving that run's + document its own transactional filesystem. --- @@ -14,10 +14,20 @@ A **workflow run** is a workflow being carried out, with its progress and outcome recorded durably. Document executions perform its work; the run itself outlives any one of them. -A run has one starting repository state, chosen once. The host supplies a -**base** — any Git revision expression — and the run resolves it to a **pinned -commit** the first time it is created. A branch that moves afterwards does not -change what the run started from. +A run has one definition, chosen once, and it comes in two versions. + +A **source-bundle** definition is the exact bytes themselves, addressed by +portable logical paths and retained with the run. A file outside a repository, +an untracked file and a file edited since its last commit are all ordinary +starts, and the run executes what each said at the moment it began. Editing, +moving or deleting the original afterwards changes nothing about the run. + +A **Git** definition names a document inside one immutable object: the host +supplies a **base** — any Git revision expression — and the run resolves it to a +**pinned commit** the first time it is created. A branch that moves afterwards +does not change what the run started from. Its Markdown is not retained, so +obtaining it means reaching a repository, which a trusted host supplies as a +direct dependency (§7.1). ```ts import { executeInstalled } from "@executablemd/core/host"; @@ -29,25 +39,59 @@ const execution = yield* executeInstalled( ); ``` -The package owns `WorkflowRun`, `workflowInstallation()`, `getWorkflowRun()` and the Git -capability. It depends on `@executablemd/core`, -`@executablemd/durable-streams` and `@executablemd/runtime`, whose contextual -`exec()` and `cwd()` the Git provider invokes. Core never imports workflow or -Git, so ordinary `execute()` and `xmd run` stay Git-independent. +The package owns `WorkflowRun`, `workflowInstallation()`, `getWorkflowRun()`, +the source-bundle identity primitives and the Git capability. It depends on +`@executablemd/core`, `@executablemd/durable-streams` and +`@executablemd/runtime`, whose contextual `exec()` and `cwd()` the Git provider +invokes. Core never imports workflow or Git, so ordinary `execute()` and +`xmd run` stay Git-independent. + +`workflowInstallation({ base })` is the one place this package still reaches Git +to establish a run. Every retained path — recognition, resume, fork, journal and +export — reaches none. Issue #822 moves that adapter into the bundled Git Plugin +and removes the final package-level dependency; until it does, the exception is +exactly that one entrypoint. ## 2. What a run is ```ts -interface WorkflowRun { +interface GitWorkflowRunV1 { readonly runId: string; readonly base: string; readonly pinnedCommit: string; } + +interface SourceBundleWorkflowRunV2 { + readonly runId: string; + readonly definitionVersion: 2; + readonly bundleHash: string; + readonly targetPath?: string; +} + +type WorkflowRun = GitWorkflowRunV1 | SourceBundleWorkflowRunV2; ``` `runId` is opaque, allocated with cryptographic randomness, and supports -equality only. `base` is the revision expression the host supplied. -`pinnedCommit` is the full object id that base resolved to. +equality only. A version-1 run's `base` is the revision expression the host +supplied and `pinnedCommit` is the full object id that base resolved to. A +version-2 run's `bundleHash` is the identity of the bytes it retains and +`targetPath` is the exact section it runs, when it runs one; it has no base and +no pinned commit, because a synthetic one would name a repository state the run +never had. + +The union is closed, and reading it is exact. A value carrying one version's +member set in any key order is that version; a value carrying members of both, +or either set with something extra, is not a run read loosely and is refused. +Ordinary serialization writes version 1 as `runId`, `base`, `pinnedCommit` and +version 2 as `runId`, `definitionVersion`, `bundleHash`, then `targetPath` when +present. Object-member order is presentation: a canonical-JSON container sorts +these keys under its own rule without changing the value. + +The retained effect description follows the same split. Version 1 records +`{ type: "workflow_run", name: "workflow_run", base }`; version 2 records +`{ type: "workflow_run", name: "workflow_run", definitionVersion: 2, bundleHash }` +and invents no Git field. Divergence detection compares only the type and the +name, so the members past them are for a reader. `getWorkflowRun()` answers with the frozen value for the document execution running now. It is a context value under a stable name, so a descriptor built @@ -92,27 +136,32 @@ executions and neither sees the other's run. ### 3.1 Installing a run that already exists A host that keeps runs in retained storage (§9) has decided what the run is -before anything executes: `create()` answered with the run id, and the -definition was established from a commit the host pinned. There is nothing left -for the execution to allocate or resolve, and a run id an execution invented -could not agree with the record storage already holds. +before anything executes: the lifecycle transition answered with the run id and +with the definition it retained. There is nothing left for the execution to +allocate or resolve, and a run id an execution invented could not agree with the +record storage already holds. ```ts -yield* executeInstalled(options, [retainedWorkflowInstallation({ runId, base, pinnedCommit })]); +yield* executeInstalled(options, [retainedWorkflowInstallation(run)]); ``` +`run` is whichever member of the `WorkflowRun` union the record retains. + This installs the same middleware in the same place and records through the same `workflow_run` durable operation. What differs is both ends of it. The live path writes exactly the value it was given: no identifier is generated and -`Git.revParse()` is never called. And every journal state holds the record to -that value in full — run id, base and pinned commit — rather than to the base -alone. +`Git.revParse()` is never called, whichever version it is. And every journal +state holds the record to that value in full — run id, base and pinned commit +for version 1; run id, bundle hash and exact target for version 2 — rather than +to the base alone. A journal recording a different run is refused as `StaleInputError`, naming the fields that differ and never their values: a run id may be caller-selected and a base is any revision expression, so both are external text on the same terms as -retained props. A value installed without a run id, a base or a pinned commit -identifies no run and is refused before any document executes. +retained props. Two versions are never the same run, and a record of the other +version disagrees as its definition version rather than field by field. A value +installed without a complete member set for its version identifies no run and is +refused before any document executes. ### 3.2 Where workflow-run identity is decided @@ -239,7 +288,8 @@ The journal is parsed, never trusted. - A record that does not describe a workflow run is refused. The stored value is described, never quoted: it is external data, and reporting it would carry - whatever it held into logs and rendered output. + whatever it held into logs and rendered output. A value matching neither + version's exact member set describes none. - A recorded base that differs from the supplied base is refused, naming both. Both are `StaleInputError`: the journal no longer describes this run, and the @@ -250,9 +300,9 @@ document is re-run from the start rather than resumed. A workflow run exists once its `WorkflowRun` value is durably recorded. A document failure or cancellation after that point does not erase it. -Failure *before* that point creates no run and expands no root document: Git -cannot be invoked, the working directory is not a Git repository, or the base -does not resolve to a commit. +Failure *before* that point creates no run and expands no root document. For a +programmatic Git run that is Git failing to be invoked, a working directory that +is not a Git repository, or a base that does not resolve to a commit. Such a failure is journaled the way every durable effect's failure is — as a recorded failed effect, and no `WorkflowRun` value exists, which is what @@ -294,8 +344,41 @@ Another provider replaces it lexically with `Git.around({ *revParse(…) {…} }, { at: "min" })`. Providers install at `min` so a nested replacement wins rather than being shadowed by an outer handler. -Workflow initialization calls it with `${base}^{commit}`, which is what makes -"does not resolve to a commit" an error rather than a tag object id. +`workflowInstallation({ base })` calls it with `${base}^{commit}`, which is what +makes "does not resolve to a commit" an error rather than a tag object id. That +entrypoint is the only caller in this package (§1). + +### 7.1 The legacy source reader + +A version-1 definition names an object and a path inside it and retains no +Markdown, so obtaining what such a run executes means reaching a repository — +which the retained lifecycle does not do. A trusted host supplies that +capability directly: + +```ts +type LegacyWorkflowSourceReader = ( + definition: GitWorkflowDefinitionV1, + retrieval: Json | undefined, +) => Operation>; +``` + +The host captures it before document code runs. It is reachable through no +Context, contextual Api, component, Plugin installation result or authored +value: a source capability something in the process could reach by name would be +a way to decide what a run executes. It receives only the parsed descriptor and +the run's replaceable retrieval metadata. + +Workflow — not the adapter — decides whether what came back is this run's. The +returned root's object format, pinned commit, repository-relative path and exact +target must be the descriptor's, the declared component set must match name for +name and path for path, and every blob identity is **recomputed from the bytes +that came back** rather than taken on the adapter's word. A closure carrying one +document's identity beside another's content is refused, and none of it +executes. + +A version-2 run never comes here. Its content is in its own store, and a host +reaching a repository for it would be a second answer to a question storage has +already answered. ## 8. Expansion identity is separate @@ -337,11 +420,11 @@ SQLite or DOFS connection. ### 9.1 What identifies a run -Identity is the run id, the definition descriptor, the base and the normalized -props. Normalized props are a JSON object: a document declares named props, so -a run receives a mapping from those names to values, and a bare scalar or array -names nothing. The descriptor carries its own version, and takes part in the -comparison rather than governing it: +Identity is the run id, the definition descriptor, the base a version-1 run +started from, and the normalized props. Normalized props are a JSON object: a +document declares named props, so a run receives a mapping from those names to +values, and a bare scalar or array names nothing. The descriptor carries its own +version and kind, and takes part in the comparison rather than governing it: ```ts interface GitWorkflowDefinitionV1 { @@ -351,9 +434,29 @@ interface GitWorkflowDefinitionV1 { objectId: string; rootDocumentPath: string; targetPath?: string; + components?: readonly WorkflowComponentEntry[]; } + +interface SourceBundleWorkflowDefinitionV2 { + version: 2; + kind: "source-bundle"; + hashAlgorithm: "sha256"; + bundleHash: string; + entrypoint: string; + sources: readonly SourceBundleEntryV2[]; + targetPath?: string; + components?: readonly SourceBundleComponentV2[]; +} + +type WorkflowDefinition = GitWorkflowDefinitionV1 | SourceBundleWorkflowDefinitionV2; ``` +`kind` chooses the parser, not `version`: a kind says what sort of thing a +descriptor identifies and a version says which revision of that sort it is, so a +Git descriptor carrying some other version is refused by the Git parser, where a +reader looking at `objectId` and `rootDocumentPath` is told what went wrong. +Both shapes are closed and admit their exact members in any object-key order. + An object id is lowercase hexadecimal of the length its format requires, so two hosts that agree about the commit agree about the run. A root document path is an already-normalized repository-relative POSIX path: absolute paths, @@ -361,6 +464,94 @@ backslashes, NULs, empty paths, empty segments and `.` or `..` segments are refused rather than normalized, because two spellings of one path would otherwise be two identities. +#### What a source bundle is + +```ts +interface SourceBundleEntryV2 { + path: string; + sourceHash: string; + byteLength: number; +} + +interface SourceBundleComponentV2 { + name: string; + path: string; +} +``` + +`sources` is the complete closure the run executes: non-empty, one entry per +logical path, in the UTF-8 byte order of those paths. `entrypoint` names exactly +one of them and ends in `.md`. `components`, when present, is non-empty, in the +UTF-8 byte order of the names, one entry per name, and every path names a +retained source — so the mapping a root resolves its component names through is +closed over the same bytes the run executes. Absence means the definition +declares no workflow components; an empty array is not a second spelling of that +and is refused. + +Arrays must **arrive** canonical. A parser that sorted them would turn two +spellings of one malformed value into one accepted identity, so a duplicate or a +non-canonical order is refused rather than repaired. The order is the UTF-8 byte +order of the encoded strings, which is not the order `<` gives: comparing UTF-16 +code units puts a supplementary character before some of the characters that +precede it in UTF-8. + +A **logical path** is a portable identity inside the bundle, never a host +filesystem path. It is a non-empty NFC-normalized string of Unicode scalar +values, `/`-separated, never beginning or ending with `/`, with no NUL, C0 +control, DEL, backslash or `#` character and no empty, `.` or `..` segment, and +it is compared case-sensitively by its UTF-8 bytes. The containing directory, +the absolute path, the invocation working directory and the platform separator +are retrieval facts and never identity: two hosts holding the same bytes under +the same logical entrypoint hold the same definition. A CLI input contributes +only its NFC-normalized final path segment as the entrypoint; declared component +paths are resolved against the source file before the run exists and are then +represented by canonical logical paths in the same bundle. + +Changing the logical entrypoint is an identity change, because source positions +and later relative references use it. + +#### How a bundle is identified + +All lengths are unsigned big-endian integers. `u32(n)` is four bytes, `u64(n)` +is eight, `utf8(value)` is the exact UTF-8 encoding, and `field(value)` is +`u32(utf8(value).length)` followed by those bytes. A hexadecimal hash +contributes its decoded 32 bytes, never its text. + +Each source hash is + +```text +SHA-256( + field("executablemd.workflow.source.v2") || + u64(source byte length) || + exact source bytes +) +``` + +and the bundle hash is + +```text +SHA-256( + field("executablemd.workflow.bundle.v2") || + field(entrypoint) || + u32(source count) || + for each canonically ordered source: + field(path) || decoded sourceHash || u64(byteLength) || + u32(component count) || + for each canonically ordered component: + field(name) || field(path) +) +``` + +`bundleHash` and every `sourceHash` are 64 lowercase hexadecimal digits; +`byteLength` is a non-negative safe integer, and zero is a length a source may +have. No property value, run id, host path, Git provenance, retrieval metadata, +timestamp or storage encoding enters either hash, and source bytes are hashed +and stored without newline, BOM, Unicode or any other content normalization. + +The target is deliberately outside the bundle hash: selecting a section does not +change the bytes in the bundle. It remains part of the complete definition +identity, so two runs over one bundle with different targets stay distinct. + #### The document target a run is a run of `targetPath` names one **document target** (executable-MDX §5.4) inside the root @@ -392,14 +583,24 @@ is document content. Serialization writes the member only when there is one, and stored identity is never normalized, decoded, repaired or re-encoded on the way through. -`version` stays `1`. The five-member untargeted shape is the current -representation of a whole-document workflow rather than a legacy format being -preserved, so there is no second version, no version union, and no migration. - -Where that object can be fetched from is deliberately not identity. A locator -and a local checkout path are **retrieval metadata** — replaceable, excluded -from the comparison, never containing credentials, and reauthorized by the host -before use. A run that moves between hosts is the same run. +`GitWorkflowDefinitionV1`, its JSON shape, its Git-object identity, its run +record, its journal binding and its compatible-reuse rules are unchanged. A +version-1 run is never rewritten as version 2 merely because its source was +retrieved successfully, and nothing migrates between the two. + +Where a version-1 object can be fetched from is deliberately not identity. A +locator and a local checkout path are **retrieval metadata** — replaceable, +excluded from the comparison, never containing credentials, and reauthorized by +the host before use. A run that moves between hosts is the same run. + +For version 2 the same boundary holds optional **provenance**: a host may record +a credential-free observation such as an object format, a commit and a +repository-relative path. It may be absent, replaced or become unreachable +without changing compatibility or preventing a resume, nothing reads it back to +find the source, and failing to obtain it cannot fail a start. The supplied +absolute path, the checkout path and the invocation working directory are never +retained in the definition, the run identity, the journal binding or an +artifact. ### 9.2 Creating a run is also how it is found @@ -408,6 +609,13 @@ refuses with a conflict when any immutable field differs. That is what makes a caller-selected id usable twice — as a retry, or as a second process addressing the same work — without a separate idempotency concept. +`CreateWorkflowRunRequest` is version-1 only, and its parser refuses a +source-bundle descriptor. The request carries a descriptor and no source, so +admitting one would initialize a database whose authoritative content nobody +supplied. Creating a version-2 run crosses the trusted lifecycle transition +instead (§9.4.1), which takes the complete snapshot with the descriptor. +`lookup()` recognizes both versions. + Props are compared canonically, so reordering a JSON object does not look like asking for a different run. Everything a run accumulates is excluded: status, stop reason, retrieval metadata, timestamps, document executions and journal @@ -421,6 +629,21 @@ runs of two different sections, so reusing one run id for the other reports a only to absent. The same exact target under the same id is the same run and is found, which is what lets a targeted run be resumed. +Each version is compared by its own complete identity. A version-2 reuse agrees +only when the version, kind and hash algorithm, the bundle hash, the entrypoint, +the complete canonical source manifest and component mapping, the exact presence +and value of `targetPath`, and the normalized props all agree. The manifest is +compared whole even though the bundle hash commits to it: a hash is not a reason +to admit a retained structure that disagrees with itself. The candidate's host +path and its optional provenance are ignored, so identical bytes reached through +another directory are the same run. A version-1 reuse keeps its existing +comparison, including `base`. Two descriptors of different versions are never +one run, and a cross-version request disagrees as `definition` — it is not then +asked about a base one of them does not have. + +**A compatible reuse executes the retained source, never the newly supplied +buffers.** + `lookup()` finds by id and creates nothing. ### 9.3 Finding a run without a registry @@ -467,6 +690,10 @@ read, listed or mistaken for a run. it was last cleared. - The filtered journal. +A version-2 run additionally retains **the exact source it executes**: its +complete manifest, and the content behind it as BLOB bytes rather than as a +database text value or a re-encoded JSON string. + Complete WorkflowRun schema version 1 also contains the pinned Cloudflare DOFS version-5 tables and indexes, immutable Workspace-root tables, exact root-to- manifest and root-to-blob reference tables, singleton current-root state, and a @@ -474,6 +701,89 @@ non-null Workspace-root association on every journal event. XMD schema version 1, DOFS schema version 5 and Workspace-root format version 1 are independent version domains. +#### Two live schema versions + +A newly created Git run keeps `PRAGMA user_version = 1`; a newly created +source-bundle run uses `PRAGMA user_version = 2`. Initialization selects the +schema from the candidate definition already parsed and never upgrades an +existing database. + +The declarations are two immutable, closed inventories. Version 1 is the current +object set byte for byte. Version 2 keeps every version-1 table, index and +trigger byte for byte except `workflow_run`, replaces that table with one that +has **no `base` column** and a `definition` CHECK pinning version 2 and kind +`source-bundle`, and adds exactly two tables: `workflow_definition_blob`, keyed +by a 64-digit lowercase source hash with its byte length and content, and +`workflow_definition_source`, one row per descriptor entry mapping a logical +path to that hash. `definition_retrieval` remains an optional replaceable +metadata table, and every Workspace, journal, session and lifecycle object is +unchanged. + +Recognition validates the application id first, then dispatches on `user_version` +1 or 2. Each version is accepted only when its inventory equals its immutable +declaration exactly. Version 0 under either declared inventory is corruption, +any other version is unsupported, and a hybrid, an altered object or an extra +object is corruption — never a migration candidate. Nothing repairs, migrates or +reinterprets a database. + +#### What a retained source has to prove + +The stored manifest equals the descriptor's complete `sources` array: one row +per entry and no extra row. Every referenced blob exists, its byte length equals +both retained lengths, and recomputing its source hash produces its key. +Multiple paths may reference one blob when their bytes are identical; +unreferenced content is corruption rather than tolerated garbage. Recomputing +the bundle hash from the retained manifest and component mapping produces +`bundleHash`. + +No reader returns a partial bundle. Before a retained source is parsed as +Markdown, the entrypoint and every declared component decodes as well-formed +UTF-8 under one strict decoder, which performs no normalization; the BLOB stays +authoritative and is what export and every later resume preserve. + +#### 9.4.1 Creating a version-2 run + +Version-2 creation crosses only the trusted, non-contextual lifecycle +transition. Its creation request is a closed union: the version-1 member carries +a Git descriptor, a base and props; the version-2 member carries the descriptor, +its props and a `sourceSnapshot` — the exact byte sequence behind each logical +path. + +`sourceSnapshot` is non-empty, admits no extra entry member, and has exactly the +descriptor's paths in exactly its order. Each length and recomputed source hash +must equal the corresponding entry. The transition takes an **owned copy of +every byte sequence before it validates or persists anything**, and retains only +those copies, so later mutation of a caller-owned array cannot change the run. A +missing, extra, reordered or mismatched entry is an invalid creation and writes +nothing. + +One initialization transaction then selects schema 2 and writes the descriptor, +the manifest rows, the de-duplicated BLOBs, the initial Workspace state, the +lifecycle state and the first document-execution record. They all appear or none +do. The transition answers with the authenticated retained closure read back +from storage, which is what a caller imports. + +#### 9.4.2 Source is proved before the lifecycle moves + +Every source check happens under the executor lock and **before** stale +recovery, a new document-execution record, Workspace attachment, journal replay +or root import. A failure therefore leaves the run's lifecycle and journal +exactly as they were. + +For an existing version-2 run the retained BLOBs are re-derived in full; the +original path, the provenance and the legacy reader are never consulted. For a +version-1 run the captured legacy reader (§7.1) is invoked under the same lock +and its answer validated, and a run being created from a version-1 descriptor is +held to that descriptor before it is persisted. Completed replay, export and +fork obtain a version-1 closure through the same seam whenever authenticated +retained history does not already provide it. + +No run id is reported before the creation transaction commits. A failure or +cancellation before that commit leaves no recognized run and permits clean reuse +of the id; an empty or private staging file is not a damaged run and is not +discoverable by lookup or list. An interruption after the commit is an ordinary +resumable execution, because the complete definition already exists. + A fresh database contains one content-addressed Workspace root whose canonical manifest describes only `/` as a directory. Its retained manifest and blob reference sets are empty, its current-root pointer names that root, and its DOFS @@ -782,6 +1092,24 @@ differently is reported as itself: | request | a value a caller supplied describes nothing storage can keep | | transaction | a transaction cannot be started, continued or committed as asked | | inspection recovery | a crashed run could not be read from a private recovered copy | +| definition source missing | a version-2 descriptor names a manifest entry or blob the store does not hold | +| definition corrupt | a retained path, length, hash, byte, mapping or bundle hash disagrees | +| legacy reader unavailable | a version-1 run needs source and this host installed no reader | +| legacy source unavailable | the installed reader cannot obtain the retained object | +| legacy source mismatch | the reader returned a closure that does not describe this definition | + +The last five are the source conditions, and they are kept apart because an +operator acts on each differently. Missing content is gone and there is nothing +to repair; corrupt content is there and no longer describes itself; no reader +means this host cannot obtain version-1 Markdown at all and the operator needs a +Git-capable XMD host; an unavailable read means the reader could not reach the +object; and a mismatch means it answered about something else. None of them +quotes source, props, retrieval metadata or an absolute path, none returns +partial content, none advances lifecycle state, and **none falls back** to +current `HEAD`, working-tree bytes, the original file, the provenance or the +legacy reader. A malformed retained descriptor or database structure stays a +storage-corruption refusal under the recognition rules above rather than being +reclassified as an unavailable external source. Inspection recovery is a distinct condition because it says nothing about the run. It reports that read-only inspection could not produce or clean up the @@ -797,7 +1125,9 @@ table's stored definition is compared with the definition this build creates, so a missing column, a dropped constraint, and a table nobody declared are all caught before a row reaches a parser that assumes they hold. A file whose header says it is a version-1 workflow run and is not shaped like one is -**damage**: the file disagrees with itself. Format and version failures are +**damage**: the file disagrees with itself, and so is a file whose header +declares one version over the other version's objects. Format and version +failures are reserved for a file that belongs to something else, or to a version this build has not learned. diff --git a/specs/workflow-workspace-spec.md b/specs/workflow-workspace-spec.md index 36ce236b2..baa4585e2 100644 --- a/specs/workflow-workspace-spec.md +++ b/specs/workflow-workspace-spec.md @@ -119,11 +119,18 @@ from the workflow host, not from another spelling of `` or ``. ## 3. Workflow lifecycle -The immutable workflow definition is the source repository, pinned commit and -root document path established by the workflow-run contract. It is not an -implicit Repository inside the Workspace. Repository components may therefore -open two or more unrelated repositories without changing definition or run -identity. +The immutable workflow definition is established by the workflow-run contract +and comes in two versions. A **source bundle** is the exact bytes the run +retains, addressed by logical paths; a **Git** definition is the source +repository, pinned commit and root document path. Neither is an implicit +Repository inside the Workspace. Repository components may therefore open two or +more unrelated repositories without changing definition or run identity. + +`xmd workflow start` establishes a source bundle. Its argument is a document +reference — a path, optionally followed by `#` and one target selector — and the +file's current bytes are what the run is of, whether or not that file is tracked, +committed or inside a repository at all. Existing version-1 runs keep their +definition, their identity and their behavior exactly. When the definition names one **document target**, that exact canonical target is part of it and therefore part of definition identity. A run of one section @@ -133,6 +140,31 @@ selector a caller wrote never occupies that field: identity is the resolved answer, so resuming re-enters the section the run actually executed rather than whatever the same glob would name against a later checkout. +### 3.1 Source is proved before the lifecycle moves + +Every source check happens under the executor lock and **before** stale +recovery, a new document-execution record, Workspace attachment, journal replay +or root import. A source-bundle run's retained content is re-derived from its own +store; a version-1 run's Markdown crosses the host-supplied legacy reader and its +answer is validated against the descriptor. A host that installed no such reader +cannot begin, fork or stage a version-1 run at all, and says so rather than +admitting one without knowing what it executes. + +A failure there leaves the run exactly as it was: no execution record, no +Workspace attached, no status published. Nothing falls back to the original +file, the optional provenance, the current `HEAD` or working-tree bytes. + +For a new run, one transaction retains the descriptor, the complete source +manifest, the exact blobs, the initial Workspace state, the first +document-execution record and `running` — all of them or none. **No run id is +reported before that commit.** A failure or cancellation before it leaves no +recognized run and permits a clean retry of the id; an interruption after it is +an ordinary resumable execution. + +After admission the command executes the closure the transition returned. It +does not reread the candidate path: a candidate that remained a second route to +execution would be a second answer to what the run is a run of. + A root may also close itself over a **component bundle**, and that bundle is part of the definition too. The root declares it in its own frontmatter: @@ -148,12 +180,18 @@ workflow: `workflow` is one closed member, `components`, holding a non-empty mapping from a component name to a relative POSIX Markdown path beside the root. Each path is -normalized against the root's own directory in the same pinned commit and read -from that commit; the blob's own object ID is the component's source hash. A -root that declares no `workflow` member is a run with no bundle and keeps the -version 1 descriptor exactly; an explicitly empty mapping is refused rather than -becoming a second spelling of the same thing. A declaration may not claim -structural syntax, a component the engine supplies, or a name the host reserved. +normalized against the root's own directory, and that normalized path is both +the logical path the bundle retains and the file this host reads beside the +root; the source's own domain-separated content hash is the component's source +hash. A root that +declares no `workflow` member is a run with no bundle and writes no `components` +member at all; an explicitly empty mapping is refused rather than becoming a +second spelling of the same thing. A declaration may not claim structural +syntax, a component the engine supplies, or a name the host reserved. + +A version-1 run keeps its own form of the same rule: each path is normalized +inside the pinned commit and read from it, and the blob's own object ID is the +component's source hash. Because the hash is identity, changing what a component says changes the definition. A run of one bundle and a run of another are different runs, exactly @@ -192,7 +230,9 @@ xmd workflow answer run_01J... 9f2c... '{"approved":true}' ``` An inspection names no document and reads no definition: `status`, `list` and -`history` reach neither Git nor the working tree. +`history` reach neither Git nor the working tree. `status` describes a +source-bundle run by its entrypoint and bundle hash, with its exact target when +it has one, and reports no base and no fabricated commit for it. `start` executes until completion, failure, suspension or interruption. `resume` continues an interrupted or ready suspended run under its retained @@ -560,14 +600,17 @@ interruption, and a completed or failed document is never relabelled **A run that ended is not a run to continue.** `resume` admits `interrupted`, `suspended` and (as a full replay) `completed`; `failed` and `cancelled` are -refused with exit 1 — before the definition is fetched from Git, before the -executor lock is acquired, before stale recovery, before a document-execution -record is begun, before a Workspace is attached, and before anything is -appended. Reusing a +refused with exit 1 — before stale recovery, before a document-execution record +is begun, before a Workspace is attached, and before anything is appended. +Source is proved first, under the executor lock and before every one of those +(§3.1), so a run whose source cannot be obtained is refused for that with its +lifecycle and journal untouched. Reusing a compatible id through `start` is a separate rule: it replays a failed run's retained failure, and that does not make the run eligible for `resume`. -- `start` takes exactly one Markdown definition path. There is no generic +- `start` takes exactly one Markdown document reference — a path, optionally + followed by `#` and one target selector, with a literal `#` in a filename + written `%23`. There is no generic `--prop`, no `--journal`, no inline `--eval`, no agent option and no host selector. Without `--id` the host generates an opaque cryptographically random identifier, so starting the same document twice makes two runs; the local @@ -592,9 +635,9 @@ retained failure, and that does not make the run eligible for `resume`. module, an extensionless path, a directory, a glob, a URL, a package specifier, an absolute path and a path that walks the tree are each refused rather than repaired. -- A declared component the pinned commit does not hold as readable Markdown - refuses the `start` before storage is created and before any component code - runs. +- A declared component that is not readable Markdown beside the root refuses + the `start` before storage is created and before any component code runs. A + version-1 run applies the same rule to the pinned commit. - The command exists on every runtime and the capability on one: the Deno entrypoint and the compiled binary own the local run store, and Node and Bun refuse before creating or executing anything. @@ -620,9 +663,10 @@ resumes the run rather than on its own. Fork is built (§11): a run's history ca be continued under a changed definition from any committed checkpoint the retained events allow. -The component bundle is built: a root declares one, `start` establishes it from -the pinned commit, the definition retains it, and `resume` and completed replay -reconstruct it. The adversarial implementation loop those five +The component bundle is built: a root declares one, `start` reads it from beside +the root, the definition retains its bytes, and `resume` and completed replay +execute the closure the lifecycle authenticated. A version-1 run's bundle is +reconstructed from its pinned commit through the host's legacy source reader. The adversarial implementation loop those five stage names describe is not — its scheduling and unattended continuation belong to the durable-suspension stack. Generated-XMD admission is built for both effect classes (§8.4): a fragment observes under `read` and mutates the run's @@ -793,31 +837,57 @@ Co-location does not make arbitrary filesystem content journal data. ### 5.1 What the run record holds -The run's own record is the part of that database the lifecycle reads. It holds -the immutable definition, the definition base and the normalized props; -the current status and its stop reason; one document-execution record per start -and per resume; replaceable retrieval metadata; and the filtered journal. - -The immutable definition is a versioned descriptor naming an object format, an -object ID and the repository-relative root document path. It also carries the -component bundle when the root declares one: an array sorted by component name, -each entry holding that name, its canonical repository-relative path inside the -pinned commit, and the blob's object ID under the descriptor's own object -format. The bundle is a member of that descriptor rather than a version past it, -so a definition retained before bundles existed parses unchanged and identifies -a run closed over no components; an empty array is not a second spelling of that -and is refused. A repository locator is not part of it. Where the definition can be fetched from, and where it is -checked out on one machine, are retrieval metadata: replaceable, free of -credentials, reauthorized by the host before use, and excluded from the -comparison that decides whether a reused run ID addresses the same run. - -Compatible reuse compares the run ID, the whole descriptor including its -version and its component bundle, the base and the normalized props, -canonically. A changed component name, canonical path, source hash or component -set conflicts as `definition`, and the refusal names that field rather than any -value behind it. Status, stop reason, -retrieval metadata, timestamps, document executions and journal records are -excluded, so a completed run asked for again is found rather than refused. +The run's own record is the part of that database the lifecycle reads, and it is +a closed alternative on the definition version it retains. Both members hold the +immutable definition and the normalized props; the current status and its stop +reason; one document-execution record per start and per resume; replaceable +retrieval metadata; and the filtered journal. A **version-1** record also holds +the definition base. A **version-2** record holds no base and no pinned commit: +those are Git fields, and a synthetic one would name a repository state the run +never had. + +The immutable definition is a closed alternative on the same split. + +A **version-1** descriptor names an object format, an object ID and the +repository-relative root document path. Its component bundle is an array sorted +by component name, each entry holding that name, its canonical +repository-relative path inside the pinned commit, and the blob's object ID +under the descriptor's own object format. Where that object can be fetched from, +and where it is checked out on one machine, are retrieval metadata: +replaceable, free of credentials, reauthorized by the host before use, and +excluded from the comparison that decides whether a reused run ID addresses the +same run. + +A **version-2** descriptor is a source bundle: a hash algorithm, a bundle hash, +a logical entrypoint, and a canonically ordered manifest holding each source's +logical path, its domain-separated content hash and its byte length. Its +component bundle maps each declared name onto a logical path the same manifest +retains. The bytes behind every path are retained with the run, so nothing has +to be fetched at all; an optional credential-free provenance observation lives +in the same replaceable metadata boundary and is never read back to find the +source. + +In both versions the bundle is a member of the descriptor rather than a version +past it, so a definition retained before bundles existed parses unchanged and +identifies a run closed over no components; an empty array is not a second +spelling of that and is refused. A repository locator is part of neither. + +Compatible reuse compares the run ID, the whole descriptor including its version +and kind and its component bundle, and the normalized props, canonically. **The +base takes part only when both sides are version-1 runs**; a source bundle has +none, so nothing is compared about one. A version-2 comparison covers the bundle +hash, the entrypoint, the complete canonical source manifest, the component +mapping and the exact presence and value of the target — the manifest whole, +even though the bundle hash commits to it, because a hash is not a reason to +admit a retained structure that disagrees with itself. + +Two descriptors of different versions are never one run. A cross-version reuse +conflicts as `definition` and is not then asked about a base one of them does +not have. A changed component name, path, source hash or component set conflicts +as `definition` in either version, and the refusal names that field rather than +any value behind it. Status, stop reason, retrieval metadata, provenance, +timestamps, document executions and journal records are excluded, so a completed +run asked for again is found rather than refused. A stop reason is a categorical host code or a reference to an already-filtered journal event. Arbitrary failure text is not retained beside the journal that @@ -2348,19 +2418,34 @@ creation identity does not. Missing, damaged or conflicting retained state is a stale-input condition: children and later siblings do not begin, `` cannot print it, and nothing is recloned or repaired. -Before any of that, a run closed over a component bundle reconstructs it: `resume` -reads every retained component from the retained commit and verifies each blob -against the retained source hash under the executor lock, before a -document-execution record is begun, before a Workspace is attached and before -anything is appended. The working tree and the current `HEAD` are not consulted. -The reconstructed bundle then holds the retained history to itself — a recorded -import naming a component the bundle does not declare, or holding a different -path, hash or source, appends nothing and invokes nothing. +Before any of that, a run closed over a component bundle obtains it, by the +version it retains — under the executor lock, before a document-execution record +is begun, before a Workspace is attached and before anything is appended. + +A **version-2** run authenticates it from its own retained source BLOBs. Every +retained length and content hash is recomputed from the bytes the run holds, and +so is the bundle hash over the resulting manifest and mapping. Nothing is read +from a repository, from the file the run was started from, or from the files its +components were read beside — all three may have been edited, moved or deleted, +and none of them is what the run is a run of. Missing content and content that +no longer describes itself are distinct refusals, and neither falls back to any +other source. + +A **version-1** run reconstructs it from the retained commit through the +host-supplied legacy source reader, and Workflow recomputes every returned blob +identity from the bytes that came back before comparing it with the retained +source hash. The working tree and the current `HEAD` are not consulted. A host +that installed no such reader cannot obtain a version-1 bundle at all and says +so rather than proceeding without one. + +Either way the obtained bundle then holds the retained history to itself — a +recorded import naming a component the bundle does not declare, or holding a +different path, hash or source, appends nothing and invokes nothing. A completed root result returns without expanding the document or attaching -Workspace, Agent or external providers. It still reconstructs the bundle and -still applies that admission, so retained output is accepted only for a history -this run is a run of. +Workspace, Agent or external providers. It still obtains the bundle the same way +its version requires, and still applies that admission, so retained output is +accepted only for a history this run is a run of. A partial replay that reaches a completed Git-host effect may still reconstruct what that effect needed locally — a Push rebuilds its checkout from the Workspace @@ -2986,9 +3071,11 @@ nor `--at`. ### 11.1 What a fork retains A fork is a new immutable WorkflowRun identity. It holds a new run ID, the -supplied definition with its base, pinned commit and complete component bundle, -the merged normalized props, its lineage, and a journal made of two records it -writes for itself followed by everything it inherited. +supplied definition entire — its complete component bundle, and whatever else +that version is made of: a source bundle's manifest and retained bytes, or a +version-1 definition's base and pinned commit — the merged normalized props, its +lineage, and a journal made of two records it writes for itself followed by +everything it inherited. The two records are its own `workflow_run` and its own root import. The first is what its journal is held to: a history carrying two run records describes two @@ -3042,6 +3129,14 @@ selected event. It does not acquire the source executor lock and never writes the source run. Concurrent source appends after the selected checkpoint do not alter the admitted prefix or root. +The fork's candidate is established exactly as a `start`'s is: its own document +reference, its own bytes, its own logical entrypoint and its own bundle. The +fork is therefore a run of what that file said, and its retained source is +independent of the source run's — deleting the source run, the candidate's file +or the repository either sat in changes nothing about what the fork replays. +Staging assembles the same candidate under the same rules, so a compatibility +replay runs the bytes the fork will retain rather than anything still on disk. + Before recognizable new-run storage exists, the source, the checkpoint, the selected root, the candidate definition, the component bundle, the normalized props, forkability and compatibility replay are all validated. Missing, corrupt, @@ -3055,8 +3150,9 @@ or death after it leaves a valid fork under ordinary lifecycle and recovery rules. Live document execution begins only after that commit. Reusing a caller-selected fork ID is compatible only when the source run, the -checkpoint, the definition including its component bundle and pinned commit, the -base and the normalized props all agree. A request differing in any one of them +checkpoint, the definition including its version and its component bundle, and +the normalized props all agree — and, for a version-1 definition, its base and +pinned commit as well. A request differing in any one of them is refused before anything is written, and the refusal names the term that differs rather than collapsing them into one cause. diff --git a/specs/xmd-artifact-spec.md b/specs/xmd-artifact-spec.md index 62f9b611b..f6180a834 100644 --- a/specs/xmd-artifact-spec.md +++ b/specs/xmd-artifact-spec.md @@ -142,13 +142,17 @@ xmd workflow fork \ --props-debug=true ``` +**The artifact-backed fork is specified here and not built** (§8). What this +revision delivers is the authenticated source input it needs: a format-2 closure +carries the descriptor and the exact bytes behind it, which is sufficient to +copy into a destination run without making the artifact path retrieval state. + For an artifact source, the definition argument is optional. When absent, the -candidate definition is the artifact's workflow definition source closure. The -new run copies that closure into its retained definition state, so later resume -does not depend on the artifact path or on access to the original repository. -When present, the candidate definition is resolved normally and the source -run's normalized props remain the baseline under the workflow history-fork -contract. +candidate definition is the artifact's workflow definition source. The new run +copies it into its retained definition state, so later resume does not depend on +the artifact path or on access to the original repository. When present, the +candidate definition is resolved normally and the source run's normalized props +remain the baseline under the workflow history-fork contract. The fork always receives a new run ID. It copies the inherited journal prefix, the Workspace roots that prefix references, the selected current root, and the @@ -239,14 +243,18 @@ frontier and attempt a compatible history fork: - in a finalized artifact, one portability classification for every logical Agent session that contributed a retained Prompt, and the opaque bundle bytes of each portable one (§2.5); and -- the workflow definition source closure. +- the workflow definition source, in the form the run's own version retains. “Complete” is bounded by XMD ownership. The artifact does not claim to capture the state of the world around the run. -### 2.4 Definition source closure +### 2.4 Definition source + +A run retains one of two definitions, and an artifact seals the one its run has. +Neither is read as the other: a format-1 artifact admits only the Git closure +and a format-2 artifact admits only the source bundle. -The closure contains: +**A Git (version-1) closure** contains: - the exact root Markdown bytes, repository-relative path, Git object format, pinned commit and root blob identity; @@ -256,15 +264,23 @@ The closure contains: expanded. Export authenticates every source against the immutable object identity in the -workflow definition. It may satisfy a source from already-retained bytes or from -reauthorized retrieval metadata. A fetch is an export input operation, not -inspection and not part of artifact identity. Missing retrieval permission, -missing objects, an object-format mismatch or bytes that do not hash to the -declared identity refuse export. - -Retrieval metadata, clone paths, remote URLs and credentials do not enter the -closure. The artifact therefore remains sufficient to inspect and fork the -original definition when its repository is unavailable. +workflow definition. It obtains that source through the host-supplied legacy +reader, and validates the answer itself: the root's descriptor terms must be the +definition's, and every blob identity is recomputed from the bytes that came +back. A fetch is an export input operation, not inspection and not part of +artifact identity. Missing retrieval permission, a missing reader, missing +objects, an object-format mismatch or bytes that do not hash to the declared +identity refuse export. + +**A source bundle (version-2)** contains the descriptor the run retains and the +exact bytes behind every logical path in it. Export reads only the run's own +source store; it reaches no repository and needs none, so a version-2 run +exports after the file it was started from has been edited, moved or deleted. + +Retrieval metadata, provenance, clone paths, remote URLs, host paths and +credentials do not enter either form. The artifact therefore remains sufficient +to inspect and fork the original definition when its repository is unavailable +— and, for a source bundle, when there never was one. ### 2.5 Agent session portability evidence @@ -388,12 +404,16 @@ their kinds, logical identities, lengths and content hashes. Ordering and value encoding are part of the manifest version. Physical SQLite page order, free space, indexes and host path are not. -The XMD artifact identity is: +The XMD artifact identity is domain-separated by the semantic format: ```text -sha256("xmd-artifact\0v1\0" || canonical-manifest-bytes) +sha256("xmd-artifact\0v1\0" || canonical-manifest-bytes) # format 1 +sha256("xmd-artifact\0v2\0" || canonical-manifest-bytes) # format 2 ``` +One format's manifest therefore never derives the other's identity, however +similar the two inventories look. + The identity is written in the container as a derived value. A reader reconstructs the manifest, recomputes the identity, validates every referenced record and byte, and compares the derived value before exposing status, history @@ -412,8 +432,9 @@ order every other XMD identity uses; arrays keep the order the manifest states. The value is: ```ts -interface XmdArtifactManifestV1 { - readonly version: 1; +interface XmdArtifactManifest { + /** The semantic format this artifact declares: 1 or 2. */ + readonly version: number; readonly entries: readonly XmdArtifactManifestEntryV1[]; } @@ -485,6 +506,40 @@ record, and therefore corruption. It is not ignored for forward compatibility: a reader that skipped it would be returning a snapshot whose inventory nobody checked. A later version declares its own set. +#### 4.1.3 The version 2 inventory is closed + +Format 2 seals a source-bundle run. It uses manifest version `2` and the +`xmd-artifact\0v2\0` identity domain; the manifest entry order, the canonical +JSON rules and the content-digest algorithm are otherwise the format-1 rules, +and the physical container is unchanged (§5.1). + +Its inventory is every group above **except** the Git definition closure, plus +one pair per retained source. It admits no format-1 definition kind at all: + +- `workflow-run` carries only the version-2 storage definition and the version-2 + `WorkflowRun`. It admits no `base` and no pinned commit; +- `definition-source-entry` is `canonical-json` under the canonical JSON string + of the source's logical path, containing exactly the members `path`, + `sourceHash` and `byteLength`. Canonical JSON writes those keys in its + existing lexicographic order; a reader requires the exact member set and does + not treat object-member order as identity; and +- `definition-source-content` is `bytes` under the same identity, containing + that source's exact BLOB bytes. + +Semantic verification requires exactly one entry and one content value per +descriptor path and no undeclared pair, checks both identities against the path, +holds each entry's declared hash and length to the descriptor's, recomputes +every source hash from the bytes carried, and recomputes the bundle hash — +before any status or history is returned. An entry that names a path the +descriptor does not retain is corruption. + +Format 1, its `xmd-artifact\0v1\0` identity domain and its Git definition +closure remain byte for byte unchanged and readable. The writer selects format 1 +for a version-1 live run and format 2 for a version-2 one; the reader selects +the exact closed inventory, manifest parser, identity domain and semantic +verifier from the header version and never interprets one version's closure as +the other's. Neither inventory is loosened into a shared superset. + ### 4.2 Integrity is not authenticity Manifest verification detects accidental corruption and unsophisticated @@ -506,11 +561,18 @@ artifact uses a distinct SQLite application ID from a live workflow-run database and carries both an artifact format version and a container schema version. -Version 1 fixes those values. The artifact format version is `1`. The container -schema version is `1`, stored as the SQLite `user_version`. The application -marker is the four bytes `XMDA`, the integer `0x584d4441`; the live -workflow-run marker is `XMD1`, `0x584d4431`, and recognizing that one is the -categorical live-run refusal rather than the foreign-container refusal. +The container schema version is `1`, stored as the SQLite `user_version`, for +both semantic formats: the artifact format version and the container schema +version are separate questions — how the records inside are to be read, and how +the bytes are laid out — and format 2 changed only the first. The application +marker is the four bytes `XMDA`, the integer `0x584d4441`; the live workflow-run +marker is `XMD1`, `0x584d4431`, and recognizing that one is the categorical +live-run refusal rather than the foreign-container refusal. + +The artifact format version is `1` or `2`. A container declaring any other is an +unsupported version this build must not touch; a container declaring one of +these two over the other's records is a file that disagrees with its own header, +and its undeclared kinds make it corrupt. Raw tables, views, indexes, triggers, PRAGMAs and SQL queries are not public API. The supported readers are XMD lifecycle commands and libraries that implement @@ -550,7 +612,11 @@ Export refuses without producing the target when: - the source run is absent, foreign, damaged or incompatible; - another workflow executor holds its lock; - a consistent committed frontier cannot be read; -- the workflow definition source closure cannot be obtained and authenticated; +- the workflow definition source cannot be obtained and authenticated: for a + version-1 run, no legacy reader is installed, the reader cannot reach the + retained object, or its answer does not describe the definition; for a + version-2 run, the retained manifest or content is missing, or disagrees with + the descriptor; - any retained state required by the artifact manifest is unreadable; - the target exists, lacks the `.xmd` extension or cannot be published atomically; or @@ -609,7 +675,8 @@ Architecture review freezes these invariants before implementation: | Contract | Status at this design revision | | --- | --- | | XMD artifact terminology and structural boundary | specified in `architecture.md`; built | -| `xmd workflow export` | specified; built, Deno provider only. Source retrieval is host-installed rather than caller-supplied | +| `xmd workflow export` | specified; built, Deno provider only. A version-2 run exports from its own retained source; a version-1 run's source retrieval is host-installed rather than caller-supplied | +| artifact format 2 for source-bundle runs | specified; the manifest version, identity domain, closed inventory, source entry/content pairs and semantic verifier are built, Deno provider only. Format 1 is unchanged and remains readable | | artifact status/history and manifest verification | specified; built, Deno provider only. `inspectArtifact()` and `historyArtifact()` are sibling lifecycle operations, and history answers with an envelope | | version 1 Agent session portability evidence | specified; the two content kinds, the closed union and the complete post-identity profile verifier are built. Production export still emits merged-legacy V1, no provider bundle capture exists, and inspection carries none of it | | artifact-backed history fork and artifact lineage | specified; unbuilt | From 6c771a1aaaea0077c527783500ca3320d2fa37d3 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Thu, 17 Sep 2026 03:43:03 -0400 Subject: [PATCH 5/8] =?UTF-8?q?=F0=9F=90=9B=20Narrow=20v1-only=20test=20ho?= =?UTF-8?q?sts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/cli/tests/support/workflow-run.ts | 11 ++++++++++- .../workflow/tests/support/git-crash-child.ts | 16 ++++++++++++---- .../tests/support/pull-request-crash-child.ts | 16 ++++++++++++---- packages/workflow/tests/support/restart-child.ts | 11 +++++++++-- .../tests/workflow-lifecycle-inspection.test.ts | 8 +++++++- 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/packages/cli/tests/support/workflow-run.ts b/packages/cli/tests/support/workflow-run.ts index 6b0ee40fb..2bce22e6b 100644 --- a/packages/cli/tests/support/workflow-run.ts +++ b/packages/cli/tests/support/workflow-run.ts @@ -9,7 +9,11 @@ import { scoped } from "effection"; import type { Operation } from "effection"; import { useTempDirectory } from "@executablemd/test-support/temp"; -import { parseWorkflowDefinition, WorkflowRunStorage } from "@executablemd/workflow"; +import { + isGitWorkflowDefinition, + parseWorkflowDefinition, + WorkflowRunStorage, +} from "@executablemd/workflow"; import type { CreateWorkflowRunRequest, WorkflowRunDatabase } from "@executablemd/workflow"; import { useWorkflowRunStorage } from "@executablemd/workflow/deno"; @@ -40,6 +44,11 @@ export function* createRun( if (!parsed.ok) { throw parsed.error; } + // The literal above is a version-1 definition, so the union the parser returns + // narrows back to one here rather than being asserted into one. + if (!isGitWorkflowDefinition(parsed.value)) { + throw new Error(`expected a Git definition, got ${parsed.value.kind}`); + } const created = yield* WorkflowRunStorage.operations.create({ runId: "observation-run", definition: parsed.value, diff --git a/packages/workflow/tests/support/git-crash-child.ts b/packages/workflow/tests/support/git-crash-child.ts index 6afa8e5c9..fd6636060 100644 --- a/packages/workflow/tests/support/git-crash-child.ts +++ b/packages/workflow/tests/support/git-crash-child.ts @@ -37,7 +37,7 @@ import process from "node:process"; import { collect, execute, inlineSource } from "@executablemd/core"; import { ensure, main, type Operation, scoped, suspend } from "effection"; -import { WorkflowRunStorage } from "../../mod.ts"; +import { isGitWorkflowRunRecord, WorkflowRunStorage } from "../../mod.ts"; import { useWorkflowRunStorage, workflowRunPath } from "../../deno.ts"; import { createWorkflowRunConnections } from "../../src/deno/connections.ts"; import { openWorkflowRunDatabase, readRunRow } from "../../src/deno/database.ts"; @@ -221,6 +221,14 @@ function* pushCrash( useAuthentication: (asked: string) => useGitAuthentication(inner, asked), }; + // This child drives a version-1 run, so its record narrows to one before the + // installation reads a base and a pinned commit. A version-2 record has + // neither, and a synthetic one would name a repository state no run had. + const record = database.record; + if (!isGitWorkflowRunRecord(record)) { + throw new Error(`expected a Git run record, got ${record.definition.kind}`); + } + yield* withWorkflowWorkspace( database, scoped(function* () { @@ -229,9 +237,9 @@ function* pushCrash( { ...inlineSource(pushDocument(locator)), stream: database.journal }, [ retainedWorkflowInstallation({ - runId: database.record.runId, - base: database.record.base, - pinnedCommit: database.record.definition.objectId, + runId: record.runId, + base: record.base, + pinnedCommit: record.definition.objectId, }), ], ), diff --git a/packages/workflow/tests/support/pull-request-crash-child.ts b/packages/workflow/tests/support/pull-request-crash-child.ts index 035301946..05bb8325a 100644 --- a/packages/workflow/tests/support/pull-request-crash-child.ts +++ b/packages/workflow/tests/support/pull-request-crash-child.ts @@ -18,7 +18,7 @@ import process from "node:process"; import { main, type Operation, scoped, suspend } from "effection"; import { collect, inlineSource } from "@executablemd/core"; import { executeInstalled } from "@executablemd/core/host"; -import { WorkflowRunStorage } from "../../mod.ts"; +import { isGitWorkflowRunRecord, WorkflowRunStorage } from "../../mod.ts"; import { useWorkflowRunStorage } from "../../deno.ts"; import { retainedWorkflowInstallation } from "../../src/run.ts"; import { withWorkflowWorkspace } from "../../src/deno/workspace/host.ts"; @@ -61,6 +61,14 @@ function* open(root: string, runId: string, locator: string, endpoint: string): } const database = opened.value; + // This child drives a version-1 run, so its record narrows to one before the + // installation reads a base and a pinned commit. A version-2 record has + // neither, and a synthetic one would name a repository state no run had. + const record = database.record; + if (!isGitWorkflowRunRecord(record)) { + throw new Error(`expected a Git run record, got ${record.definition.kind}`); + } + yield* withWorkflowWorkspace( database, scoped(function* () { @@ -72,9 +80,9 @@ function* open(root: string, runId: string, locator: string, endpoint: string): }, [ retainedWorkflowInstallation({ - runId: database.record.runId, - base: database.record.base, - pinnedCommit: database.record.definition.objectId, + runId: record.runId, + base: record.base, + pinnedCommit: record.definition.objectId, }), ], ), diff --git a/packages/workflow/tests/support/restart-child.ts b/packages/workflow/tests/support/restart-child.ts index 5037f774f..0a990450b 100644 --- a/packages/workflow/tests/support/restart-child.ts +++ b/packages/workflow/tests/support/restart-child.ts @@ -29,7 +29,7 @@ import process from "node:process"; import { durableCall, durableRun } from "@executablemd/durable-streams"; import type { Workflow } from "@executablemd/durable-streams"; import { main, until } from "effection"; -import { WorkflowLifecycle, WorkflowStorageError } from "../../mod.ts"; +import { isGitWorkflowRunRecord, WorkflowLifecycle, WorkflowStorageError } from "../../mod.ts"; import { useWorkflowRunHost } from "../../deno.ts"; import { legacySourceReader } from "./legacy-source.ts"; @@ -113,10 +113,17 @@ main(function* () { throw entries.error; } + // This child restarts a version-1 run, so its record narrows to one before the + // base is reported. A version-2 record has none to report. + const record = database.record; + if (!isGitWorkflowRunRecord(record)) { + throw new Error(`expected a Git run record, got ${record.definition.kind}`); + } + console.log( JSON.stringify({ value, - base: database.record.base, + base: record.base, // The record settlement returned, not the handle's snapshot from when the // execution began — that one still says `running`. status: settled.value.status, diff --git a/packages/workflow/tests/workflow-lifecycle-inspection.test.ts b/packages/workflow/tests/workflow-lifecycle-inspection.test.ts index 6232360c6..e73c20846 100644 --- a/packages/workflow/tests/workflow-lifecycle-inspection.test.ts +++ b/packages/workflow/tests/workflow-lifecycle-inspection.test.ts @@ -29,6 +29,7 @@ import type { Close, DurableEvent, Yield } from "@executablemd/durable-streams"; import { SOURCE_POSITION_FIELD } from "@executablemd/core"; import { Git, + isGitWorkflowRunRecord, WorkflowDatabaseFormatError, WorkflowInspectionRecoveryError, WorkflowLifecycle, @@ -239,7 +240,12 @@ describe("Tier WLI — immutable lifecycle inspection", () => { expect(snapshot.record.runId).toBe("release-1.4"); expect(snapshot.record.status).toBe("completed"); expect(snapshot.record.props).toEqual({ channel: "stable" }); - expect(snapshot.record.definition.rootDocumentPath).toBe("workflows/release.md"); + // The fixture is a version-1 run, and the root document path is a member + // only that version has: a source bundle names a logical entrypoint. + expect(isGitWorkflowRunRecord(snapshot.record)).toBe(true); + if (isGitWorkflowRunRecord(snapshot.record)) { + expect(snapshot.record.definition.rootDocumentPath).toBe("workflows/release.md"); + } expect(snapshot.executions).toHaveLength(1); expect(snapshot.executions[0]?.stopStatus).toBe("completed"); expect(snapshot.currentWorkspaceRootId).toBe(EMPTY_WORKSPACE_ROOT_ID); From 8ccf0a138c792013e9313e06d0cb147d62398d5b Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Thu, 17 Sep 2026 04:24:57 -0400 Subject: [PATCH 6/8] =?UTF-8?q?=F0=9F=90=9B=20Install=20the=20legacy=20sou?= =?UTF-8?q?rce=20reader=20in=20v1=20CLI=20test=20hosts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/workflow-lifecycle-control.test.ts | 12 +++++++---- .../cli/tests/workflow-suspension.test.ts | 20 +++++++++++-------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/packages/cli/tests/workflow-lifecycle-control.test.ts b/packages/cli/tests/workflow-lifecycle-control.test.ts index 13d2a754d..d6e7226f0 100644 --- a/packages/cli/tests/workflow-lifecycle-control.test.ts +++ b/packages/cli/tests/workflow-lifecycle-control.test.ts @@ -29,6 +29,7 @@ import { Git, suspendFor, WorkflowLifecycle } from "@executablemd/workflow"; import type { WorkflowRunDatabase } from "@executablemd/workflow"; import { collect, inlineSource, registerComponents } from "@executablemd/core"; import { executeInstalled } from "@executablemd/core/host"; +import { readLegacyDefinitionSource } from "../src/workflow-source.ts"; import { runWorkflow } from "../src/workflow.ts"; import type { WorkflowExecution, WorkflowHost, WorkflowRequest } from "../src/workflow.ts"; @@ -267,10 +268,10 @@ interface Attachment { function liveHost(root: string, attachment: Attachment): WorkflowHost { return { useRunHost(): Operation { - return useWorkflowRunHost({ root }); + return useWorkflowRunHost({ root, legacySource: readLegacyDefinitionSource }); }, useLifecycle(): Operation { - return useWorkflowLifecycle({ root }); + return useWorkflowLifecycle({ root, legacySource: readLegacyDefinitionSource }); }, useDelivery(): Operation { return useWorkflowInputDelivery({ root }); @@ -347,7 +348,10 @@ function lifecycleRows(path: string): { status: string; executions: string[] } { function* startedRun(root: string, repository: string, objectId: string): Operation { return yield* scoped(function* () { - const transitions = yield* useWorkflowRunHost({ root }); + const transitions = yield* useWorkflowRunHost({ + root, + legacySource: readLegacyDefinitionSource, + }); const runId = randomUUID(); const acquired = yield* WorkflowLifecycle.operations.acquireExecutor(runId); if (!acquired.ok || acquired.value.kind !== "acquired") { @@ -407,7 +411,7 @@ describe("Tier WFC3 — cancelling a run that is settling into a suspension", () events: [], *onRelease(): Operation { yield* scoped(function* () { - yield* useWorkflowLifecycle({ root }); + yield* useWorkflowLifecycle({ root, legacySource: readLegacyDefinitionSource }); refusal = yield* WorkflowLifecycle.operations.cancel(runId); }); duringRows = lifecycleRows(path); diff --git a/packages/cli/tests/workflow-suspension.test.ts b/packages/cli/tests/workflow-suspension.test.ts index adc54d1bf..8ce29b4fd 100644 --- a/packages/cli/tests/workflow-suspension.test.ts +++ b/packages/cli/tests/workflow-suspension.test.ts @@ -63,6 +63,7 @@ import type { } from "@executablemd/core"; import type { Stream } from "effection"; import { createHash } from "node:crypto"; +import { readLegacyDefinitionSource } from "../src/workflow-source.ts"; import { establishDefinition } from "../src/workflow-definition.ts"; import { runWorkflow } from "../src/workflow.ts"; import type { @@ -113,10 +114,10 @@ function useRunStore(): Operation { function host(root: string, events: string[] = []): WorkflowHost { return { useRunHost(): Operation { - return useWorkflowRunHost({ root }); + return useWorkflowRunHost({ root, legacySource: readLegacyDefinitionSource }); }, useLifecycle(): Operation { - return useWorkflowLifecycle({ root }); + return useWorkflowLifecycle({ root, legacySource: readLegacyDefinitionSource }); }, useDelivery(): Operation { return useWorkflowInputDelivery({ root }); @@ -286,7 +287,10 @@ function retained(path: string): Retained { */ function* createRun(root: string, fixture: Fixture): Operation { return yield* scoped(function* () { - const transitions = yield* useWorkflowRunHost({ root }); + const transitions = yield* useWorkflowRunHost({ + root, + legacySource: readLegacyDefinitionSource, + }); const runId = crypto.randomUUID(); const acquired = yield* WorkflowLifecycle.operations.acquireExecutor(runId); if (!acquired.ok) { @@ -506,7 +510,7 @@ describe("Tier WFS — a suspended run and its no-input resumes", () => { // The lock is free. Acquisition refuses outright while an executor holds // it, so acquiring at all is the proof it was released. yield* scoped(function* () { - yield* useWorkflowLifecycle({ root }); + yield* useWorkflowLifecycle({ root, legacySource: readLegacyDefinitionSource }); const acquired = yield* WorkflowLifecycle.operations.acquireExecutor(runId); expect(acquired.ok).toBe(true); expect(acquired.ok && acquired.value.kind).toBe("acquired"); @@ -725,7 +729,7 @@ describe("Tier WFS — a suspended run and its no-input resumes", () => { // And the real wait, answered while a live workflow executor holds the // lock. Delivery never asks for it, so holding it changes nothing. const delivered = yield* scoped(function* () { - yield* useWorkflowLifecycle({ root }); + yield* useWorkflowLifecycle({ root, legacySource: readLegacyDefinitionSource }); const acquired = yield* WorkflowLifecycle.operations.acquireExecutor(runId); expect(acquired.ok && acquired.value.kind).toBe("acquired"); return yield* manage( @@ -886,10 +890,10 @@ function* useStubAgent(calls: AgentCalls): Operation { function productionHost(root: string, calls: AgentCalls, events: string[] = []): WorkflowHost { return { useRunHost(): Operation { - return useWorkflowRunHost({ root }); + return useWorkflowRunHost({ root, legacySource: readLegacyDefinitionSource }); }, useLifecycle(): Operation { - return useWorkflowLifecycle({ root }); + return useWorkflowLifecycle({ root, legacySource: readLegacyDefinitionSource }); }, useDelivery(): Operation { return useWorkflowInputDelivery({ root }); @@ -1198,7 +1202,7 @@ describe("Tier CKX — a checkpoint a document asked for", () => { // The lock is free: acquisition refuses outright while an executor holds // it, so acquiring at all is the proof it was released. yield* scoped(function* () { - yield* useWorkflowLifecycle({ root }); + yield* useWorkflowLifecycle({ root, legacySource: readLegacyDefinitionSource }); const acquired = yield* WorkflowLifecycle.operations.acquireExecutor(runId); expect(acquired.ok).toBe(true); expect(acquired.ok && acquired.value.kind).toBe("acquired"); From 3e45a6babe00c487dcb740b9aeb893b69de7cbaa Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Thu, 17 Sep 2026 08:13:53 -0400 Subject: [PATCH 7/8] =?UTF-8?q?=F0=9F=90=9B=20Split=20the=20Deno-only=20so?= =?UTF-8?q?urce-bundle=20fork=20tier=20into=20its=20own=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/workflow-fork-source.test.ts | 314 ++++++++++++++++++ packages/workflow/tests/workflow-fork.test.ts | 308 +---------------- scripts/runtime-test-exclusions.ts | 6 + 3 files changed, 324 insertions(+), 304 deletions(-) create mode 100644 packages/workflow/tests/workflow-fork-source.test.ts diff --git a/packages/workflow/tests/workflow-fork-source.test.ts b/packages/workflow/tests/workflow-fork-source.test.ts new file mode 100644 index 000000000..59d467776 --- /dev/null +++ b/packages/workflow/tests/workflow-fork-source.test.ts @@ -0,0 +1,314 @@ +/** + * Tier WFK — admitting a fork of a retained source bundle. + * + * Deno-only, and deliberately its own file: every case here takes the run's + * executor lock, which `packages/workflow/src/deno/advisory-lock.ts` reaches + * through the `Deno` global. The provider-neutral half of the same contract — + * forkability and fork selection, which opens no database — stays in + * `workflow-fork.test.ts` and runs under all three runtimes. + */ + +import { describe, it } from "@executablemd/test-support/bdd"; +import { expect } from "@executablemd/test-support/expect"; +import type { DurableEvent, Json } from "@executablemd/durable-streams"; +import { scoped } from "effection"; +import type { Operation } from "effection"; +import { + forkRunRecordEvent, + LegacyWorkflowSourceReaderUnavailableError, + WorkflowLifecycle, + WorkflowRequestError, +} from "@executablemd/workflow"; +import type { ExecutorLock } from "@executablemd/workflow"; +import type { WorkflowExecutionBegun, WorkflowExecutionTransitions } from "../deno.ts"; +import { useWorkflowRunConnections } from "../src/deno/connections.ts"; +import { SavepointObservation } from "../src/deno/savepoints.ts"; +import { installWorkflowRunStorage } from "../src/deno/provider.ts"; +import { installWorkflowLifecycle } from "../src/deno/lifecycle.ts"; +import { legacySourceReader } from "./support/legacy-source.ts"; +import { + BUNDLE_ENTRYPOINT, + BUNDLE_SOURCE, + creation, + runPath, + SHA1, + sourceBundleCreation, + storedBytes, + tamper, + useStorageRoot, + withExecutor, + withExecutorRun, + withRunHost, +} from "./support/storage.ts"; +import { DatabaseSync } from "node:sqlite"; +import { workflowForkStaging } from "../deno.ts"; + +/** One retained yield, as the journal holds it. */ +function retained(type: string, name = type, result: Json = null): DurableEvent { + return { + type: "yield", + coroutineId: "root", + description: { type, name }, + result: { status: "ok", value: result }, + }; +} + +describe("Tier WFK — a source-bundle fork", () => { + /** One settled source run, and the checkpoint a fork of it may select. */ + function* useForkSource( + root: string, + transitions: WorkflowExecutionTransitions, + ): Operation<{ checkpointEventId: string; rootImport: DurableEvent }> { + const creation = yield* sourceBundleCreation(); + return yield* withExecutorRun( + transitions, + { runId: "fork-source", action: "start", creation }, + function* (begun, executorLock) { + const rootImport: DurableEvent = { + type: "yield", + coroutineId: "root", + description: { type: "import_component", name: "__root__" }, + result: { status: "ok", value: { source: BUNDLE_SOURCE } }, + }; + yield* begun.database.journal.append( + forkRunRecordEvent({ + runId: "fork-source", + definitionVersion: 2, + bundleHash: creation.definition.bundleHash, + }), + ); + yield* begun.database.journal.append(rootImport); + yield* begun.database.journal.append(retained("checkpoint")); + + const entries = yield* begun.database.readJournalEntries(); + if (!entries.ok) { + throw entries.error; + } + const last = entries.value.at(-1); + if (last === undefined) { + throw new Error("the source run retained no checkpoint"); + } + const settled = yield* transitions.settle(executorLock, { + executionId: begun.execution.executionId, + status: "suspended", + }); + if (!settled.ok) { + throw settled.error; + } + void root; + return { checkpointEventId: last.eventId, rootImport }; + }, + ); + } + + it("WFK40: a fork is admitted from its own bytes, and retains its own copy", function* () { + const root = yield* useStorageRoot(); + const candidate = yield* sourceBundleCreation({ content: "# Forked\n\nits own bytes\n" }); + + yield* withRunHost(root, function* (transitions) { + const source = yield* useForkSource(root, transitions); + const forked = yield* withExecutor("fork-destination", function* (executorLock) { + return yield* transitions.fork(executorLock, { + runId: "fork-destination", + selection: { sourceRunId: "fork-source", checkpointEventId: source.checkpointEventId }, + creation: candidate, + rootImport: source.rootImport, + }); + }); + if (!forked.ok) { + throw forked.error; + } + + // The fork's own definition, and the closure it returns is the one its + // store now holds rather than the buffers the caller supplied. + expect(forked.value.record.definition.kind).toBe("source-bundle"); + const sources = forked.value.sources; + expect(sources.definitionVersion).toBe(2); + if (sources.definitionVersion !== 2) { + throw new Error("expected a source bundle"); + } + expect(new TextDecoder().decode(sources.sources[0]?.bytes)).toBe( + "# Forked\n\nits own bytes\n", + ); + expect(sources.definition.bundleHash).toBe(candidate.definition.bundleHash); + expect(sources.definition.bundleHash).not.toBe( + (yield* sourceBundleCreation()).definition.bundleHash, + ); + }); + + // Its schema is version 2, with the source store beside it. + tamper(runPath(root, "fork-destination"), (database) => { + expect(database.prepare("PRAGMA user_version").get()?.["user_version"]).toBe(2); + const stored = database.prepare("SELECT content FROM workflow_definition_blob").get(); + expect(new TextDecoder().decode(storedBytes(stored, "content"))).toBe( + "# Forked\n\nits own bytes\n", + ); + }); + }); + + it("WFK41: a candidate snapshot that is not its descriptor's leaves no fork", function* () { + const root = yield* useStorageRoot(); + const honest = yield* sourceBundleCreation(); + const lying = { + ...honest, + sourceSnapshot: [ + { path: BUNDLE_ENTRYPOINT, bytes: new TextEncoder().encode("# Something else\n") }, + ], + }; + + yield* withRunHost(root, function* (transitions) { + const source = yield* useForkSource(root, transitions); + const refused = yield* withExecutor("fork-lying", function* (executorLock) { + return yield* transitions.fork(executorLock, { + runId: "fork-lying", + selection: { sourceRunId: "fork-source", checkpointEventId: source.checkpointEventId }, + creation: lying, + rootImport: source.rootImport, + }); + }); + + expect(refused.ok).toBe(false); + expect(!refused.ok && refused.error).toBeInstanceOf(WorkflowRequestError); + const found = yield* WorkflowLifecycle.operations.inspect("fork-lying"); + expect(found.ok).toBe(false); + }); + }); + + it("WFK42: a staged fork retains the candidate's own bytes, discoverable by nobody", function* () { + const root = yield* useStorageRoot(); + const staging = "# Staged\n\nthe candidate's own bytes\n"; + const candidate = yield* sourceBundleCreation({ content: staging }); + + yield* withRunHost(root, function* (transitions) { + const source = yield* useForkSource(root, transitions); + const staged = yield* transitions.stageFork({ + runId: "fork-staged", + selection: { sourceRunId: "fork-source", checkpointEventId: source.checkpointEventId }, + creation: candidate, + rootImport: source.rootImport, + }); + if (!staged.ok) { + throw staged.error; + } + expect(staged.value.record.definition.kind).toBe("source-bundle"); + + // What it assembled, read out of the staging file while the resource that + // owns it is still alive. The kind alone would be satisfied by a staging + // copy that retained some other document; the bytes are what a + // compatibility replay would actually run. + const database = new DatabaseSync(workflowForkStaging(root, "fork-staged"), { + readOnly: true, + }); + try { + expect(database.prepare("PRAGMA user_version").get()?.["user_version"]).toBe(2); + const blob = database.prepare("SELECT content FROM workflow_definition_blob").get(); + expect(new TextDecoder().decode(storedBytes(blob, "content"))).toBe(staging); + + const manifest = database + .prepare("SELECT path FROM workflow_definition_source") + .all() + .map((row) => row["path"]); + expect(manifest).toEqual([BUNDLE_ENTRYPOINT]); + } finally { + database.close(); + } + + // And none of it is a run: staging assembles a Workspace to replay + // against, not a destination a host would find. + const found = yield* WorkflowLifecycle.operations.inspect("fork-staged"); + expect(found.ok).toBe(false); + }); + }); + + it("WFK43: every v1 admission is reader-gated, and leaves no destination", function* () { + const root = yield* useStorageRoot(); + + yield* scoped(function* () { + const connections = yield* useWorkflowRunConnections(yield* SavepointObservation.get()); + yield* installWorkflowRunStorage({ root }, {}, connections); + // A host with the reader, so a version-1 source run can exist to fork. + const capable = yield* installWorkflowLifecycle( + { root, legacySource: legacySourceReader() }, + connections, + ); + const source = yield* withExecutorRun( + capable, + { runId: "git-source", action: "start", creation: creation() }, + function* (begun, executorLock) { + const rootImport: DurableEvent = { + type: "yield", + coroutineId: "root", + description: { type: "import_component", name: "__root__" }, + result: { status: "ok", value: { source: "# Release\n" } }, + }; + yield* begun.database.journal.append( + forkRunRecordEvent({ runId: "git-source", base: "main", pinnedCommit: SHA1 }), + ); + yield* begun.database.journal.append(rootImport); + yield* begun.database.journal.append(retained("checkpoint")); + const entries = yield* begun.database.readJournalEntries(); + if (!entries.ok) { + throw entries.error; + } + const last = entries.value.at(-1); + if (last === undefined) { + throw new Error("the source run retained no checkpoint"); + } + const settled = yield* transitionsSettle(capable, executorLock, begun); + void settled; + return { checkpointEventId: last.eventId, rootImport }; + }, + ); + + // And a second host over the same storage with no reader at all. + const blind = yield* installWorkflowLifecycle({ root }, connections); + const request = { + selection: { sourceRunId: "git-source", checkpointEventId: source.checkpointEventId }, + creation: creation(), + rootImport: source.rootImport, + }; + + const resumed = yield* withExecutor("git-source", function* (executorLock) { + return yield* blind.begin(executorLock, { runId: "git-source", action: "resume" }); + }); + expect(resumed.ok).toBe(false); + expect(!resumed.ok && resumed.error).toBeInstanceOf( + LegacyWorkflowSourceReaderUnavailableError, + ); + + const forked = yield* withExecutor("git-fork", function* (executorLock) { + return yield* blind.fork(executorLock, { ...request, runId: "git-fork" }); + }); + expect(forked.ok).toBe(false); + expect(!forked.ok && forked.error).toBeInstanceOf(LegacyWorkflowSourceReaderUnavailableError); + + const staged = yield* blind.stageFork({ ...request, runId: "git-staged" }); + expect(staged.ok).toBe(false); + expect(!staged.ok && staged.error).toBeInstanceOf(LegacyWorkflowSourceReaderUnavailableError); + + // None of the three left a destination anything recognizes, and the + // source run is exactly as it was. + for (const runId of ["git-fork", "git-staged"]) { + const found = yield* WorkflowLifecycle.operations.inspect(runId); + expect({ runId, found: found.ok }).toEqual({ runId, found: false }); + } + const intact = yield* WorkflowLifecycle.operations.inspect("git-source"); + expect(intact.ok).toBe(true); + }); + }); +}); + +/** Settle one begun execution, so a source run stops before it is forked. */ +function* transitionsSettle( + transitions: WorkflowExecutionTransitions, + executorLock: ExecutorLock, + begun: WorkflowExecutionBegun, +): Operation { + const settled = yield* transitions.settle(executorLock, { + executionId: begun.execution.executionId, + status: "suspended", + }); + if (!settled.ok) { + throw settled.error; + } +} diff --git a/packages/workflow/tests/workflow-fork.test.ts b/packages/workflow/tests/workflow-fork.test.ts index 4fec2992c..ef0796b18 100644 --- a/packages/workflow/tests/workflow-fork.test.ts +++ b/packages/workflow/tests/workflow-fork.test.ts @@ -6,6 +6,10 @@ * Neither opens a database, so both are exercised here against retained shapes * a run could hold rather than against a run that had to be produced. * + * Nothing here opens a database, so this file runs under every runtime. The + * admission half of the same contract takes the run's executor lock and is + * therefore Deno's: it lives in `workflow-fork-source.test.ts`. + * * The blockers are the reason this tier is not folded into the CLI's: an Agent * turn and an effect a later build wrote are histories this build cannot * produce on purpose, and a test that waited for one would assert nothing on @@ -25,37 +29,6 @@ import { selectForkPrefix, } from "@executablemd/workflow"; import type { ForkCandidate } from "@executablemd/workflow"; -import { scoped } from "effection"; -import type { Operation } from "effection"; -import { - forkRunRecordEvent, - LegacyWorkflowSourceReaderUnavailableError, - WorkflowLifecycle, - WorkflowRequestError, -} from "@executablemd/workflow"; -import type { ExecutorLock } from "@executablemd/workflow"; -import type { WorkflowExecutionBegun, WorkflowExecutionTransitions } from "../deno.ts"; -import { useWorkflowRunConnections } from "../src/deno/connections.ts"; -import { SavepointObservation } from "../src/deno/savepoints.ts"; -import { installWorkflowRunStorage } from "../src/deno/provider.ts"; -import { installWorkflowLifecycle } from "../src/deno/lifecycle.ts"; -import { legacySourceReader } from "./support/legacy-source.ts"; -import { - BUNDLE_ENTRYPOINT, - BUNDLE_SOURCE, - creation, - runPath, - SHA1, - sourceBundleCreation, - storedBytes, - tamper, - useStorageRoot, - withExecutor, - withExecutorRun, - withRunHost, -} from "./support/storage.ts"; -import { DatabaseSync } from "node:sqlite"; -import { workflowForkStaging } from "../deno.ts"; const ROOT_A = "a".repeat(64); const ROOT_B = "b".repeat(64); @@ -288,276 +261,3 @@ describe("Tier WFK — forkability and fork selection", () => { } }); }); - -/** - * Tier WFK — admitting a fork of a retained source bundle. - * - * A fork's candidate is its own definition, so a version-2 fork is created from - * its own exact bytes exactly as a version-2 start is: the snapshot is copied - * and held to the descriptor before the destination exists, and what the fork - * retains afterwards is the store's copy rather than the caller's array. - * - * The same reader gate applies to every version-1 lifecycle admission. `begin`, - * `fork` and the private staging path each obtain and validate the Markdown a - * Git definition names before they write, so a host that cannot obtain it - * leaves no destination behind at all. - */ -describe("Tier WFK — a source-bundle fork", () => { - /** One settled source run, and the checkpoint a fork of it may select. */ - function* useForkSource( - root: string, - transitions: WorkflowExecutionTransitions, - ): Operation<{ checkpointEventId: string; rootImport: DurableEvent }> { - const creation = yield* sourceBundleCreation(); - return yield* withExecutorRun( - transitions, - { runId: "fork-source", action: "start", creation }, - function* (begun, executorLock) { - const rootImport: DurableEvent = { - type: "yield", - coroutineId: "root", - description: { type: "import_component", name: "__root__" }, - result: { status: "ok", value: { source: BUNDLE_SOURCE } }, - }; - yield* begun.database.journal.append( - forkRunRecordEvent({ - runId: "fork-source", - definitionVersion: 2, - bundleHash: creation.definition.bundleHash, - }), - ); - yield* begun.database.journal.append(rootImport); - yield* begun.database.journal.append(retained("checkpoint")); - - const entries = yield* begun.database.readJournalEntries(); - if (!entries.ok) { - throw entries.error; - } - const last = entries.value.at(-1); - if (last === undefined) { - throw new Error("the source run retained no checkpoint"); - } - const settled = yield* transitions.settle(executorLock, { - executionId: begun.execution.executionId, - status: "suspended", - }); - if (!settled.ok) { - throw settled.error; - } - void root; - return { checkpointEventId: last.eventId, rootImport }; - }, - ); - } - - it("WFK40: a fork is admitted from its own bytes, and retains its own copy", function* () { - const root = yield* useStorageRoot(); - const candidate = yield* sourceBundleCreation({ content: "# Forked\n\nits own bytes\n" }); - - yield* withRunHost(root, function* (transitions) { - const source = yield* useForkSource(root, transitions); - const forked = yield* withExecutor("fork-destination", function* (executorLock) { - return yield* transitions.fork(executorLock, { - runId: "fork-destination", - selection: { sourceRunId: "fork-source", checkpointEventId: source.checkpointEventId }, - creation: candidate, - rootImport: source.rootImport, - }); - }); - if (!forked.ok) { - throw forked.error; - } - - // The fork's own definition, and the closure it returns is the one its - // store now holds rather than the buffers the caller supplied. - expect(forked.value.record.definition.kind).toBe("source-bundle"); - const sources = forked.value.sources; - expect(sources.definitionVersion).toBe(2); - if (sources.definitionVersion !== 2) { - throw new Error("expected a source bundle"); - } - expect(new TextDecoder().decode(sources.sources[0]?.bytes)).toBe( - "# Forked\n\nits own bytes\n", - ); - expect(sources.definition.bundleHash).toBe(candidate.definition.bundleHash); - expect(sources.definition.bundleHash).not.toBe( - (yield* sourceBundleCreation()).definition.bundleHash, - ); - }); - - // Its schema is version 2, with the source store beside it. - tamper(runPath(root, "fork-destination"), (database) => { - expect(database.prepare("PRAGMA user_version").get()?.["user_version"]).toBe(2); - const stored = database.prepare("SELECT content FROM workflow_definition_blob").get(); - expect(new TextDecoder().decode(storedBytes(stored, "content"))).toBe( - "# Forked\n\nits own bytes\n", - ); - }); - }); - - it("WFK41: a candidate snapshot that is not its descriptor's leaves no fork", function* () { - const root = yield* useStorageRoot(); - const honest = yield* sourceBundleCreation(); - const lying = { - ...honest, - sourceSnapshot: [ - { path: BUNDLE_ENTRYPOINT, bytes: new TextEncoder().encode("# Something else\n") }, - ], - }; - - yield* withRunHost(root, function* (transitions) { - const source = yield* useForkSource(root, transitions); - const refused = yield* withExecutor("fork-lying", function* (executorLock) { - return yield* transitions.fork(executorLock, { - runId: "fork-lying", - selection: { sourceRunId: "fork-source", checkpointEventId: source.checkpointEventId }, - creation: lying, - rootImport: source.rootImport, - }); - }); - - expect(refused.ok).toBe(false); - expect(!refused.ok && refused.error).toBeInstanceOf(WorkflowRequestError); - const found = yield* WorkflowLifecycle.operations.inspect("fork-lying"); - expect(found.ok).toBe(false); - }); - }); - - it("WFK42: a staged fork retains the candidate's own bytes, discoverable by nobody", function* () { - const root = yield* useStorageRoot(); - const staging = "# Staged\n\nthe candidate's own bytes\n"; - const candidate = yield* sourceBundleCreation({ content: staging }); - - yield* withRunHost(root, function* (transitions) { - const source = yield* useForkSource(root, transitions); - const staged = yield* transitions.stageFork({ - runId: "fork-staged", - selection: { sourceRunId: "fork-source", checkpointEventId: source.checkpointEventId }, - creation: candidate, - rootImport: source.rootImport, - }); - if (!staged.ok) { - throw staged.error; - } - expect(staged.value.record.definition.kind).toBe("source-bundle"); - - // What it assembled, read out of the staging file while the resource that - // owns it is still alive. The kind alone would be satisfied by a staging - // copy that retained some other document; the bytes are what a - // compatibility replay would actually run. - const database = new DatabaseSync(workflowForkStaging(root, "fork-staged"), { - readOnly: true, - }); - try { - expect(database.prepare("PRAGMA user_version").get()?.["user_version"]).toBe(2); - const blob = database.prepare("SELECT content FROM workflow_definition_blob").get(); - expect(new TextDecoder().decode(storedBytes(blob, "content"))).toBe(staging); - - const manifest = database - .prepare("SELECT path FROM workflow_definition_source") - .all() - .map((row) => row["path"]); - expect(manifest).toEqual([BUNDLE_ENTRYPOINT]); - } finally { - database.close(); - } - - // And none of it is a run: staging assembles a Workspace to replay - // against, not a destination a host would find. - const found = yield* WorkflowLifecycle.operations.inspect("fork-staged"); - expect(found.ok).toBe(false); - }); - }); - - it("WFK43: every v1 admission is reader-gated, and leaves no destination", function* () { - const root = yield* useStorageRoot(); - - yield* scoped(function* () { - const connections = yield* useWorkflowRunConnections(yield* SavepointObservation.get()); - yield* installWorkflowRunStorage({ root }, {}, connections); - // A host with the reader, so a version-1 source run can exist to fork. - const capable = yield* installWorkflowLifecycle( - { root, legacySource: legacySourceReader() }, - connections, - ); - const source = yield* withExecutorRun( - capable, - { runId: "git-source", action: "start", creation: creation() }, - function* (begun, executorLock) { - const rootImport: DurableEvent = { - type: "yield", - coroutineId: "root", - description: { type: "import_component", name: "__root__" }, - result: { status: "ok", value: { source: "# Release\n" } }, - }; - yield* begun.database.journal.append( - forkRunRecordEvent({ runId: "git-source", base: "main", pinnedCommit: SHA1 }), - ); - yield* begun.database.journal.append(rootImport); - yield* begun.database.journal.append(retained("checkpoint")); - const entries = yield* begun.database.readJournalEntries(); - if (!entries.ok) { - throw entries.error; - } - const last = entries.value.at(-1); - if (last === undefined) { - throw new Error("the source run retained no checkpoint"); - } - const settled = yield* transitionsSettle(capable, executorLock, begun); - void settled; - return { checkpointEventId: last.eventId, rootImport }; - }, - ); - - // And a second host over the same storage with no reader at all. - const blind = yield* installWorkflowLifecycle({ root }, connections); - const request = { - selection: { sourceRunId: "git-source", checkpointEventId: source.checkpointEventId }, - creation: creation(), - rootImport: source.rootImport, - }; - - const resumed = yield* withExecutor("git-source", function* (executorLock) { - return yield* blind.begin(executorLock, { runId: "git-source", action: "resume" }); - }); - expect(resumed.ok).toBe(false); - expect(!resumed.ok && resumed.error).toBeInstanceOf( - LegacyWorkflowSourceReaderUnavailableError, - ); - - const forked = yield* withExecutor("git-fork", function* (executorLock) { - return yield* blind.fork(executorLock, { ...request, runId: "git-fork" }); - }); - expect(forked.ok).toBe(false); - expect(!forked.ok && forked.error).toBeInstanceOf(LegacyWorkflowSourceReaderUnavailableError); - - const staged = yield* blind.stageFork({ ...request, runId: "git-staged" }); - expect(staged.ok).toBe(false); - expect(!staged.ok && staged.error).toBeInstanceOf(LegacyWorkflowSourceReaderUnavailableError); - - // None of the three left a destination anything recognizes, and the - // source run is exactly as it was. - for (const runId of ["git-fork", "git-staged"]) { - const found = yield* WorkflowLifecycle.operations.inspect(runId); - expect({ runId, found: found.ok }).toEqual({ runId, found: false }); - } - const intact = yield* WorkflowLifecycle.operations.inspect("git-source"); - expect(intact.ok).toBe(true); - }); - }); -}); - -/** Settle one begun execution, so a source run stops before it is forked. */ -function* transitionsSettle( - transitions: WorkflowExecutionTransitions, - executorLock: ExecutorLock, - begun: WorkflowExecutionBegun, -): Operation { - const settled = yield* transitions.settle(executorLock, { - executionId: begun.execution.executionId, - status: "suspended", - }); - if (!settled.ok) { - throw settled.error; - } -} diff --git a/scripts/runtime-test-exclusions.ts b/scripts/runtime-test-exclusions.ts index 5bdca9e29..33456d99d 100644 --- a/scripts/runtime-test-exclusions.ts +++ b/scripts/runtime-test-exclusions.ts @@ -389,6 +389,12 @@ const DENO_ONLY_TOOLING: RuntimeExclusion[] = [ "every case takes the run's executor lock, which packages/workflow/src/deno/advisory-lock.ts reaches through the `Deno` global; no other runtime has one, so the acquisition refuses before an export begins", issue: DERIVED_SCOPE, }, + { + path: "packages/workflow/tests/workflow-fork-source.test.ts", + reason: + "every case admits a fork through the lifecycle, which takes the run's executor lock; packages/workflow/src/deno/advisory-lock.ts reaches that lock through the `Deno` global and refuses outright under any other runtime. The provider-neutral half of the fork contract — forkability classification and fork selection, which opens no database — is packages/workflow/tests/workflow-fork.test.ts and runs on all three", + issue: DERIVED_SCOPE, + }, { path: "packages/cli/tests/workflow-retention.test.ts", reason: From 897cef35698d21337c4c7eac7f6bc666bb3c5415 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Thu, 17 Sep 2026 08:57:58 -0400 Subject: [PATCH 8/8] =?UTF-8?q?=F0=9F=90=9B=20Settle=20a=20committed=20run?= =?UTF-8?q?=20interrupted=20from=20the=20executor=20hold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/cli/src/workflow.ts | 64 +-- packages/workflow/src/deno/executor.ts | 24 +- packages/workflow/src/deno/transitions.ts | 81 ++++ .../tests/support/executor-death-child.ts | 71 +++ packages/workflow/tests/support/storage.ts | 57 +++ .../tests/workflow-lifecycle-control.test.ts | 422 +++++++++++++++++- .../workflow-lifecycle-inspection.test.ts | 28 +- .../tests/workflow-run-storage.test.ts | 9 +- 8 files changed, 708 insertions(+), 48 deletions(-) create mode 100644 packages/workflow/tests/support/executor-death-child.ts diff --git a/packages/cli/src/workflow.ts b/packages/cli/src/workflow.ts index c86217330..a49c880ef 100644 --- a/packages/cli/src/workflow.ts +++ b/packages/cli/src/workflow.ts @@ -976,39 +976,24 @@ export function runWorkflow( } const { database, record, execution, replay } = begun.value; - // Only after the creation transaction committed. A run id reported before - // it would name something a failure could still leave absent. - reportRun(record.runId); - // What the transition authenticated, and nothing this command read. - const source = executableSources(begun.value.sources); - if (!source.ok) { - report(source.error.message); - return { exitCode: 1 }; - } - - // Interruption is the outcome nothing else publishes. Registered before the - // execution starts, so a scope torn down by Ctrl-C settles the run rather - // than leaving a record with no end and a status of `running`. The executor - // lock outlives this finalizer, so its settlement remains authorized. + // Before anything that can suspend. `admit()` handed back a committed + // execution receipt, so the run is already durable and already `running`; + // everything below — reporting the id, projecting the sources, importing + // the Deno adapter — yields, and a cancellation landing in any of them has + // to find this finalizer already registered rather than still to come. The + // executor lock outlives it, so its settlement stays authorized. + // + // This is the reporting half of interruption. The begin transition installs + // its own settlement on the executor hold, inside the transaction that + // recorded the execution, which is what makes the durable outcome + // guaranteed rather than merely early; this runs first, so that backstop + // finds the execution already finished and leaves it alone. // // The phase, rather than a boolean: "the document produced an outcome" and // "this invocation is durably settled" are different facts, and collapsing // them is how a post-execution storage refusal would be republished as an // interruption. Teardown speaks only while the phase is still `running`. - // Imported where it is used rather than at the top of this module. This - // file is on the ordinary `xmd run` path too, and the Deno workflow adapter - // reaches `node:sqlite` — which Node greets with an experimental warning on - // standard error the moment it loads. A run that opens no workflow storage - // should not be announcing that it might have. - // `evaluationProfile` comes through the same import, and for the same - // reason: it is closed over this run's storage, and a run that opens no - // workflow storage must not load the adapter that reaches `node:sqlite` — - // which Bun does not have at all. - const { createSuspensionController, evaluationProfile } = yield* until( - import("@executablemd/workflow/deno"), - ); - const suspension = createSuspensionController({ database }); const phase: LifecyclePhase = { state: "running" }; yield* ensure(function* () { if (phase.state !== "running") { @@ -1035,6 +1020,31 @@ export function runWorkflow( reportStatus("interrupted"); }); + // Only after the creation transaction committed. A run id reported before + // it would name something a failure could still leave absent. + reportRun(record.runId); + + // What the transition authenticated, and nothing this command read. + const source = executableSources(begun.value.sources); + if (!source.ok) { + report(source.error.message); + return { exitCode: 1 }; + } + + // Imported where it is used rather than at the top of this module. This + // file is on the ordinary `xmd run` path too, and the Deno workflow adapter + // reaches `node:sqlite` — which Node greets with an experimental warning on + // standard error the moment it loads. A run that opens no workflow storage + // should not be announcing that it might have. + // `evaluationProfile` comes through the same import, and for the same + // reason: it is closed over this run's storage, and a run that opens no + // workflow storage must not load the adapter that reaches `node:sqlite` — + // which Bun does not have at all. + const { createSuspensionController, evaluationProfile } = yield* until( + import("@executablemd/workflow/deno"), + ); + const suspension = createSuspensionController({ database }); + const completed = yield* isCompleted(database.journal); const documentExecution: WorkflowExecution = { // The exact target the run retains, never a selector re-resolved now: a diff --git a/packages/workflow/src/deno/executor.ts b/packages/workflow/src/deno/executor.ts index 2fcdfa2b5..dc05894c2 100644 --- a/packages/workflow/src/deno/executor.ts +++ b/packages/workflow/src/deno/executor.ts @@ -62,6 +62,22 @@ export interface ExecutorLockHold { * it away. */ execution?: string; + /** + * Settle this acquisition's execution as interrupted, if it is still running. + * + * Installed by the begin transaction itself, at the statement that records + * the execution — so from the instant a run commits there is a way to finish + * it, and no window exists in which a caller has yet to register one. It + * closes over the open connection, the run's path and the exact execution + * id, so teardown performs no lookup that could fail or suspend after + * cancellation has begun. + * + * Synchronous on purpose. Teardown runs after every child of this + * acquisition has been halted, so nothing else holds the connection, and a + * settlement that could suspend here would be one more thing cancellation + * could arrive in the middle of. + */ + settleInterruption?: () => void; } /** @@ -136,8 +152,14 @@ export function createExecutorLockRegistry(): ExecutorLockRegistry { // Registered as soon as the lock is held, so a failure between here and // the caller's first transition still retires it. It runs before the // acquisition beneath releases the file, so no lock is ever open to the - // operating system while this registry still answers for it. + // operating system while this registry still answers for it — and the + // run is therefore published interrupted before anybody else can take + // the lock and read it. yield* ensure(() => { + // Whatever this acquisition began and nobody finished. A settled + // execution is left exactly as it settled: this finishes what is + // still running rather than relabelling an outcome that already won. + hold.settleInterruption?.(); hold.open = false; holds.delete(hold.lock); }); diff --git a/packages/workflow/src/deno/transitions.ts b/packages/workflow/src/deno/transitions.ts index 64a4e4496..de6b00e02 100644 --- a/packages/workflow/src/deno/transitions.ts +++ b/packages/workflow/src/deno/transitions.ts @@ -167,6 +167,7 @@ export function* beginExecution( if (!outcome.ok) { // The transaction rolled back, so nothing was begun after all. hold.execution = undefined; + hold.settleInterruption = undefined; return outcome; } if (outcome.value.kind === "refused") { @@ -395,6 +396,80 @@ function* settleSources( return yield* authenticate(stored.value, readLegacySource); } +/** + * What finishes this execution if its host is torn down before it settles. + * + * Built inside the transaction that inserted the execution, so it closes over + * the connection that is already open, the path, and the exact execution id — + * and therefore asks nothing of the world at the moment it runs. The executor + * hold calls it during teardown, before the advisory lock beneath is released, + * which is what makes `interrupted` the status the next acquisition finds + * rather than a stale `running`. + * + * Three things it deliberately does not do. It does not reread the run's + * source: what a run executes was settled when it began. It does not look the + * execution up first — one guarded `UPDATE` is the whole decision, and what it + * changed is what says whether there was anything to finish. And it does not + * throw: a teardown backstop that raised would replace the failure that caused + * the teardown with its own. + * + * A transaction that rolled back leaves no row at all, and an execution that + * settled on its own terms leaves one that no longer matches. Both change + * nothing, so neither is relabelled and neither publishes a run status. A + * cancellation before the creation committed therefore leaves nothing + * recognized and nothing to reuse the run id around. + */ +function interruptionSettler( + connection: RunConnection, + path: string, + executionId: string, +): () => void { + let spent = false; + return () => { + if (spent) { + return; + } + spent = true; + const completion: DocumentExecutionCompletion = { + executionId, + status: "interrupted", + reason: { kind: "host", code: "executor-interrupted" }, + }; + try { + inLifecycleTransaction(connection, path, () => { + // The update decides it, rather than a read this then acts on. Its own + // `WHERE execution_id = ? AND stopped_at IS NULL` is the condition, so + // a row that was never inserted and a row that already settled are the + // same answer — nothing changed — and neither is asked about twice. + const columns = stopReasonColumns(completion.reason); + const changed = withStopReason(path, () => + connection.database + .prepare(FINISH_EXECUTION) + .run( + new Date().toISOString(), + completion.status, + columns.kind, + columns.code, + columns.eventId, + completion.executionId, + ), + ); + // Published only for an execution this actually finished. A run whose + // execution settled on its own terms keeps the status that settled it. + if (changed.changes === 0) { + return; + } + publish(connection.database, path, completion.status, completion.reason); + }); + } catch { + // Teardown owes the caller nothing it can act on here. The run keeps + // whatever it last held, and the next acquisition reconciles it as the + // unfinished execution of a workflow executor that went away. + return; + } + }; +} + /** * What one begin transaction committed. * @@ -501,6 +576,10 @@ function beginOnce( const { database } = connection; const begun = begin(connection, path, hold, request, recovery, owned); hold.execution = begun.execution.executionId; + // Inside the transaction that records the execution, so the run is never + // durable without a way to finish it. Everything the settlement needs is + // captured here; nothing is looked up after cancellation has begun. + hold.settleInterruption = interruptionSettler(connection, path, begun.execution.executionId); return { ...begun, ...(recovery.closed === undefined ? {} : { closed: recovery.closed }), @@ -565,6 +644,7 @@ export function* forkExecution( if (!outcome.ok) { // The transaction rolled back, so nothing was forked after all. hold.execution = undefined; + hold.settleInterruption = undefined; return outcome; } if (outcome.value.kind === "refused") { @@ -728,6 +808,7 @@ function forkOnce( writeForkInheritance(connection, transaction, snapshot, head); const execution = insertExecution(database); hold.execution = execution.executionId; + hold.settleInterruption = interruptionSettler(connection, path, execution.executionId); return { kind: "begun", record: readRunRow(database, path), execution, replay: false }; } diff --git a/packages/workflow/tests/support/executor-death-child.ts b/packages/workflow/tests/support/executor-death-child.ts new file mode 100644 index 000000000..fa479fbfb --- /dev/null +++ b/packages/workflow/tests/support/executor-death-child.ts @@ -0,0 +1,71 @@ +/** + * A workflow executor that begins a run and is then lost, running no cleanup. + * + * ```sh + * deno run -A executor-death-child.ts + * ``` + * + * It takes the run's executor lock, begins the first document execution, says + * it is ready and waits to be killed. What it leaves is the state the + * architecture's staleness premise is about: a run durably `running`, one + * execution with no end, and an advisory lock the kernel released because the + * process holding it is gone. + * + * It is a whole process on purpose, and the reason is the same one + * `workflow-recovery-child.ts` gives for its own: nothing a test can do to + * itself leaves an execution unfinished any more. A thrown error, a cancelled + * task and a closed scope all unwind, and unwinding now settles the execution + * the acquisition began — that is what the executor hold's teardown is for. A + * killed process runs no finalizer, which is exactly what makes it the only + * honest way to produce a dead executor's leftovers. + */ + +import process from "node:process"; +import { main, suspend } from "effection"; +import { WorkflowLifecycle } from "../../mod.ts"; +import { useWorkflowRunHost } from "../../deno.ts"; +import { creation } from "./storage.ts"; +import { legacySourceReader } from "./legacy-source.ts"; + +/** Said on stdout, and nowhere else, once the execution is durably begun. */ +const READY = "READY"; + +await main(function* () { + // `process.argv` rather than `Deno.args`: this file is Deno-only to run, and + // still has to typecheck under the Node project like every other source. + const [root, runId] = process.argv.slice(2); + if (root === undefined || runId === undefined) { + throw new Error("usage: executor-death-child.ts "); + } + + // `creation()` is a version-1 fixture, and every executable v1 admission is + // reader-gated, so this host supplies the reader exactly as a Git-capable + // one does. + const transitions = yield* useWorkflowRunHost({ root, legacySource: legacySourceReader() }); + const acquired = yield* WorkflowLifecycle.operations.acquireExecutor(runId); + if (!acquired.ok) { + throw acquired.error; + } + if (acquired.value.kind !== "acquired") { + throw new Error(`${runId} already has a live workflow executor`); + } + + const begun = yield* transitions.begin(acquired.value.lock, { + runId, + action: "start", + creation: creation(), + }); + if (!begun.ok) { + throw begun.error; + } + + // Only after the transaction committed: a reader told this was ready before + // the run existed would be told about a run that might never appear. + console.log(READY); + + // Nothing settles it, and nothing here will. A timer the event loop can see + // is what keeps a suspended Effection task's process alive until the signal + // arrives. + setInterval(() => {}, 1_000); + yield* suspend(); +}); diff --git a/packages/workflow/tests/support/storage.ts b/packages/workflow/tests/support/storage.ts index e6d0fbe87..bfa72e4cd 100644 --- a/packages/workflow/tests/support/storage.ts +++ b/packages/workflow/tests/support/storage.ts @@ -43,6 +43,11 @@ import { legacySourceReader } from "./legacy-source.ts"; import { parseSourceBundleDefinition, sourceBundleHash, sourceContentHash } from "../../mod.ts"; import type { SourceBundleWorkflowRunCreationV2 } from "../../deno.ts"; import type { Json } from "@executablemd/durable-streams"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { exec } from "@effectionx/process"; +import { when } from "@effectionx/converge"; +import { ensure, spawn } from "effection"; export const SHA1 = "9fceb02d0ae598e95dc970b74767f19372d61af8"; @@ -408,3 +413,55 @@ export function storedBytes(row: Record | undefined, column: st } return value; } + +const DEATH_CHILD = fileURLToPath(new URL("./executor-death-child.ts", import.meta.url)); +const REPOSITORY = fileURLToPath(new URL("../../..", import.meta.url)); + +/** + * Leave `runId` the way a workflow executor that died leaves it. + * + * A run durably `running`, one execution with no end, and an advisory lock the + * kernel released rather than a host did. It takes a whole process because a + * process is the only thing that can be lost: a scope that closes in this one + * runs the executor hold's teardown, and that teardown settles the execution + * the acquisition began. Ending a scope therefore proves the opposite of what + * a dead executor leaves, which is why nothing here stands in for the child. + */ +export function* runLeftUnfinished(root: string, runId: string): Operation { + yield* scoped(function* () { + const child = yield* exec(process.execPath, { + arguments: ["run", "--allow-all", "--frozen", DEATH_CHILD, root, runId], + cwd: REPOSITORY, + }); + let announced = false; + yield* spawn(function* () { + const output = yield* child.stdout; + let next = yield* output.next(); + while (!next.done) { + if (new TextDecoder().decode(next.value).includes("READY")) { + announced = true; + } + next = yield* output.next(); + } + }); + // Killed rather than asked: this child exists to be lost, and a process + // suspended on purpose has no other way to end. + yield* ensure(function* () { + try { + process.kill(child.pid, "SIGKILL"); + } catch { + // Already gone, which is the outcome this wanted. + } + yield* child.join(); + }); + yield* when( + function* () { + if (!announced) { + throw new Error(`the executor child has not begun ${runId} yet`); + } + }, + { timeout: 30_000 }, + ); + process.kill(child.pid, "SIGKILL"); + }); +} diff --git a/packages/workflow/tests/workflow-lifecycle-control.test.ts b/packages/workflow/tests/workflow-lifecycle-control.test.ts index 65976062c..78dc56576 100644 --- a/packages/workflow/tests/workflow-lifecycle-control.test.ts +++ b/packages/workflow/tests/workflow-lifecycle-control.test.ts @@ -15,13 +15,31 @@ import { DatabaseSync } from "node:sqlite"; import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { exists } from "@effectionx/fs"; -import { scoped, until } from "effection"; +import { scoped, spawn, until, withResolvers } from "effection"; import type { Operation } from "effection"; import { WorkflowLifecycle, WorkflowRunNotFoundError } from "../mod.ts"; import type { WorkflowRunRecord, WorkflowRunStatus } from "../mod.ts"; import { useWorkflowLifecycle, workflowRunLock, workflowRunPath } from "../deno.ts"; -import { creation, useStorageRoot, withExecutorRun, withRunHost } from "./support/storage.ts"; +import { + BUNDLE_ENTRYPOINT, + BUNDLE_SOURCE, + creation, + runLeftUnfinished, + sourceBundleCreation, + storedBytes, + useStorageRoot, + withExecutor, + withExecutorRun, + withRunHost, +} from "./support/storage.ts"; +import { useWorkflowRunConnections } from "../src/deno/connections.ts"; +import type { RunConnection, WorkflowRunConnections } from "../src/deno/connections.ts"; +import { SavepointObservation } from "../src/deno/savepoints.ts"; +import { installWorkflowRunStorage } from "../src/deno/provider.ts"; +import { installWorkflowLifecycle } from "../src/deno/lifecycle.ts"; import { legacySourceReader } from "./support/legacy-source.ts"; +import { parseSourceBundleDefinition } from "../mod.ts"; +import type { LegacyWorkflowSourceReader, WorkflowBeginRequest } from "../deno.ts"; const { cancel } = WorkflowLifecycle.operations; @@ -38,15 +56,17 @@ function* runEndedAs( runId: string, status: WorkflowRunStatus | "unfinished", ): Operation { + if (status === "unfinished") { + // A whole process, killed. Returning from a scope here would run the + // executor hold's teardown, which settles the execution the acquisition + // began — the opposite of what a workflow executor that went away leaves. + return yield* runLeftUnfinished(root, runId); + } yield* withRunHost(root, function* (transitions) { yield* withExecutorRun( transitions, { runId, action: "start", creation: creation() }, function* (begun, executorLock) { - if (status === "unfinished") { - // Nothing settles it: the workflow executor went away mid-execution. - return; - } const settled = yield* transitions.settle(executorLock, { executionId: begun.execution.executionId, status, @@ -289,3 +309,393 @@ describe("Tier WLC — cancellation and deletion", () => { expect((yield* until(readFile(neighbour))).toString("base64")).toBe(neighbourBytes); }); }); + +/** + * Tier WLT — a run cancelled after its creation committed. + * + * Between the moment a creation transaction commits and the moment its caller + * has registered anything of its own there is real work: the begin transition + * reads its retained source back and authenticates it, the caller reports the + * run id, projects the closure and imports the Deno adapter. Every one of those + * suspends, and a cancellation arriving in any of them used to leave the + * durable run and its execution saying `running` while the executor lock was + * released — a run nothing was advancing, that the next acquisition would have + * to reconcile as somebody else's leftovers. + * + * The two cases below are those two windows, taken deterministically rather + * than by timing: one halts the begin transition while it is still inside + * source settlement, the other halts after it returned and before anything was + * registered. Both read the durable file directly afterwards, with no executor + * lock taken, so what they observe is what teardown itself wrote and not what a + * later acquisition's recovery would have made of it. + * + * The negative control is the third case. Before the transaction commits there + * is nothing to settle and nothing to find, and the run id is free. + */ +describe("Tier WLT — interrupted after the creation transaction committed", () => { + /** The durable row, read without taking the run's executor lock. */ + /** + * One row, narrowed to what a row is rather than asserted into it. + * + * SQLite hands back `unknown`, and a value that is not an object is a failure + * about the row — not a `TypeError` somewhere downstream reading a member off + * it. + */ + function row(value: unknown, what: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${what} is not a row`); + } + return { ...value }; + } + + function rows(values: readonly unknown[], what: string): Record[] { + return values.map((value) => row(value, what)); + } + + function count(value: unknown, what: string): number { + const n = row(value, what)["n"]; + if (typeof n !== "number") { + throw new Error(`${what} counted nothing`); + } + return n; + } + + interface Retained { + readonly status: unknown; + readonly reasonKind: unknown; + readonly reasonCode: unknown; + readonly executions: Record[]; + readonly sources: Record[]; + readonly definition: unknown; + readonly events: number; + } + + function retained(root: string, runId: string): Retained { + const database = new DatabaseSync(workflowRunPath(root, runId), { readOnly: true }); + try { + const run = row(database.prepare("SELECT * FROM workflow_run").get(), "the run"); + return { + status: run["status"], + reasonKind: run["stop_reason_kind"], + reasonCode: run["stop_reason_code"], + definition: run["definition"], + executions: rows( + database.prepare("SELECT * FROM document_executions ORDER BY sequence ASC").all(), + "a document execution", + ), + sources: rows( + database + .prepare( + "SELECT s.path, s.source_hash, b.byte_length, b.content FROM " + + "workflow_definition_source s JOIN workflow_definition_blob b " + + "ON b.source_hash = s.source_hash ORDER BY s.path ASC", + ) + .all(), + "a retained source", + ), + events: count( + database.prepare("SELECT count(*) AS n FROM journal_events").get(), + "the journal", + ), + }; + } finally { + database.close(); + } + } + + /** Everything both post-commit windows owe, whichever one produced them. */ + function* settledInterrupted(root: string, runId: string): Operation { + const after = retained(root, runId); + + // The complete version-2 snapshot is durable: the descriptor, and the exact + // bytes behind every path it names. Read through the same parser a host + // reads it through, so what is inspected is a descriptor rather than + // whatever shape `JSON.parse` happened to produce. + const parsed = parseSourceBundleDefinition(JSON.parse(String(after.definition))); + if (!parsed.ok) { + throw parsed.error; + } + const definition = parsed.value; + expect(definition.version).toBe(2); + expect(definition.kind).toBe("source-bundle"); + expect(definition.entrypoint).toBe(BUNDLE_ENTRYPOINT); + expect(definition.sources).toHaveLength(1); + expect(after.sources).toHaveLength(1); + expect(after.sources[0]?.["path"]).toBe(BUNDLE_ENTRYPOINT); + expect(after.sources[0]?.["source_hash"]).toBe(definition.sources[0]?.sourceHash); + expect(new TextDecoder().decode(storedBytes(after.sources[0], "content"))).toBe(BUNDLE_SOURCE); + + // The exact first execution is finished, and finished as this and nothing + // else. `executor-interrupted` is what teardown writes; a later + // acquisition reconciling leftovers would have written something else. + expect(after.executions).toHaveLength(1); + expect(after.executions[0]?.["stopped_at"]).not.toBe(null); + expect(after.executions[0]?.["stop_status"]).toBe("interrupted"); + expect(after.executions[0]?.["stop_reason_kind"]).toBe("host"); + expect(after.executions[0]?.["stop_reason_code"]).toBe("executor-interrupted"); + + // And the run publishes it, rather than staying `running` beside a + // finished execution. + expect(after.status).toBe("interrupted"); + expect(after.reasonKind).toBe("host"); + expect(after.reasonCode).toBe("executor-interrupted"); + + // Nothing ran: no root import, no authored effect, no journal at all. + expect(after.events).toBe(0); + + // The lock is free, and it was already settled when it became free — the + // reads above took no lock, so nothing between teardown and here could + // have written what they saw. Acquiring is what proves the release. + yield* withRunHost(root, function* () { + yield* withExecutor(runId, function* () { + expect(retained(root, runId).status).toBe("interrupted"); + }); + }); + + // And the run resumes from the bytes it retained, not from anything a host + // would have to go and find. + yield* withRunHost(root, function* (transitions) { + yield* withExecutor(runId, function* (executorLock) { + const resumed = yield* transitions.begin(executorLock, { runId, action: "resume" }); + if (!resumed.ok) { + throw resumed.error; + } + const { sources } = resumed.value; + if (sources.definitionVersion !== 2) { + throw new Error("the resumed run is not a source bundle"); + } + expect(sources.sources.map((one) => one.path)).toEqual([BUNDLE_ENTRYPOINT]); + const retainedBytes = sources.sources[0]?.bytes; + if (retainedBytes === undefined) { + throw new Error("the resumed run retains no entrypoint"); + } + expect(new TextDecoder().decode(retainedBytes)).toBe(BUNDLE_SOURCE); + }); + }); + } + + it("WLT1: halted inside source settlement, the committed run settles interrupted", function* () { + const root = yield* useStorageRoot(); + const runId = "committed-then-halted"; + const request: WorkflowBeginRequest = { + runId, + action: "start", + creation: yield* sourceBundleCreation(), + }; + + // The first time the connection lock is taken after the creation + // transaction committed is source settlement reading the retained bytes + // back to authenticate them. Holding that acquisition open is what puts + // this case inside the window by construction rather than by winning a race + // against it: the transition is provably still in `settleSources`, because + // it is stopped there. + const arrived = withResolvers(); + const held = withResolvers(); + let gated = false; + /** Whether the creation transaction has committed, asked of the file. */ + function committed(): boolean { + try { + return retained(root, runId).executions.length === 1; + } catch { + // No database, or no table in it yet: nothing has committed. + return false; + } + } + function gate(inner: RunConnection): RunConnection { + return { + ...inner, + lock: { + *hold(): Operation { + yield* inner.lock.hold(); + if (!gated && committed()) { + gated = true; + arrived.resolve(); + yield* held.operation; + } + }, + }, + }; + } + + yield* scoped(function* () { + const connections = yield* useWorkflowRunConnections(yield* SavepointObservation.get()); + const gated: WorkflowRunConnections = { + ...connections, + *at(path: string): Operation { + return gate(yield* connections.at(path)); + }, + }; + yield* installWorkflowRunStorage({ root }, {}, gated); + const transitions = yield* installWorkflowLifecycle( + { root, legacySource: legacySourceReader() }, + gated, + ); + + yield* scoped(function* () { + const acquisition = yield* WorkflowLifecycle.operations.acquireExecutor(runId); + if (!acquisition.ok) { + throw acquisition.error; + } + if (acquisition.value.kind !== "acquired") { + throw new Error(`${runId} already has a live workflow executor`); + } + const executorLock = acquisition.value.lock; + + let returned = false; + const begun = yield* spawn(function* () { + const result = yield* transitions.begin(executorLock, request); + returned = true; + return result; + }); + + // Settlement has the lock and is stopped in it. Everything below is + // therefore about a run whose creation transaction committed and whose + // transition has not returned. + yield* arrived.operation; + expect(returned).toBe(false); + expect(retained(root, runId).status).toBe("running"); + expect(retained(root, runId).executions).toHaveLength(1); + expect(retained(root, runId).executions[0]?.["stopped_at"]).toBe(null); + + // Halted where it stands, and the acquisition torn down under it — + // structured cancellation, inside the window. + yield* begun.halt(); + held.resolve(); + }); + }); + + yield* settledInterrupted(root, runId); + }); + + it("WLT2: halted after admission returned and before a caller registered anything", function* () { + const root = yield* useStorageRoot(); + const runId = "admitted-then-halted"; + const creation = yield* sourceBundleCreation(); + + yield* withRunHost(root, function* (transitions) { + yield* scoped(function* () { + const acquisition = yield* WorkflowLifecycle.operations.acquireExecutor(runId); + if (!acquisition.ok) { + throw acquisition.error; + } + if (acquisition.value.kind !== "acquired") { + throw new Error(`${runId} already has a live workflow executor`); + } + const begun = yield* transitions.begin(acquisition.value.lock, { + runId, + action: "start", + creation, + }); + if (!begun.ok) { + throw begun.error; + } + + // Exactly where the command stands when it reports the run id, + // projects the closure and imports the Deno adapter: admitted, running, + // and with nothing of its own registered yet. The scope ends here. + expect(retained(root, runId).status).toBe("running"); + expect(retained(root, runId).executions[0]?.["stopped_at"]).toBe(null); + }); + }); + + yield* settledInterrupted(root, runId); + }); + + it("WLT3: halted inside pre-commit authentication, nothing is recognized and the id is free", function* () { + const root = yield* useStorageRoot(); + const runId = "never-committed"; + + // The source-authentication window, held open from inside it. A version-1 + // creation is what makes this deterministic: its source lives in a + // repository this package does not reach, so `begin` asks the host's + // legacy reader for it — before the lifecycle transaction opens, and + // therefore before anything at all is written. That reader is a public + // seam, and a reader that does not answer is a `begin` provably stopped in + // the window this case is about. + const arrived = withResolvers(); + const held = withResolvers(); + const blockingReader: LegacyWorkflowSourceReader = function* () { + arrived.resolve(); + yield* held.operation; + throw new Error("the reader was expected to be halted, not resumed"); + }; + + yield* scoped(function* () { + const connections = yield* useWorkflowRunConnections(yield* SavepointObservation.get()); + yield* installWorkflowRunStorage({ root }, {}, connections); + const transitions = yield* installWorkflowLifecycle( + { root, legacySource: blockingReader }, + connections, + ); + + yield* scoped(function* () { + const acquisition = yield* WorkflowLifecycle.operations.acquireExecutor(runId); + if (!acquisition.ok) { + throw acquisition.error; + } + if (acquisition.value.kind !== "acquired") { + throw new Error(`${runId} already has a live workflow executor`); + } + const executorLock = acquisition.value.lock; + const request: WorkflowBeginRequest = { + runId, + action: "start", + creation: creation(), + }; + let returned = false; + const begun = yield* spawn(function* () { + const result = yield* transitions.begin(executorLock, request); + returned = true; + return result; + }); + + // Stopped in authentication, with the transaction not yet opened. + yield* arrived.operation; + expect(returned).toBe(false); + + // Halted where it stands, and the acquisition torn down under it. + yield* begun.halt(); + held.resolve(); + }); + }); + + // No run is recognized, and no document execution was retained. Inspection + // is the question a host asks, and the file — if a connection created one + // at all — is the question nothing can hide from. + yield* withLifecycle(root, function* () { + const inspected = yield* WorkflowLifecycle.operations.inspect(runId); + expect(inspected.ok).toBe(false); + }); + expect(() => retained(root, runId)).toThrow(); + + // The executor lock is released: acquiring it is the only proof that takes. + yield* withRunHost(root, function* () { + yield* withExecutor(runId, function* () { + expect(true).toBe(true); + }); + }); + + // And the id is free: the same one starts cleanly, and what it retains is + // its own creation rather than anything the halted attempt left. The status + // below belongs to that clean start, which this scope then ends — the + // checks above are the control, and they hold whether or not a committed + // run settles on teardown. + yield* withRunHost(root, function* (transitions) { + yield* withExecutor(runId, function* (executorLock) { + const started = yield* transitions.begin(executorLock, { + runId, + action: "start", + creation: yield* sourceBundleCreation(), + }); + if (!started.ok) { + throw started.error; + } + expect(started.value.record.runId).toBe(runId); + }); + }); + expect(retained(root, runId).status).toBe("interrupted"); + expect(new TextDecoder().decode(storedBytes(retained(root, runId).sources[0], "content"))).toBe( + BUNDLE_SOURCE, + ); + }); +}); diff --git a/packages/workflow/tests/workflow-lifecycle-inspection.test.ts b/packages/workflow/tests/workflow-lifecycle-inspection.test.ts index e73c20846..598cc2ce3 100644 --- a/packages/workflow/tests/workflow-lifecycle-inspection.test.ts +++ b/packages/workflow/tests/workflow-lifecycle-inspection.test.ts @@ -55,6 +55,7 @@ import { translateSqliteError, WorkflowReadonlyRollbackError } from "../src/deno import { EMPTY_WORKSPACE_ROOT_ID } from "../src/deno/workspace/manifest.ts"; import { creation, + runLeftUnfinished, withExecutorRun, runPath, tamper, @@ -131,17 +132,24 @@ function* retainedRun(root: string, runId: string): Operation { /** A run interrupted mid-execution: two events, no root Close, still `running`. */ function* partialRun(root: string, runId: string): Operation { - yield* withRunHost(root, function* (transitions) { - yield* withExecutorRun( - transitions, - { runId, action: "start", creation: creation() }, - function* (begun) { - yield* begun.database.journal.append( - sourced("import_component", `Release:${runId}`, { line: 4, column: 2 }), - ); - yield* begun.database.journal.append(unsourced("exec", `exec:echo ${runId}`)); - }, + // A workflow executor that was lost, not one that returned. A scope closing + // in this process runs the executor hold's teardown, and that settles the + // execution the acquisition began; only a killed process leaves a run + // `running` with an execution that has no end. + yield* runLeftUnfinished(root, runId); + + // The history it got through before it went. Appended through a lookup, which + // takes no executor lock and therefore settles nothing: what this leaves is + // still a run mid-flight. + yield* withRunHost(root, function* () { + const found = yield* WorkflowRunStorage.operations.lookup(runId); + if (!found.ok) { + throw found.error; + } + yield* found.value.journal.append( + sourced("import_component", `Release:${runId}`, { line: 4, column: 2 }), ); + yield* found.value.journal.append(unsourced("exec", `exec:echo ${runId}`)); }); } diff --git a/packages/workflow/tests/workflow-run-storage.test.ts b/packages/workflow/tests/workflow-run-storage.test.ts index e315e189a..811f2333c 100644 --- a/packages/workflow/tests/workflow-run-storage.test.ts +++ b/packages/workflow/tests/workflow-run-storage.test.ts @@ -61,6 +61,7 @@ import { SHA1, tamper, useStorageRoot, + runLeftUnfinished, withBegunRun, withStorage, } from "./support/storage.ts"; @@ -901,9 +902,10 @@ describe("Tier WS — surviving the process", () => { it("WS17: an unfinished document execution is still unfinished afterwards", function* () { const root = yield* useStorageRoot(); - const executionId = yield* withBegunRun(root, function* (run) { - return run.execution.executionId; - }); + // A workflow executor that was lost, not one that returned. A scope closing + // in this process runs the executor hold's teardown, and that settles the + // execution it began; only a killed process leaves one unfinished. + yield* runLeftUnfinished(root, "release-1.4"); const restored = yield* withStorage(root, function* () { const found = yield* lookup("release-1.4"); @@ -918,7 +920,6 @@ describe("Tier WS — surviving the process", () => { }); expect(restored).toHaveLength(1); - expect(restored[0].executionId).toBe(executionId); expect(restored[0].stoppedAt).toBeUndefined(); expect(restored[0].stopStatus).toBeUndefined(); });