diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1ba51ab..b00a149 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,21 +3,12 @@ env: NO_COLOR: "1" agents: - queue: "linux-small" + queue: "macos-medium" steps: - - label: ":git: Git 2.50.1" - key: "git-toolchain" - command: "bash .buildkite/scripts/build-modern-git.sh" - if: build.pull_request.id != null || build.env("TABELLIO_BUILD_CONTEXT") == "preflight" || build.branch == pipeline.default_branch - timeout_in_minutes: 15 - artifact_paths: - - ".artifacts/toolchain/git-2.50.1-linux-amd64.tar.gz" - - label: ":test_tube: Repository check" key: "repository-check" command: "bash .buildkite/scripts/tests.sh" - depends_on: "git-toolchain" if: build.pull_request.id != null || build.env("TABELLIO_BUILD_CONTEXT") == "preflight" || build.branch == pipeline.default_branch timeout_in_minutes: 10 artifact_paths: @@ -29,7 +20,6 @@ steps: - label: ":mag: Fallow changed-code" key: "fallow" command: "bash .buildkite/scripts/fallow.sh" - depends_on: "git-toolchain" if: build.pull_request.id != null || build.env("TABELLIO_BUILD_CONTEXT") == "preflight" || build.branch == pipeline.default_branch timeout_in_minutes: 10 artifact_paths: @@ -52,7 +42,6 @@ steps: - label: ":shield: Product validation" key: "product-validation" command: "bash .buildkite/scripts/product-validation.sh" - depends_on: "git-toolchain" if: build.pull_request.id != null || build.env("TABELLIO_BUILD_CONTEXT") == "preflight" || build.branch == pipeline.default_branch timeout_in_minutes: 50 artifact_paths: diff --git a/.buildkite/scripts/build-modern-git.sh b/.buildkite/scripts/build-modern-git.sh deleted file mode 100644 index 305896a..0000000 --- a/.buildkite/scripts/build-modern-git.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -version="2.50.1" -archive="git-${version}.tar.xz" -expected_sha256="7e3e6c36decbd8f1eedd14d42db6674be03671c2204864befa2a41756c5c8fc4" -workspace="$(mktemp -d)" -install_root="/tmp/intelip-tabellio-git-${version}" -artifact_dir=".artifacts/toolchain" -apt_lists="${workspace}/apt-lists" -apt_source_list="${workspace}/sources.list" -apt_source_parts="${workspace}/apt-sources.list.d" -apt_options=( - -o Dir::Etc::sourcelist="$apt_source_list" - -o Dir::Etc::sourceparts="$apt_source_parts" - -o Dir::State::lists="$apt_lists" - -o APT::Get::List-Cleanup="1" -) -packages=( - build-essential - ca-certificates - gettext - libcurl4-gnutls-dev - libexpat1-dev - libssl-dev - zlib1g-dev -) - -trap 'rm -rf "$workspace" "$install_root"' EXIT -rm -rf "$install_root" -mkdir -p "$install_root" - -missing_packages=() -for package in "${packages[@]}"; do - if ! dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -qx "install ok installed"; then - missing_packages+=("$package") - fi -done - -if ((${#missing_packages[@]} > 0)); then - mkdir -p "${apt_lists}/partial" "$apt_source_parts" - chmod 755 "$workspace" "$apt_lists" "${apt_lists}/partial" "$apt_source_parts" - if [[ -f /etc/apt/sources.list ]]; then - awk ' - /^[[:space:]]*deb(-src)?[[:space:]]/ && - /(ubuntu\.com\/(ubuntu|ubuntu-ports)|debian\.org\/debian(-security|-ports)?)/ { print } - ' /etc/apt/sources.list > "$apt_source_list" - else - : > "$apt_source_list" - fi - for source_file in \ - /etc/apt/sources.list.d/ubuntu.sources \ - /etc/apt/sources.list.d/debian.sources \ - /etc/apt/sources.list.d/ubuntu.list \ - /etc/apt/sources.list.d/debian.list; do - [[ -f "$source_file" ]] || continue - cp -- "$source_file" "$apt_source_parts/" - done - sudo apt-get "${apt_options[@]}" update - sudo apt-get "${apt_options[@]}" install -y --no-install-recommends "${missing_packages[@]}" -fi - -curl --fail --location --silent --show-error \ - "https://www.kernel.org/pub/software/scm/git/${archive}" \ - --output "${workspace}/${archive}" - -printf '%s %s\n' "$expected_sha256" "${workspace}/${archive}" | sha256sum --check -tar -C "$workspace" -xf "${workspace}/${archive}" - -make -C "${workspace}/git-${version}" -j2 prefix="$install_root" all -make -C "${workspace}/git-${version}" prefix="$install_root" install - -mkdir -p "$artifact_dir" -tar -C "$install_root" -czf "${artifact_dir}/git-${version}-linux-amd64.tar.gz" . diff --git a/.buildkite/scripts/use-modern-git.sh b/.buildkite/scripts/use-modern-git.sh index 9f7d626..0bbb539 100644 --- a/.buildkite/scripts/use-modern-git.sh +++ b/.buildkite/scripts/use-modern-git.sh @@ -1,17 +1,28 @@ #!/usr/bin/env bash set -euo pipefail -version="2.50.1" -artifact=".artifacts/toolchain/git-${version}-linux-amd64.tar.gz" -install_root="/tmp/intelip-tabellio-git-${version}" +required_version="2.50.1" +actual_version="$(git version | awk '{print $3}')" -buildkite-agent artifact download "$artifact" . -rm -rf "$install_root" -mkdir -p "$install_root" -tar -C "$install_root" -xzf "$artifact" +if ! awk -v actual="$actual_version" -v required="$required_version" ' + BEGIN { + split(actual, actual_parts, ".") + split(required, required_parts, ".") + for (part_index = 1; part_index <= 3; part_index++) { + actual_part = actual_parts[part_index] + 0 + required_part = required_parts[part_index] + 0 + if (actual_part > required_part) { + exit 0 + } + if (actual_part < required_part) { + exit 1 + } + } + exit 0 + } +'; then + printf 'Git %s or newer is required; found %s.\n' "$required_version" "$actual_version" >&2 + exit 1 +fi -export PATH="${install_root}/bin:${PATH}" -export GIT_EXEC_PATH="${install_root}/libexec/git-core" -export GIT_TEMPLATE_DIR="${install_root}/share/git-core/templates" -export GITPERLLIB="${install_root}/share/perl5" git --version diff --git a/.tabellio/validators.json b/.tabellio/validators.json index 96a3a7e..a11c6e0 100644 --- a/.tabellio/validators.json +++ b/.tabellio/validators.json @@ -348,6 +348,42 @@ "metrics": [{"name": "baseline_integration_security_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, "summary": "Packaged artifact completeness and private-evidence boundaries" + }, + "v060-identity-static": { + "commands": [["node", "--test", "tests/runner-identity.test.mjs", "tests/validation-runner.test.mjs"]], + "metrics": [{"name": "runner_identity_static_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "Runner identity implementation and validation-result compatibility" + }, + "v060-identity-schema": { + "commands": [["node", "--test", "--test-name-pattern", "runner identity schema|stable schema identifiers", "tests/runner-identity.test.mjs", "tests/context-and-evidence.test.mjs"]], + "metrics": [{"name": "runner_identity_schema_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "v0.4 validation result and runner identity schema" + }, + "v060-identity-semantic": { + "commands": [["node", "--test", "--test-name-pattern", "runner identity schema binds|typed validators enforce", "tests/runner-identity.test.mjs", "tests/validation-runner.test.mjs"]], + "metrics": [{"name": "runner_identity_semantic_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "Package version, source commit, cleanliness, tag, and typed evidence semantics" + }, + "v060-identity-workflow": { + "commands": [["node", "--test", "--test-name-pattern", "runner identity CLI workflow|typed validation distinguishes", "tests/runner-identity.test.mjs", "tests/validation-runner.test.mjs"]], + "metrics": [{"name": "runner_identity_workflow_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "Machine-readable version query and validation evidence workflow" + }, + "v060-identity-operational": { + "commands": [["node", "--test", "--test-name-pattern", "runner identity operational lookup", "tests/runner-identity.test.mjs"]], + "metrics": [{"name": "runner_identity_10x_duration_ms", "unit": "milliseconds", "pattern": "runner_identity_10x_duration_ms=([0-9.]+)"}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "Bounded local runner identity resolution" + }, + "v060-identity-security": { + "commands": [["node", "--test", "--test-name-pattern", "runner identity security reports|runner identity CLI workflow", "tests/runner-identity.test.mjs"]], + "metrics": [{"name": "runner_identity_security_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "No private path disclosure and fail-closed expectation checks" } } } diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ae8463..a2dd6a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,25 @@ All notable changes to Tabellio are recorded here. ## Unreleased +## 0.6.0 - release candidate + +### Added + +- Machine-readable `tabellio-version` identity reporting for package version, exact source commit, source cleanliness, and matching release tag. +- Validation-result v0.4 runner provenance and focused INTB-279 product-validation evidence. + +### Changed + +- Typed validation evidence now binds the Tabellio package and exact runner source while retaining v0.1 through v0.3 reader compatibility. + +### Release Gates + +- `tabellio-preflight --profile release` +- `npm run check` +- Fallow whole-repository and changed-code scans +- `npm pack --dry-run --json` +- Exact candidate and merged-head Tabellio validation + ## 0.5.0 - 2026-07-20 This is the first publication candidate after v0.2.0. Versions 0.3.0 and 0.4.0 were development milestones and were not tagged, released on GitHub, or published to npm. diff --git a/docs/getting-started.md b/docs/getting-started.md index ce0ac4e..8bf00ab 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -73,6 +73,16 @@ node scripts/tabellio-control-ref.mjs plan \ ## Preflight And Release +Inspect the local runner identity before trusting a version claim: + +```bash +npm run tabellio:version -- \ + --expect-version 0.6.0 \ + --expect-ref HEAD +``` + +Add `--require-clean` for an immutable candidate and `--require-release-tag` only after the approved, non-draft GitHub Release exists. The release check requires an annotated `origin` tag at the exact source commit; a local-only tag is insufficient. + Run preflight before agent work and again from clean merged `main`: ```bash @@ -104,8 +114,8 @@ node scripts/tabellio-release.mjs plan \ --owner example \ --remote-repo repository \ --number 42 \ - --version 0.5.0 \ - --notes docs/releases/v0.5.0.md \ + --version 0.6.0 \ + --notes docs/releases/v0.6.0.md \ --out /tmp/tabellio-release-intent.json ``` diff --git a/docs/intb-279-v060-execution-contract.md b/docs/intb-279-v060-execution-contract.md new file mode 100644 index 0000000..4c87d37 --- /dev/null +++ b/docs/intb-279-v060-execution-contract.md @@ -0,0 +1,35 @@ +# INTB-279: Tabellio v0.6.0 Identity Contract + +## Required outcomes + +- `package.json`, the changelog, and release notes identify version `0.6.0`. +- `tabellio-version` emits JSON containing the package name and version, exact Git commit, source cleanliness, and matching release tag when one exists. +- New validation results use `tabellio-validation-result/v0.4` and bind the same runner identity. +- Readers continue to accept validation-result versions v0.1 through v0.3. + +## Invariants + +- Inspection is local and read-only. +- Unavailable Git identity remains `null`; it is never inferred. +- Dirty source remains visible as `sourceDirty: true`. +- No filesystem path or source content is included in runner identity. +- Every required validator has available zero-cost telemetry. + +## Forbidden outcomes + +- Package, evidence, and release-note versions diverge. +- A missing or mismatched tag is represented as released. +- A moved commit reuses earlier validation evidence. +- Release publication, merge, deployment, billing, DNS, or consumer-repository mutation occurs without its separate approval. + +## Validation + +The exact candidate commit must pass: + +- `tabellio.v060-identity.validation.json` +- `tabellio.validation.json` +- repository checks and tests +- package dry-run inspection +- hosted CI and terminal review sync after push + +The release is not shipped until a GitHub Release for tag `v0.6.0` exists. It is not deployed without an exact runtime receipt. diff --git a/docs/releases/v0.6.0.md b/docs/releases/v0.6.0.md new file mode 100644 index 0000000..176a0cf --- /dev/null +++ b/docs/releases/v0.6.0.md @@ -0,0 +1,28 @@ +# Tabellio v0.6.0 + +Tabellio v0.6.0 makes the running control-plane version independently identifiable. + +## Added + +- `tabellio-version`, a machine-readable local identity command. +- Runner identity in validation-result v0.4: package name, package version, exact source commit, source cleanliness, and matching release tag. +- A focused product-validation manifest for the v0.6.0 identity contract. + +## Compatibility + +- Existing validation-result v0.1, v0.2, and v0.3 documents remain supported. +- Git commit pins remain the distribution mechanism for consumer repositories. + +## Verification + +```sh +tabellio-version \ + --expect-version 0.6.0 \ + --expect-ref HEAD \ + --require-clean \ + --require-release-tag +``` + +The release-tag requirement passes only when `v0.6.0` is an annotated tag on the +exact commit in the GitHub `origin` remote and a published, non-draft GitHub +Release exists for that tag. A local-only tag reports `tagged`, never `released`. diff --git a/docs/validation-runner.md b/docs/validation-runner.md index b042f22..9cc9de8 100644 --- a/docs/validation-runner.md +++ b/docs/validation-runner.md @@ -53,7 +53,7 @@ The runner: 7. Removes the worktree even after failure. 8. Writes an integrity-protected result to `refs/tabellio/validations` with compare-and-swap retries. -Command-manifest results use `tabellio-validation-result/v0.2`. Product-validation results use v0.3 and embed the acceptance digest, typed validator results, bounded evidence reports, total observed validation cost, and final policy decision. Both require `checkpointRevision` so checkpoint proof remains bound to the pull-request head when the validated revision is a later squash-merge commit. Runtime readers continue to accept legacy v0.1 and v0.2 results. +Command-manifest results use `tabellio-validation-result/v0.2`. Product-validation results use v0.4 and embed the acceptance digest, typed validator results, bounded evidence reports, total observed validation cost, final policy decision, and exact Tabellio runner identity when available. Both require `checkpointRevision` so checkpoint proof remains bound to the pull-request head when the validated revision is a later squash-merge commit. Runtime readers continue to accept v0.1 through v0.4 results. Read the newest result for a commit: diff --git a/package.json b/package.json index fb7e7db..3d8d1c2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@intelip/tabellio", - "version": "0.5.0", + "version": "0.6.0", "description": "Tabellio: GitHub-native context and evidence for agentic development.", "type": "module", "license": "Apache-2.0", @@ -16,6 +16,7 @@ "tabellio-stack-operation": "scripts/tabellio-stack-operation.mjs", "tabellio-review": "scripts/tabellio-review.mjs", "tabellio-validate": "scripts/tabellio-validate.mjs", + "tabellio-version": "scripts/tabellio-version.mjs", "tabellio-merge-ready": "scripts/tabellio-merge-ready.mjs", "tabellio-validator": "scripts/tabellio-validator.mjs", "tabellio-design-memory": "scripts/check-tabellio-design-memory.mjs", @@ -51,6 +52,7 @@ "tabellio.platform.json", "tabellio.validation.json", "tabellio.*.validation.json", + ".tabellio/validators.json", "schemas", "scripts", "examples", @@ -70,6 +72,7 @@ "tabellio:review": "node scripts/tabellio-review.mjs", "tabellio:review:example:check": "node scripts/check-tabellio-review-cycle.mjs --cycle examples/tabellio-review/minimal-cycle.json && node scripts/check-tabellio-agent-review.mjs --review examples/tabellio-review/minimal-agent-review.json", "tabellio:validate": "node scripts/tabellio-validate.mjs", + "tabellio:version": "node scripts/tabellio-version.mjs", "tabellio:merge-ready": "node scripts/tabellio-merge-ready.mjs", "tabellio:control-ref": "node scripts/tabellio-control-ref.mjs", "tabellio:preflight": "node scripts/tabellio-preflight.mjs", diff --git a/schemas/validation-result.v0.4.schema.json b/schemas/validation-result.v0.4.schema.json new file mode 100644 index 0000000..0f4170d --- /dev/null +++ b/schemas/validation-result.v0.4.schema.json @@ -0,0 +1,314 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validation-result:v0.4", + "title": "Tabellio Product Validation Result", + "type": "object", + "required": [ + "schemaVersion", + "runId", + "repository", + "revision", + "checkpointRevision", + "suite", + "runner", + "status", + "checkpoints", + "commands", + "acceptance", + "validators", + "decision", + "startedAt", + "completedAt", + "integrity" + ], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validation-result/v0.4" }, + "runId": { "type": "string", "minLength": 1 }, + "repository": { + "type": "object", + "required": ["id"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 } + } + }, + "revision": { + "$ref": "#/$defs/revision" + }, + "checkpointRevision": { + "$ref": "#/$defs/revision" + }, + "suite": { + "type": "object", + "required": ["id", "manifestPath", "manifestDigest"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "manifestPath": { "type": "string", "minLength": 1 }, + "manifestDigest": { + "$ref": "#/$defs/sha256" + } + } + }, + "runner": { + "type": "object", + "required": [ + "id", + "runtime", + "packageName", + "packageVersion", + "sourceCommit", + "sourceDirty", + "releaseTag" + ], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "runtime": { "type": "string", "minLength": 1 }, + "packageName": { "const": "@intelip/tabellio" }, + "packageVersion": { + "type": "string", + "pattern": "^(?:0|[1-9][0-9]*)\\.(?:0|[1-9][0-9]*)\\.(?:0|[1-9][0-9]*)(?:-(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(?:\\.(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$" + }, + "sourceCommit": { + "oneOf": [ + { "$ref": "#/$defs/oid" }, + { "type": "null" } + ] + }, + "sourceDirty": { "type": ["boolean", "null"] }, + "releaseTag": { + "oneOf": [ + { + "type": "string", + "pattern": "^v(?:0|[1-9][0-9]*)\\.(?:0|[1-9][0-9]*)\\.(?:0|[1-9][0-9]*)(?:-(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(?:\\.(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$" + }, + { "type": "null" } + ] + } + }, + "allOf": [ + { + "if": { + "properties": { "sourceCommit": { "type": "null" } }, + "required": ["sourceCommit"] + }, + "then": { + "properties": { + "sourceDirty": { "type": "null" }, + "releaseTag": { "type": "null" } + } + }, + "else": { + "properties": { + "sourceDirty": { "type": "boolean" } + } + } + } + ] + }, + "status": { "enum": ["passed", "failed", "blocked"] }, + "checkpoints": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string", "minLength": 1 } + }, + "commands": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/commandResult" + } + }, + "acceptance": { + "$ref": "#/$defs/acceptanceResult" + }, + "validators": { + "type": "array", + "minItems": 1, + "maxItems": 50, + "items": { + "$ref": "#/$defs/validatorResult" + } + }, + "decision": { + "$ref": "#/$defs/decision" + }, + "startedAt": { "type": "string", "format": "date-time" }, + "completedAt": { "type": "string", "format": "date-time" }, + "integrity": { + "type": "object", + "required": ["algorithm", "digest"], + "additionalProperties": false, + "properties": { + "algorithm": { "const": "sha256" }, + "digest": { + "$ref": "#/$defs/sha256" + } + } + } + }, + "$defs": { + "oid": { "type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$" }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "validatorType": { "enum": ["static", "schema", "semantic", "workflow", "visual", "operational", "security"] }, + "revision": { + "type": "object", + "required": ["baseCommit", "mergeBase", "headCommit"], + "additionalProperties": false, + "properties": { + "baseCommit": { "$ref": "#/$defs/oid" }, + "mergeBase": { "$ref": "#/$defs/oid" }, + "headCommit": { "$ref": "#/$defs/oid" } + } + }, + "output": { + "type": "object", + "required": ["bytes", "digest", "tail", "truncated"], + "additionalProperties": false, + "properties": { + "bytes": { "type": "integer", "minimum": 0 }, + "digest": { "$ref": "#/$defs/sha256" }, + "tail": { "type": "string" }, + "truncated": { "type": "boolean" } + } + }, + "commandResult": { + "type": "object", + "required": ["id", "argv", "cwd", "required", "status", "exitCode", "signal", "durationMs", "stdout", "stderr", "startedAt", "completedAt", "error"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "argv": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, + "cwd": { "type": "string", "minLength": 1 }, + "required": { "type": "boolean" }, + "status": { "enum": ["passed", "failed", "error", "timed_out", "skipped"] }, + "exitCode": { "type": ["integer", "null"] }, + "signal": { "type": ["string", "null"] }, + "durationMs": { "type": "integer", "minimum": 0 }, + "stdout": { "$ref": "#/$defs/output" }, + "stderr": { "$ref": "#/$defs/output" }, + "startedAt": { "oneOf": [{ "type": "string", "format": "date-time" }, { "type": "null" }] }, + "completedAt": { "oneOf": [{ "type": "string", "format": "date-time" }, { "type": "null" }] }, + "error": { "type": ["string", "null"] } + } + }, + "acceptanceResult": { + "type": "object", + "required": ["id", "source", "risk", "digest", "requiredValidatorTypes"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "source": { "type": "string", "minLength": 1 }, + "risk": { "enum": ["low", "medium", "high", "critical"] }, + "digest": { "$ref": "#/$defs/sha256" }, + "requiredValidatorTypes": { "type": "array", "minItems": 1, "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/validatorType" } } + } + }, + "validatorResult": { + "type": "object", + "required": ["id", "type", "required", "status", "evidence", "reasons"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "type": { "$ref": "#/$defs/validatorType" }, + "required": { "type": "boolean" }, + "status": { "enum": ["passed", "failed", "blocked", "skipped"] }, + "evidence": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "required": ["path", "digest", "report"], + "additionalProperties": false, + "properties": { + "path": { "type": "string", "minLength": 1 }, + "digest": { "$ref": "#/$defs/sha256" }, + "report": { "$ref": "#/$defs/evidenceReport" } + } + } + ] + }, + "reasons": { "type": "array", "maxItems": 128, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 2000 } } + } + }, + "evidenceReport": { + "type": "object", + "required": ["schemaVersion", "validatorId", "status", "summary", "metrics", "cost", "artifacts"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validator-evidence/v0.1" }, + "validatorId": { "type": "string", "minLength": 1 }, + "status": { "enum": ["passed", "failed", "blocked"] }, + "summary": { "type": "string", "minLength": 1, "maxLength": 2000 }, + "metrics": { "type": "array", "maxItems": 100, "items": { "$ref": "#/$defs/metric" } }, + "cost": { "$ref": "#/$defs/cost" }, + "artifacts": { "type": "array", "maxItems": 50, "items": { "$ref": "#/$defs/artifact" } } + } + }, + "metric": { + "type": "object", + "required": ["name", "value", "unit"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "value": { "type": "number" }, + "unit": { "type": "string", "minLength": 1 } + } + }, + "cost": { + "type": "object", + "required": ["telemetry", "usd", "modelCalls", "toolCalls"], + "additionalProperties": false, + "properties": { + "telemetry": { "enum": ["available", "unavailable", "not_applicable"] }, + "usd": { "type": ["number", "null"], "minimum": 0 }, + "modelCalls": { "type": ["integer", "null"], "minimum": 0 }, + "toolCalls": { "type": ["integer", "null"], "minimum": 0 } + }, + "allOf": [ + { + "if": { "properties": { "telemetry": { "const": "available" } }, "required": ["telemetry"] }, + "then": { + "properties": { + "usd": { "type": "number" }, + "modelCalls": { "type": "integer" }, + "toolCalls": { "type": "integer" } + } + }, + "else": { + "properties": { + "usd": { "type": "null" }, + "modelCalls": { "type": "null" }, + "toolCalls": { "type": "null" } + } + } + } + ] + }, + "artifact": { + "type": "object", + "required": ["name", "uri", "digest", "mediaType", "bytes"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "uri": { "type": "string", "format": "uri", "not": { "pattern": "^file:" } }, + "digest": { "$ref": "#/$defs/sha256" }, + "mediaType": { "type": "string", "minLength": 1 }, + "bytes": { "type": "integer", "minimum": 0 } + } + }, + "decision": { + "type": "object", + "required": ["status", "reasons", "totalCostUsd", "costTelemetryComplete"], + "additionalProperties": false, + "properties": { + "status": { "enum": ["passed", "failed", "blocked"] }, + "reasons": { "type": "array", "maxItems": 6400, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 2000 } }, + "totalCostUsd": { "type": "number", "minimum": 0 }, + "costTelemetryComplete": { "type": "boolean" } + } + } + } +} diff --git a/scripts/lib/runner-identity.mjs b/scripts/lib/runner-identity.mjs new file mode 100644 index 0000000..6b3619f --- /dev/null +++ b/scripts/lib/runner-identity.mjs @@ -0,0 +1,185 @@ +import { createHash } from "node:crypto"; +import { lstat, mkdtemp, readFile, readlink, realpath, rm, writeFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; + +import { runGit } from "./git-process.mjs"; + +const DEFAULT_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); + +export async function tabellioRunnerIdentity({ root = DEFAULT_ROOT } = {}) { + return (await tabellioRunnerState({ root })).identity; +} + +export async function tabellioRunnerState({ root = DEFAULT_ROOT } = {}) { + const packageMetadata = JSON.parse(await readFile(resolve(root, "package.json"), "utf8")); + const packageName = requiredString(packageMetadata.name, "package name"); + const packageVersion = requiredString(packageMetadata.version, "package version"); + const source = await gitSourceIdentity(root, packageVersion); + const identity = { + packageName, + packageVersion, + sourceCommit: source.commit, + sourceDirty: source.dirty, + releaseTag: source.releaseTag, + }; + return { identity, fingerprint: source.fingerprint ?? identityFingerprint(identity) }; +} + +async function gitSourceIdentity(root, packageVersion) { + try { + return await readGitSourceIdentity(root, packageVersion); + } catch (error) { + if (isNotGitRepository(error)) return { commit: null, dirty: null, releaseTag: null, fingerprint: null }; + throw error; + } +} + +async function readGitSourceIdentity(root, packageVersion) { + const worktree = await readGit(root, ["rev-parse", "--show-toplevel"]); + if (await realpath(worktree.stdout.trim()) !== await realpath(root)) { + return { commit: null, dirty: null, releaseTag: null }; + } + const revision = await readGit(root, ["rev-parse", "HEAD"]); + const commit = revision.stdout.trim(); + assertGitObjectId(commit); + const status = await readGit(root, ["status", "--porcelain=v1", "-z", "--untracked-files=all"]); + const indexFlags = await readIndexFlags(root); + const flaggedPaths = unsafeIndexPaths(indexFlags.stdout); + const tags = await readGit(root, ["tag", "--points-at", commit, "--list", `v${packageVersion}`]); + return { + commit, + dirty: status.stdout.length > 0 || flaggedPaths.length > 0, + releaseTag: matchingReleaseTag(tags.stdout), + fingerprint: await worktreeFingerprint(root, commit, status.stdout, flaggedPaths), + }; +} + +async function worktreeFingerprint(root, commit, status, flaggedPaths = []) { + const changed = await readGit(root, ["diff", "--name-only", "--no-renames", "-z", "HEAD", "--"]); + return fingerprintPaths(root, commit, status, changed.stdout, flaggedPaths); +} + +async function fingerprintPaths(root, commit, status, changed, flaggedPaths = []) { + const untracked = await readGit(root, ["ls-files", "--others", "--exclude-standard", "-z"]); + const hash = createHash("sha256"); + hash.update(commit).update("\0").update(status).update("\0"); + const paths = new Set([ + ...changed.split("\0"), + ...untracked.stdout.split("\0"), + ...flaggedPaths, + ].filter(Boolean)); + for (const path of [...paths].sort()) { + hash.update(path).update("\0").update(await entryFingerprint(root, path)).update("\0"); + } + return hash.digest("hex"); +} + +async function entryFingerprint(root, path) { + const absolute = resolve(root, path); + const metadata = await lstat(absolute).catch((error) => { + if (error?.code === "ENOENT") return null; + throw error; + }); + if (metadata === null) return "missing"; + if (metadata.isFile()) return fileFingerprint(root, path); + return nonFileFingerprint(absolute, metadata); +} + +async function fileFingerprint(root, path) { + const result = await readGit(root, ["hash-object", "--no-filters", "--", path]).catch((error) => { + if (isMissingFile(error)) return null; + throw error; + }); + if (result === null) return "missing"; + const object = result.stdout.trim(); + assertGitObjectId(object); + return `file:${object}`; +} + +async function nonFileFingerprint(path, metadata) { + if (metadata.isSymbolicLink()) return `symlink:${await readlink(path)}`; + if (metadata.isDirectory()) return directoryFingerprint(path); + return `special:${metadata.mode}`; +} + +async function directoryFingerprint(path) { + const repository = await readGit(path, ["rev-parse", "--is-inside-work-tree"]).catch((error) => { + if (isNotGitRepository(error)) return null; + throw error; + }); + if (repository?.stdout.trim() !== "true") return "directory"; + const revision = await readGit(path, ["rev-parse", "HEAD"]).catch((error) => { + if (isUnknownRevision(error)) return null; + throw error; + }); + const status = await readGit(path, ["status", "--porcelain=v1", "-z", "--untracked-files=all"]); + const fingerprint = revision === null + ? await fingerprintPaths(path, "unborn", status.stdout, "") + : await worktreeFingerprint(path, revision.stdout.trim(), status.stdout); + return `repository:${fingerprint}`; +} + +function readGit(root, args) { + return runGit({ args, cwd: root, env: { GIT_OPTIONAL_LOCKS: "0" } }); +} + +async function readIndexFlags(root) { + const indexPath = (await readGit(root, ["rev-parse", "--git-path", "index"])).stdout.trim(); + const directory = await mkdtemp(join(tmpdir(), "TabellioIndex-")); + const copiedIndex = join(directory, "index"); + try { + await writeFile(copiedIndex, await readFile(resolve(root, indexPath)), { mode: 0o600 }); + return await runGit({ + args: ["ls-files", "-v", "-z"], + cwd: root, + env: { + GIT_INDEX_FILE: copiedIndex, + GIT_OPTIONAL_LOCKS: "0", + }, + }); + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +function isMissingFile(error) { + return error instanceof Error && /could not open .*: No such file or directory/.test(error.stderr ?? ""); +} + +function isUnknownRevision(error) { + return error instanceof Error && /unknown revision|ambiguous argument 'HEAD'|Needed a single revision/.test(error.stderr ?? ""); +} + +function identityFingerprint(identity) { + return createHash("sha256").update(JSON.stringify(identity)).digest("hex"); +} + +function assertGitObjectId(value) { + if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(value)) { + throw new Error("Tabellio source commit is not a Git object ID."); + } +} + +function matchingReleaseTag(stdout) { + const tags = stdout.split("\n").map((value) => value.trim()).filter(Boolean); + return tags.length === 1 ? tags[0] : null; +} + +function unsafeIndexPaths(stdout) { + return stdout.split("\0") + .filter((entry) => entry.length > 2 && entry[0] !== "H" && entry[1] === " ") + .map((entry) => entry.slice(2)); +} + +function isNotGitRepository(error) { + if (!(error instanceof Error)) return false; + const diagnostic = typeof error.stderr === "string" ? error.stderr : error.message; + return /not a git repository|must be run in a work tree|unknown revision/i.test(diagnostic); +} + +function requiredString(value, label) { + if (typeof value !== "string" || value.length === 0) throw new Error(`${label} must be a non-empty string.`); + return value; +} diff --git a/scripts/lib/runner-release.mjs b/scripts/lib/runner-release.mjs new file mode 100644 index 0000000..cb646fd --- /dev/null +++ b/scripts/lib/runner-release.mjs @@ -0,0 +1,58 @@ +import { runExternalCommand } from "./external-command.mjs"; +import { effectiveGitHubRepository } from "./github-repository.mjs"; +import { runGit } from "./git-process.mjs"; + +export async function verifyPublishedRunnerRelease({ + root, + identity, + remote = "origin", + ghBinary = "gh", + commandRunner = runExternalCommand, + repositoryReader = effectiveGitHubRepository, + remoteTagReader = readPublishedTag, +} = {}) { + const tag = `v${identity.packageVersion}`; + if (!hasLocalTag(identity, tag)) return false; + const repository = await repositoryReader({ repoPath: root }, remote); + const remoteTag = await remoteTagReader({ root, remote, tag }); + if (!isExactAnnotatedTag(remoteTag, identity.sourceCommit)) return false; + const release = await commandRunner({ + binary: ghBinary, + args: ["release", "view", tag, "--repo", repository.fullName, "--json", "tagName,isDraft,isPrerelease"], + cwd: root, + timeoutMs: 30_000, + }); + return isPublishedRelease(JSON.parse(release.stdout), tag); +} + +function hasLocalTag(identity, tag) { + return identity.sourceCommit !== null && identity.releaseTag === tag; +} + +function isExactAnnotatedTag(remoteTag, commit) { + return remoteTag?.annotated === true && remoteTag.commit === commit; +} + +function isPublishedRelease(release, tag) { + return release.tagName === tag && release.isDraft === false && release.isPrerelease === false; +} + +async function readPublishedTag({ root, remote, tag }) { + const directRef = `refs/tags/${tag}`; + const peeledRef = `${directRef}^{}`; + const result = await runGit({ + args: ["ls-remote", "--tags", remote, directRef, peeledRef], + cwd: root, + timeoutMs: 30_000, + }); + const refs = new Map(result.stdout.trim().split(/\r?\n/) + .filter(Boolean) + .map((row) => { + const [oid, ref] = row.split(/\s+/); + return [ref, oid]; + })); + const direct = refs.get(directRef); + const commit = refs.get(peeledRef); + if (!direct || !commit) return null; + return { annotated: direct !== commit, commit }; +} diff --git a/scripts/lib/validation-runner.mjs b/scripts/lib/validation-runner.mjs index ebaac56..409002c 100644 --- a/scripts/lib/validation-runner.mjs +++ b/scripts/lib/validation-runner.mjs @@ -6,6 +6,7 @@ import { basename, dirname, isAbsolute, join, relative, resolve } from "node:pat import { LedgerConflictError } from "./git-json-ledger.mjs"; import { runGit } from "./git-process.mjs"; +import { tabellioRunnerState } from "./runner-identity.mjs"; import { digestObject } from "./stack-operation.mjs"; const VALIDATION_MANIFEST_SCHEMA_VERSION_V1 = "tabellio-validation/v0.1"; @@ -13,16 +14,19 @@ const VALIDATION_MANIFEST_SCHEMA_VERSION_V2 = "tabellio-validation/v0.2"; const VALIDATION_RESULT_SCHEMA_VERSION_V1 = "tabellio-validation-result/v0.1"; const VALIDATION_RESULT_SCHEMA_VERSION_V2 = "tabellio-validation-result/v0.2"; const VALIDATION_RESULT_SCHEMA_VERSION_V3 = "tabellio-validation-result/v0.3"; +const VALIDATION_RESULT_SCHEMA_VERSION_V4 = "tabellio-validation-result/v0.4"; const VALIDATOR_EVIDENCE_SCHEMA_VERSION = "tabellio-validator-evidence/v0.1"; const VALIDATOR_TYPES = ["static", "schema", "semantic", "workflow", "visual", "operational", "security"]; +const SEMANTIC_VERSION = /^(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)(?:-(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/; const MAX_OUTPUT_TAIL_BYTES = 16 * 1024; const MAX_EVIDENCE_BYTES = 1024 * 1024; export class ValidationRunner { - constructor({ store, ledger, workspaceRoot = null }) { + constructor({ store, ledger, workspaceRoot = null, runnerIdentity = null, runnerState = null }) { this.store = store; this.ledger = ledger; this.workspaceRoot = workspaceRoot; + this.runnerState = resolveRunnerState(runnerIdentity, runnerState); } async run({ @@ -53,6 +57,8 @@ export class ValidationRunner { if (manifest.requireEntireCheckpoint && checkpoints.length === 0) { throw new Error(`Checkpoint range ${checkpointRevision.mergeBase}..${checkpointRevision.headCommit} has no Entire checkpoint.`); } + const initialRunnerState = await this.runnerState(); + const runnerIdentity = initialRunnerState.identity; const runId = `validation-${randomUUID()}`; const common = await runGit({ args: ["rev-parse", "--git-common-dir"], cwd: this.store.repoPath }); @@ -90,6 +96,13 @@ export class ValidationRunner { } } const completedAt = new Date().toISOString(); + const completedRunnerState = await this.runnerState(); + if ( + completedRunnerState.fingerprint !== initialRunnerState.fingerprint + || JSON.stringify(completedRunnerState.identity) !== JSON.stringify(runnerIdentity) + ) { + throw new Error("Tabellio runner identity changed during validation."); + } const result = buildValidationResult({ manifest, definitions, @@ -100,6 +113,7 @@ export class ValidationRunner { checkpointRevision, manifestPath, runnerId, + runnerIdentity, checkpoints, startedAt, completedAt, @@ -112,6 +126,29 @@ export class ValidationRunner { } } +function resolveRunnerState(runnerIdentity, runnerState) { + optionalFunction(runnerIdentity, "runnerIdentity"); + optionalFunction(runnerState, "runnerState"); + rejectCompetingRunnerSources(runnerIdentity, runnerState); + return runnerState ?? runnerStateFromIdentity(runnerIdentity); +} + +function rejectCompetingRunnerSources(runnerIdentity, runnerState) { + if (runnerIdentity !== null && runnerState !== null) throw new TypeError("Supply runnerIdentity or runnerState, not both."); +} + +function runnerStateFromIdentity(runnerIdentity) { + if (runnerIdentity === null) return tabellioRunnerState; + return async () => { + const identity = await runnerIdentity(); + return { identity, fingerprint: digestObject(identity) }; + }; +} + +function optionalFunction(value, label) { + if (value !== null && typeof value !== "function") throw new TypeError(`${label} must be a function.`); +} + async function registeredWorktree(repoPath, workspace) { const listing = await runGit({ args: ["worktree", "list", "--porcelain", "-z"], cwd: repoPath }); return listing.stdout.split("\0").includes(`worktree ${workspace}`); @@ -229,6 +266,7 @@ function buildValidationResult({ checkpointRevision, manifestPath, runnerId, + runnerIdentity, checkpoints, startedAt, completedAt, @@ -237,7 +275,7 @@ function buildValidationResult({ const decision = typed ? validationDecision(execution.validators) : null; const requiredFailed = execution.commands.some((command, index) => definitions[index].required && command.status !== "passed"); const result = { - schemaVersion: typed ? VALIDATION_RESULT_SCHEMA_VERSION_V3 : VALIDATION_RESULT_SCHEMA_VERSION_V2, + schemaVersion: typed ? VALIDATION_RESULT_SCHEMA_VERSION_V4 : VALIDATION_RESULT_SCHEMA_VERSION_V2, runId, repository: { id: repositoryId }, revision, @@ -247,7 +285,11 @@ function buildValidationResult({ manifestPath, manifestDigest: digestObject(manifest), }, - runner: { id: runnerId, runtime: `node-${process.version}` }, + runner: typed ? { + id: runnerId, + runtime: `node-${process.version}`, + ...runnerIdentity, + } : { id: runnerId, runtime: `node-${process.version}` }, status: decision?.status ?? (requiredFailed ? "failed" : "passed"), checkpoints, commands: execution.commands, @@ -359,7 +401,12 @@ export function validateValidationResult(value) { object(value, "validation result"); member( value.schemaVersion, - [VALIDATION_RESULT_SCHEMA_VERSION_V1, VALIDATION_RESULT_SCHEMA_VERSION_V2, VALIDATION_RESULT_SCHEMA_VERSION_V3], + [ + VALIDATION_RESULT_SCHEMA_VERSION_V1, + VALIDATION_RESULT_SCHEMA_VERSION_V2, + VALIDATION_RESULT_SCHEMA_VERSION_V3, + VALIDATION_RESULT_SCHEMA_VERSION_V4, + ], "validation result.schemaVersion", ); exactKeys(value, validationResultKeys(value.schemaVersion), "validation result"); @@ -374,13 +421,10 @@ export function validateValidationResult(value) { requiredString(value.suite.id, "validation result.suite.id"); validateRelativePath(value.suite.manifestPath, "validation result.suite.manifestPath"); sha256(value.suite.manifestDigest, "validation result.suite.manifestDigest"); - object(value.runner, "validation result.runner"); - exactKeys(value.runner, ["id", "runtime"], "validation result.runner"); - requiredString(value.runner.id, "validation result.runner.id"); - requiredString(value.runner.runtime, "validation result.runner.runtime"); + validateResultRunner(value.runner, value.schemaVersion); member( value.status, - value.schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V3 ? ["passed", "failed", "blocked"] : ["passed", "failed"], + isTypedValidationResult(value.schemaVersion) ? ["passed", "failed", "blocked"] : ["passed", "failed"], "validation result.status", ); stringArray(value.checkpoints, "validation result.checkpoints"); @@ -393,19 +437,8 @@ export function validateValidationResult(value) { equals(value.integrity.algorithm, "sha256", "validation result.integrity.algorithm"); sha256(value.integrity.digest, "validation result.integrity.digest"); if (validationResultDigest(value) !== value.integrity.digest) throw new Error("validation result integrity digest does not match."); - if (value.schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V3) { - validateAcceptanceResult(value.acceptance); - if (!Array.isArray(value.validators) || value.validators.length === 0) { - throw new Error("validation result.validators must be a non-empty array."); - } - value.validators.forEach((validator, index) => validateValidatorResult(validator, `validation result.validators[${index}]`)); - const commandIds = value.commands.map((command) => command.id); - const validatorIds = value.validators.map((validator) => validator.id); - if (JSON.stringify(commandIds) !== JSON.stringify(validatorIds)) { - throw new Error("validation result validators must align with command ids and order."); - } - validateValidationDecision(value.decision, value.validators); - if (value.status !== value.decision.status) throw new Error("validation result status does not match decision status."); + if (isTypedValidationResult(value.schemaVersion)) { + validateTypedValidationResult(value); return value; } const expectedStatus = value.commands.some((command) => command.required && command.status !== "passed") ? "failed" : "passed"; @@ -415,18 +448,85 @@ export function validateValidationResult(value) { function validationResultKeys(schemaVersion) { const keys = ["schemaVersion", "runId", "repository", "revision", "suite", "runner", "status", "checkpoints", "commands", "startedAt", "completedAt", "integrity"]; - if (schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V3) { + if (isTypedValidationResult(schemaVersion)) { return [...keys, "checkpointRevision", "acceptance", "validators", "decision"]; } return schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V2 ? [...keys, "checkpointRevision"] : keys; } function validateCheckpointRevision(value) { - if ([VALIDATION_RESULT_SCHEMA_VERSION_V2, VALIDATION_RESULT_SCHEMA_VERSION_V3].includes(value.schemaVersion)) { + if ([ + VALIDATION_RESULT_SCHEMA_VERSION_V2, + VALIDATION_RESULT_SCHEMA_VERSION_V3, + VALIDATION_RESULT_SCHEMA_VERSION_V4, + ].includes(value.schemaVersion)) { validateRevision(value.checkpointRevision, "validation result.checkpointRevision"); } } +function isTypedValidationResult(schemaVersion) { + return [VALIDATION_RESULT_SCHEMA_VERSION_V3, VALIDATION_RESULT_SCHEMA_VERSION_V4].includes(schemaVersion); +} + +function validateResultRunner(value, schemaVersion) { + object(value, "validation result.runner"); + const runnerKeys = schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V4 + ? ["id", "runtime", "packageName", "packageVersion", "sourceCommit", "sourceDirty", "releaseTag"] + : ["id", "runtime"]; + exactKeys(value, runnerKeys, "validation result.runner"); + requiredString(value.id, "validation result.runner.id"); + requiredString(value.runtime, "validation result.runner.runtime"); + if (schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V4) validateRunnerIdentity(value); +} + +function validateRunnerIdentity(value) { + equals(value.packageName, "@intelip/tabellio", "validation result.runner.packageName"); + if (typeof value.packageVersion !== "string" || !SEMANTIC_VERSION.test(value.packageVersion)) { + throw new Error("validation result.runner.packageVersion must be a semantic version."); + } + validateRunnerSource(value); + validateRunnerReleaseTag(value); +} + +function validateRunnerSource(value) { + nullable(value.sourceCommit, (sourceCommit) => oid(sourceCommit, "validation result.runner.sourceCommit")); + nullable(value.sourceDirty, (sourceDirty) => boolean(sourceDirty, "validation result.runner.sourceDirty")); + if ((value.sourceCommit === null) !== (value.sourceDirty === null)) { + throw new Error("validation result.runner sourceCommit and sourceDirty must be available together."); + } +} + +function validateRunnerReleaseTag(value) { + if (value.releaseTag === null) return; + equals(value.releaseTag, `v${value.packageVersion}`, "validation result.runner.releaseTag"); + if (value.sourceCommit === null) { + throw new Error("validation result.runner releaseTag requires a source commit."); + } +} + +function validateTypedValidationResult(value) { + validateAcceptanceResult(value.acceptance); + if (!Array.isArray(value.validators) || value.validators.length === 0) { + throw new Error("validation result.validators must be a non-empty array."); + } + value.validators.forEach((validator, index) => validateValidatorResult(validator, `validation result.validators[${index}]`)); + validateValidatorAlignment(value.commands, value.validators); + validateValidationDecision(value.decision, value.validators); + equals(value.status, value.decision.status, "validation result status"); +} + +function validateValidatorAlignment(commands, validators) { + const commandIds = commands.map((command) => command.id); + const validatorIds = validators.map((validator) => validator.id); + if (JSON.stringify(commandIds) !== JSON.stringify(validatorIds)) { + throw new Error("validation result validators must align with command ids and order."); + } +} + +function nullable(value, validate) { + if (value !== null) validate(value); +} + function validateRevision(value, path) { object(value, path); exactKeys(value, ["baseCommit", "mergeBase", "headCommit"], path); diff --git a/scripts/tabellio-version.mjs b/scripts/tabellio-version.mjs new file mode 100644 index 0000000..5e7c26c --- /dev/null +++ b/scripts/tabellio-version.mjs @@ -0,0 +1,76 @@ +#!/usr/bin/env node + +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { runGit } from "./lib/git-process.mjs"; +import { assertAllowedOptions, parseOptionPairs } from "./lib/cli-options.mjs"; +import { tabellioRunnerIdentity } from "./lib/runner-identity.mjs"; +import { verifyPublishedRunnerRelease } from "./lib/runner-release.mjs"; + +const BOOLEAN_FLAGS = new Set(["--require-clean", "--require-release-tag"]); +const ALLOWED_OPTIONS = ["expectVersion", "expectRef", "requireClean", "requireReleaseTag"]; +const RUNNER_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +try { + const options = parseArgs(process.argv.slice(2)); + const identity = await tabellioRunnerIdentity({ root: RUNNER_ROOT }); + const expectedCommit = options.expectRef + ? verifiedCommit((await runGit({ + args: ["rev-parse", "--verify", "--end-of-options", `${options.expectRef}^{commit}`], + cwd: RUNNER_ROOT, + })).stdout.trim()) + : null; + const blockers = []; + if (options.expectVersion && identity.packageVersion !== options.expectVersion) { + blockers.push(`package_version_mismatch:${options.expectVersion}`); + } + if (expectedCommit && identity.sourceCommit !== expectedCommit) { + blockers.push(`source_commit_mismatch:${expectedCommit}`); + } + if (options.requireClean && identity.sourceDirty !== false) blockers.push("runner_source_not_clean"); + const publishedRelease = options.requireReleaseTag + ? await verifyPublishedRunnerRelease({ root: RUNNER_ROOT, identity }) + : false; + if (options.requireReleaseTag && !publishedRelease) { + blockers.push(`published_release_missing:v${identity.packageVersion}`); + } + const status = identity.sourceCommit === null + ? "source_unavailable" + : identity.sourceDirty + ? "dirty" + : publishedRelease + ? "released" + : identity.releaseTag + ? "tagged" + : "identified"; + const result = { ok: blockers.length === 0, status, runner: identity, blockers }; + console.log(JSON.stringify(result, null, 2)); + if (!result.ok) process.exitCode = 1; +} catch (error) { + console.error(JSON.stringify({ + ok: false, + status: "blocked", + error: error instanceof Error ? error.message : String(error), + }, null, 2)); + process.exitCode = 1; +} + +function verifiedCommit(value) { + if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(value)) { + throw new Error("Expected ref did not resolve to a Git commit."); + } + return value; +} + +function parseArgs(args) { + const pairedArgs = args.flatMap((argument) => BOOLEAN_FLAGS.has(argument) ? [argument, "true"] : [argument]); + const values = parseOptionPairs(pairedArgs, "tabellio-version"); + assertAllowedOptions(values, ALLOWED_OPTIONS); + return { + expectVersion: values.expectVersion ?? null, + expectRef: values.expectRef ?? null, + requireClean: values.requireClean === "true", + requireReleaseTag: values.requireReleaseTag === "true", + }; +} diff --git a/tabellio.v060-identity.validation.json b/tabellio.v060-identity.validation.json new file mode 100644 index 0000000..ca9a822 --- /dev/null +++ b/tabellio.v060-identity.validation.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": "tabellio-validation/v0.2", + "id": "tabellio-v060-runner-identity", + "failFast": false, + "requireEntireCheckpoint": true, + "acceptance": { + "id": "intb-279-v060-runner-identity", + "source": "INTB-279 and docs/intb-279-v060-execution-contract.md", + "risk": "high", + "outcomes": [ + "Package metadata, release notes, and validation evidence identify Tabellio 0.6.0.", + "A machine-readable command reports package version, exact source commit, source cleanliness, and a matching release tag when present.", + "Typed validation evidence identifies the runner package version and exact source commit.", + "Existing v0.1, v0.2, and v0.3 validation-result readers remain supported." + ], + "invariants": [ + "Version inspection performs no external writes.", + "Unknown source identity remains unavailable rather than inferred.", + "A dirty source tree remains visibly dirty.", + "Cost telemetry is available and zero for every required validator." + ], + "forbiddenOutcomes": [ + "Package version and evidence version diverge.", + "A missing or mismatched release tag is reported as a release.", + "Private filesystem paths are copied into runner identity evidence.", + "A prior validation result format becomes unreadable." + ], + "requiredValidatorTypes": ["static", "schema", "semantic", "workflow", "operational", "security"] + }, + "validators": [ + {"id": "v060-identity-static", "type": "static", "argv": ["node", "scripts/tabellio-validator.mjs", "--profile", "v060-identity-static", "--validator-id", "v060-identity-static", "--out", ".artifacts/tabellio/v060-identity-static.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/v060-identity-static.json"}, "policy": {"metricThresholds": [{"metric": "runner_identity_static_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}}, + {"id": "v060-identity-schema", "type": "schema", "argv": ["node", "scripts/tabellio-validator.mjs", "--profile", "v060-identity-schema", "--validator-id", "v060-identity-schema", "--out", ".artifacts/tabellio/v060-identity-schema.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/v060-identity-schema.json"}, "policy": {"metricThresholds": [{"metric": "runner_identity_schema_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}}, + {"id": "v060-identity-semantic", "type": "semantic", "argv": ["node", "scripts/tabellio-validator.mjs", "--profile", "v060-identity-semantic", "--validator-id", "v060-identity-semantic", "--out", ".artifacts/tabellio/v060-identity-semantic.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/v060-identity-semantic.json"}, "policy": {"metricThresholds": [{"metric": "runner_identity_semantic_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}}, + {"id": "v060-identity-workflow", "type": "workflow", "argv": ["node", "scripts/tabellio-validator.mjs", "--profile", "v060-identity-workflow", "--validator-id", "v060-identity-workflow", "--out", ".artifacts/tabellio/v060-identity-workflow.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/v060-identity-workflow.json"}, "policy": {"metricThresholds": [{"metric": "runner_identity_workflow_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}}, + {"id": "v060-identity-operational", "type": "operational", "argv": ["node", "scripts/tabellio-validator.mjs", "--profile", "v060-identity-operational", "--validator-id", "v060-identity-operational", "--out", ".artifacts/tabellio/v060-identity-operational.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/v060-identity-operational.json"}, "policy": {"metricThresholds": [{"metric": "runner_identity_10x_duration_ms", "operator": "lte", "value": 2000}], "maxCostUsd": 0, "requireCostTelemetry": true}}, + {"id": "v060-identity-security", "type": "security", "argv": ["node", "scripts/tabellio-validator.mjs", "--profile", "v060-identity-security", "--validator-id", "v060-identity-security", "--out", ".artifacts/tabellio/v060-identity-security.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/v060-identity-security.json"}, "policy": {"metricThresholds": [{"metric": "runner_identity_security_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}} + ] +} diff --git a/tabellio.validation.json b/tabellio.validation.json index 4e04d68..f7d654e 100644 --- a/tabellio.validation.json +++ b/tabellio.validation.json @@ -4,8 +4,8 @@ "failFast": false, "requireEntireCheckpoint": true, "acceptance": { - "id": "tabellio-v0.5-control-plane-and-analytics", - "source": "repository product contract, docs/product-validation.md, and INTB-261", + "id": "tabellio-v0.6-control-plane-analytics-and-runner-identity", + "source": "repository product contract, docs/product-validation.md, INTB-261, and INTB-279", "risk": "high", "outcomes": [ "Agent runs, checkpoints, reviews, and validation results remain bound to exact Git state.", @@ -13,14 +13,16 @@ "External actions and release operations remain approval-gated and fail closed.", "Validation worktrees and isolated home directories remain outside the repository and Git common directory and are removed after each run.", "Analytics validator evidence distinguishes malformed inputs from valid contract failures.", - "The packaged cross-repository baseline binds exact repository heads, provider snapshots, and a reproducible report." + "The packaged cross-repository baseline binds exact repository heads, provider snapshots, and a reproducible report.", + "Every new typed validation result identifies the Tabellio package version and exact runner source commit." ], "invariants": [ "Validation never publishes, deploys, or mutates a live external system.", "Unknown or missing cost telemetry cannot be treated as zero.", "A validation run cannot select a workspace root inside the repository or Git common directory.", "Analytics validator summaries and artifacts never copy rejected private source text.", - "Baseline unavailable reasons remain derived from their packaged provider snapshots." + "Baseline unavailable reasons remain derived from their packaged provider snapshots.", + "Unavailable or dirty runner source identity remains explicit and is never inferred as released." ], "forbiddenOutcomes": [ "A stale validation result is accepted for a moved head.", @@ -28,7 +30,8 @@ "An unapproved external action is executed.", "A validation worktree, temporary HOME, dependency cache, or generated validator output is created under .git.", "Malformed analytics input is reported as passed or as a valid product failure.", - "The shipped validation manifest references baseline evidence omitted from the package." + "The shipped validation manifest references baseline evidence omitted from the package.", + "Package metadata, runner evidence, and release identity disagree." ], "requiredValidatorTypes": ["static", "schema", "semantic", "workflow", "operational", "security"] }, diff --git a/tests/baseline-integration.test.mjs b/tests/baseline-integration.test.mjs index 9f2b868..809c6c6 100644 --- a/tests/baseline-integration.test.mjs +++ b/tests/baseline-integration.test.mjs @@ -33,6 +33,7 @@ test("npm package includes every baseline validation input", async () => { }); const files = new Set(JSON.parse(stdout)[0].files.map((file) => file.path)); const required = [ + ".tabellio/validators.json", datasetPath, reportPath, "reports/analytics/sources/2026-07-28-condere-provider-snapshot.json", diff --git a/tests/buildkite-high-gates.test.mjs b/tests/buildkite-high-gates.test.mjs index 954ff33..b53db55 100644 --- a/tests/buildkite-high-gates.test.mjs +++ b/tests/buildkite-high-gates.test.mjs @@ -6,23 +6,26 @@ async function repositoryFile(path) { return readFile(new URL(`../${path}`, import.meta.url), "utf8"); } -test("Buildkite adds bounded pull-request quality gates without CI cutover", async () => { +test("Buildkite runs bounded pull-request quality gates on the included macOS queue", async () => { const [pipeline, productValidation, fallow, packageCheck, gitToolchain] = await Promise.all([ repositoryFile(".buildkite/pipeline.yml"), repositoryFile(".buildkite/scripts/product-validation.sh"), repositoryFile(".buildkite/scripts/fallow.sh"), repositoryFile(".buildkite/scripts/package.sh"), - repositoryFile(".buildkite/scripts/build-modern-git.sh"), + repositoryFile(".buildkite/scripts/use-modern-git.sh"), ]); + assert.match(pipeline, /queue: "macos-medium"/); assert.match(pipeline, /key: "repository-check"/); assert.match(pipeline, /key: "fallow"/); assert.match(pipeline, /key: "package"/); assert.match(pipeline, /key: "product-validation"/); + assert.doesNotMatch(pipeline, /linux-small/); + assert.doesNotMatch(pipeline, /git-toolchain/); assert.doesNotMatch(pipeline, /BUILDKITE_GITHUB_EVENT/); assert.equal( pipeline.match(/build\.pull_request\.id != null/g)?.length, - 5, + 4, ); assert.match( pipeline, @@ -60,17 +63,11 @@ test("Buildkite adds bounded pull-request quality gates without CI cutover", asy assert.match(fallow, /--gate new-only/); assert.match(packageCheck, /npm pack --dry-run --json/); assert.match(packageCheck, /forgejo\|change-request-provider/); - assert.match(gitToolchain, /dpkg-query/); - assert.match(gitToolchain, /Dir::Etc::sourcelist="\$apt_source_list"/); - assert.match(gitToolchain, /Dir::Etc::sourceparts="\$apt_source_parts"/); - assert.match(gitToolchain, /Dir::State::lists="\$apt_lists"/); - assert.match(gitToolchain, /ubuntu\\\.com/); - assert.match(gitToolchain, /debian\\\.org/); - assert.match(gitToolchain, /sources\.list\.d\/ubuntu\.sources/); - assert.match(gitToolchain, /sources\.list\.d\/debian\.sources/); - assert.doesNotMatch(gitToolchain, /sources\.list\.d\/\*/); - assert.equal(gitToolchain.match(/sudo apt-get "\$\{apt_options\[@\]\}"/g)?.length, 2); - assert.doesNotMatch(gitToolchain, /^\s*sudo apt-get update\s*$/m); + assert.match(gitToolchain, /required_version="2\.50\.1"/); + assert.match(gitToolchain, /git version/); + assert.match(gitToolchain, /awk -v actual=/); + assert.doesNotMatch(gitToolchain, /buildkite-agent artifact download/); + assert.doesNotMatch(gitToolchain, /apt-get/); }); function assertMatches(value, patterns) { diff --git a/tests/context-and-evidence.test.mjs b/tests/context-and-evidence.test.mjs index f69ba65..2be9c4a 100644 --- a/tests/context-and-evidence.test.mjs +++ b/tests/context-and-evidence.test.mjs @@ -288,6 +288,8 @@ test("stable schema identifiers keep external references and released contracts validationAlias, validationV1, validationV2, + validationV3, + validationV4, ] = await Promise.all([ readFile(`${projectRoot}/schemas/evidence-envelope.schema.json`, "utf8").then(JSON.parse), readFile(`${projectRoot}/schemas/external-action-policy.schema.json`, "utf8").then(JSON.parse), @@ -299,6 +301,8 @@ test("stable schema identifiers keep external references and released contracts readFile(`${projectRoot}/schemas/validation-result.schema.json`, "utf8").then(JSON.parse), readFile(`${projectRoot}/schemas/validation-result.v0.1.schema.json`, "utf8").then(JSON.parse), readFile(`${projectRoot}/schemas/validation-result.v0.2.schema.json`, "utf8").then(JSON.parse), + readFile(`${projectRoot}/schemas/validation-result.v0.3.schema.json`, "utf8").then(JSON.parse), + readFile(`${projectRoot}/schemas/validation-result.v0.4.schema.json`, "utf8").then(JSON.parse), ]); assert.equal(evidenceSchema.properties.externalActionPolicy.$ref, policySchema.$id); assert.equal(releaseSchema.properties.control.properties.intent.$ref, controlSchema.$id); @@ -312,6 +316,13 @@ test("stable schema identifiers keep external references and released contracts assert.equal(validationV2.$id, "urn:tabellio:schema:validation-result:v0.2"); assert.equal(Object.hasOwn(validationV1.properties, "checkpointRevision"), false); assert.equal(validationV2.required.includes("checkpointRevision"), true); + assert.equal(validationV3.$id, "urn:tabellio:schema:validation-result:v0.3"); + assert.equal(validationV4.$id, "urn:tabellio:schema:validation-result:v0.4"); + assert.deepEqual(validationV3.properties.runner.required, ["id", "runtime"]); + assert.equal(validationV4.properties.runner.required.includes("packageVersion"), true); + assert.equal(validationV4.properties.runner.required.includes("sourceCommit"), true); + assert.equal(JSON.stringify(validationV4).includes("validation-result.v0.3.schema.json"), false); + assert.equal(Object.hasOwn(validationV4.$defs, "revision"), true); }); test("required repository validation is recorded in evidence", async (t) => { diff --git a/tests/runner-identity.test.mjs b/tests/runner-identity.test.mjs new file mode 100644 index 0000000..5afbc34 --- /dev/null +++ b/tests/runner-identity.test.mjs @@ -0,0 +1,280 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, stat, utimes, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { execFile } from "node:child_process"; +import test from "node:test"; + +import { runGit } from "../scripts/lib/git-process.mjs"; +import { tabellioRunnerIdentity, tabellioRunnerState } from "../scripts/lib/runner-identity.mjs"; +import { verifyPublishedRunnerRelease } from "../scripts/lib/runner-release.mjs"; +import { identityEnv } from "./helpers/git-fixture.mjs"; + +const execFileAsync = promisify(execFile); + +test("runner identity schema binds package version, source commit, cleanliness, and release tag", async (t) => { + const root = await identityFixture(t); + const clean = await tabellioRunnerIdentity({ root }); + assert.equal(clean.packageName, "@intelip/tabellio"); + assert.equal(clean.packageVersion, "0.6.0"); + assert.match(clean.sourceCommit, /^[0-9a-f]{40}$/); + assert.equal(clean.sourceDirty, false); + assert.equal(clean.releaseTag, null); + + await runGit({ args: ["tag", "v0.6.0"], cwd: root }); + assert.equal((await tabellioRunnerIdentity({ root })).releaseTag, "v0.6.0"); + + await writeFile(join(root, "private-customer-name.txt"), "not exported\n"); + const dirtyState = await tabellioRunnerState({ root }); + const dirty = dirtyState.identity; + assert.equal(dirty.sourceDirty, true); + assert.equal(Object.values(dirty).some((value) => String(value).includes("private-customer-name")), false); + await writeFile(join(root, "private-customer-name.txt"), "changed while still dirty\n"); + const changedState = await tabellioRunnerState({ root }); + assert.deepEqual(changedState.identity, dirtyState.identity); + assert.notEqual(changedState.fingerprint, dirtyState.fingerprint); + + const nested = join(root, "nested"); + await mkdir(nested); + await runGit({ args: ["init", "-b", "main"], cwd: nested }); + await writeFile(join(nested, "nested.txt"), "one\n"); + const nestedState = await tabellioRunnerState({ root }); + assert.equal(nestedState.identity.sourceDirty, true); + assert.match(nestedState.fingerprint, /^[0-9a-f]{64}$/); + await runGit({ args: ["add", "nested.txt"], cwd: nested }); + await runGit({ args: ["commit", "-m", "Add nested source"], cwd: nested, env: identityEnv() }); + const committedNestedState = await tabellioRunnerState({ root }); + assert.notEqual(committedNestedState.fingerprint, nestedState.fingerprint); + await writeFile(join(nested, "nested.txt"), "two\n"); + const changedNestedState = await tabellioRunnerState({ root }); + assert.notEqual(changedNestedState.fingerprint, committedNestedState.fingerprint); +}); + +test("runner identity security reports unavailable non-Git source without exposing paths", async (t) => { + const root = await temporaryDirectory(t, "TabellioPackage-"); + await writeFile(join(root, "package.json"), JSON.stringify({ + name: "@intelip/tabellio", + version: "0.6.0", + })); + const identity = await tabellioRunnerIdentity({ root }); + assert.deepEqual(identity, { + packageName: "@intelip/tabellio", + packageVersion: "0.6.0", + sourceCommit: null, + sourceDirty: null, + releaseTag: null, + }); + assert.equal(JSON.stringify(identity).includes(root), false); +}); + +test("runner identity does not attribute a parent consumer repository to an installed package", async (t) => { + const consumer = await temporaryDirectory(t, "TabellioConsumer-"); + await runGit({ args: ["init", "-b", "main"], cwd: consumer }); + await writeFile(join(consumer, "package.json"), JSON.stringify({ name: "consumer", version: "1.0.0" })); + await runGit({ args: ["add", "package.json"], cwd: consumer }); + await runGit({ args: ["commit", "-m", "Add consumer"], cwd: consumer, env: identityEnv() }); + const installed = join(consumer, "node_modules", "@intelip", "tabellio"); + await mkdir(installed, { recursive: true }); + await writeFile(join(installed, "package.json"), JSON.stringify({ + name: "@intelip/tabellio", + version: "0.6.0", + })); + + assert.deepEqual(await tabellioRunnerIdentity({ root: installed }), { + packageName: "@intelip/tabellio", + packageVersion: "0.6.0", + sourceCommit: null, + sourceDirty: null, + releaseTag: null, + }); +}); + +test("runner identity CLI workflow reports current checkout and enforces expectations", async (t) => { + const result = await execFileAsync(process.execPath, [ + "scripts/tabellio-version.mjs", + "--expect-version", "0.6.0", + "--expect-ref", "HEAD", + ], { cwd: new URL("..", import.meta.url), encoding: "utf8" }); + const value = JSON.parse(result.stdout); + assert.equal(value.ok, true); + assert.equal(value.runner.packageVersion, "0.6.0"); + assert.match(value.runner.sourceCommit, /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/); + assert.deepEqual(Object.keys(value.runner).sort(), [ + "packageName", + "packageVersion", + "releaseTag", + "sourceCommit", + "sourceDirty", + ]); + + const outside = await temporaryDirectory(t, "TabellioCaller-"); + const script = fileURLToPath(new URL("../scripts/tabellio-version.mjs", import.meta.url)); + const outsideResult = await execFileAsync(process.execPath, [ + script, + "--expect-ref", "HEAD", + ], { cwd: outside, encoding: "utf8" }); + assert.equal(JSON.parse(outsideResult.stdout).ok, true); + + await assert.rejects( + execFileAsync(process.execPath, [ + "scripts/tabellio-version.mjs", + "--expect-version", "9.9.9", + ], { cwd: new URL("..", import.meta.url), encoding: "utf8" }), + (error) => { + const blocked = JSON.parse(error.stdout); + assert.equal(error.code, 1); + assert.equal(blocked.ok, false); + assert.deepEqual(blocked.blockers, ["package_version_mismatch:9.9.9"]); + return true; + }, + ); + + await assert.rejects( + execFileAsync(process.execPath, [ + "scripts/tabellio-version.mjs", + "--expect-ref", "--show-toplevel", + ], { cwd: new URL("..", import.meta.url), encoding: "utf8" }), + (error) => { + assert.equal(error.code, 1); + assert.equal(error.stdout, ""); + assert.equal(error.stderr.includes(new URL("..", import.meta.url).pathname), false); + return true; + }, + ); +}); + +test("runner identity operational lookup stays bounded", async () => { + const started = performance.now(); + for (let index = 0; index < 10; index += 1) await tabellioRunnerIdentity(); + const duration = performance.now() - started; + console.log(`runner_identity_10x_duration_ms=${duration.toFixed(3)}`); +}); + +test("runner release proof rejects local-only tags and binds published GitHub evidence", async (t) => { + const root = await identityFixture(t); + await runGit({ args: ["tag", "v0.6.0"], cwd: root }); + const identity = await tabellioRunnerIdentity({ root }); + assert.equal(await verifyPublishedRunnerRelease({ + root, + identity, + repositoryReader: async () => ({ fullName: "IntelIP/Tabellio" }), + remoteTagReader: async () => null, + commandRunner: async () => { + throw new Error("GitHub must not be queried without a published remote tag."); + }, + }), false); + const remote = await temporaryDirectory(t, "TabellioReleaseRemote-"); + await runGit({ args: ["init", "--bare"], cwd: remote }); + await runGit({ args: ["tag", "--delete", "v0.6.0"], cwd: root }); + await runGit({ + args: ["tag", "--annotate", "v0.6.0", "--message", "Tabellio v0.6.0"], + cwd: root, + env: identityEnv(), + }); + await runGit({ args: ["remote", "add", "origin", remote], cwd: root }); + await runGit({ args: ["push", "origin", "v0.6.0"], cwd: root }); + assert.equal(await verifyPublishedRunnerRelease({ + root, + identity: await tabellioRunnerIdentity({ root }), + repositoryReader: async () => ({ fullName: "IntelIP/Tabellio" }), + commandRunner: async () => ({ + stdout: JSON.stringify({ tagName: "v0.6.0", isDraft: false, isPrerelease: false }), + }), + }), true); + assert.equal(await verifyPublishedRunnerRelease({ + root, + identity, + repositoryReader: async () => ({ fullName: "IntelIP/Tabellio" }), + remoteTagReader: async () => ({ annotated: true, commit: identity.sourceCommit }), + commandRunner: async ({ args }) => { + assert.deepEqual(args, [ + "release", "view", "v0.6.0", + "--repo", "IntelIP/Tabellio", + "--json", "tagName,isDraft,isPrerelease", + ]); + return { + stdout: JSON.stringify({ tagName: "v0.6.0", isDraft: false, isPrerelease: false }), + }; + }, + }), true); + assert.equal(await verifyPublishedRunnerRelease({ + root, + identity, + repositoryReader: async () => ({ fullName: "IntelIP/Tabellio" }), + remoteTagReader: async () => ({ annotated: false, commit: identity.sourceCommit }), + commandRunner: async () => ({ stdout: "{}" }), + }), false); +}); + +test("runner identity fingerprints tracked changes larger than the Git output buffer", async (t) => { + const root = await identityFixture(t); + const large = join(root, "large.bin"); + await writeFile(large, Buffer.alloc(11 * 1024 * 1024, 1)); + await runGit({ args: ["add", "large.bin"], cwd: root }); + await runGit({ args: ["commit", "-m", "Add large file"], cwd: root, env: identityEnv() }); + await writeFile(large, Buffer.alloc(11 * 1024 * 1024, 2)); + const state = await tabellioRunnerState({ root }); + assert.equal(state.identity.sourceDirty, true); + assert.match(state.fingerprint, /^[0-9a-f]{64}$/); +}); + +test("runner identity inspection does not refresh the Git index", async (t) => { + const root = await identityFixture(t); + const packagePath = join(root, "package.json"); + const now = new Date(); + await utimes(packagePath, now, now); + const before = await stat(join(root, ".git", "index")); + await tabellioRunnerState({ root }); + const after = await stat(join(root, ".git", "index")); + assert.equal(after.mtimeMs, before.mtimeMs); + assert.equal(after.ctimeMs, before.ctimeMs); +}); + +test("runner identity treats unsafe index flags and their combination as dirty", async (t) => { + for (const flags of [ + ["--assume-unchanged"], + ["--skip-worktree"], + ["--assume-unchanged", "--skip-worktree"], + ]) { + const root = await identityFixture(t); + const packagePath = join(root, "package.json"); + for (const flag of flags) { + await runGit({ args: ["update-index", flag, "package.json"], cwd: root }); + } + await writeFile(packagePath, JSON.stringify({ + name: "@intelip/tabellio", + version: "9.9.9", + })); + const state = await tabellioRunnerState({ root }); + assert.equal(state.identity.sourceDirty, true); + assert.match(state.fingerprint, /^[0-9a-f]{64}$/); + } +}); + +async function identityFixture(t) { + const root = await temporaryDirectory(t, "TabellioIdentity-"); + await writeFile(join(root, "package.json"), JSON.stringify({ + name: "@intelip/tabellio", + version: "0.6.0", + })); + await runGit({ args: ["init", "-b", "main"], cwd: root }); + await runGit({ args: ["add", "package.json"], cwd: root }); + await runGit({ + args: ["commit", "-m", "Add package identity"], + cwd: root, + env: identityEnv(), + }); + assert.equal(JSON.parse(await readFile(join(root, "package.json"), "utf8")).version, "0.6.0"); + return root; +} + +async function temporaryDirectory(t, prefix) { + const root = await mkdtemp(join(tmpdir(), prefix)); + t.after(async () => { + const { rm } = await import("node:fs/promises"); + await rm(root, { recursive: true, force: true }); + }); + return root; +} diff --git a/tests/validation-runner.test.mjs b/tests/validation-runner.test.mjs index e710d86..1150a01 100644 --- a/tests/validation-runner.test.mjs +++ b/tests/validation-runner.test.mjs @@ -6,6 +6,7 @@ import test from "node:test"; import { GitJsonLedger } from "../scripts/lib/git-json-ledger.mjs"; import { runGit } from "../scripts/lib/git-process.mjs"; import { repositoryIdentity } from "../scripts/lib/repository-identity.mjs"; +import { digestObject } from "../scripts/lib/stack-operation.mjs"; import { latestValidationResult, ValidationRunner, @@ -68,6 +69,49 @@ test("validation runner executes exact committed manifests and stores bounded re assert.equal(passed.result.commands[4].status, "passed"); assert.equal(validateValidationResult(passed.result), passed.result); assert.deepEqual(await latestValidationResult(ledger, passingHead), passed.result); + let identityRead = 0; + const stableIdentity = { + packageName: "@intelip/tabellio", + packageVersion: "0.6.0", + sourceCommit: "a".repeat(40), + sourceDirty: false, + releaseTag: null, + }; + const driftingRunner = new ValidationRunner({ + store, + ledger, + runnerIdentity: async () => ({ + ...stableIdentity, + sourceCommit: (identityRead++ === 0 ? "a" : "b").repeat(40), + }), + }); + await assert.rejects( + driftingRunner.run({ + repositoryId, + commit: passingHead, + base: "main", + runnerId: "drift-test", + }), + /runner identity changed during validation/, + ); + let stateRead = 0; + const dirtyStateRunner = new ValidationRunner({ + store, + ledger, + runnerState: async () => ({ + identity: { ...stableIdentity, sourceDirty: true }, + fingerprint: stateRead++ === 0 ? "dirty-state-before" : "dirty-state-after", + }), + }); + await assert.rejects( + dirtyStateRunner.run({ + repositoryId, + commit: passingHead, + base: "main", + runnerId: "dirty-state-drift-test", + }), + /runner identity changed during validation/, + ); const otherRepository = await runner.run({ repositoryId: "other/repository", commit: passingHead, @@ -332,14 +376,29 @@ test("typed validators enforce semantic metrics and cost budgets with durable ev const store = await NativeGitStore.open(fixture.seed); const ledger = await GitJsonLedger.open({ repoPath: fixture.seed, ref: "refs/tabellio/validations" }); - const result = await new ValidationRunner({ store, ledger }).run({ + const result = await new ValidationRunner({ + store, + ledger, + runnerIdentity: async () => ({ + packageName: "@intelip/tabellio", + packageVersion: "0.6.0", + sourceCommit: "a".repeat(40), + sourceDirty: false, + releaseTag: null, + }), + }).run({ repositoryId: "example/repository", commit: "HEAD", base: "main", runnerId: "product-validator", }); - assert.equal(result.result.schemaVersion, "tabellio-validation-result/v0.3"); + assert.equal(result.result.schemaVersion, "tabellio-validation-result/v0.4"); + assert.equal(result.result.runner.packageName, "@intelip/tabellio"); + assert.equal(result.result.runner.packageVersion, "0.6.0"); + assert.equal(result.result.runner.sourceCommit, "a".repeat(40)); + assert.equal(result.result.runner.sourceDirty, false); + assert.equal(result.result.runner.releaseTag, null); assert.equal(result.result.status, "passed"); assert.equal(result.result.acceptance.id, "PLANE-101"); assert.deepEqual(result.result.acceptance.requiredValidatorTypes, ["semantic", "operational"]); @@ -352,6 +411,23 @@ test("typed validators enforce semantic metrics and cost budgets with durable ev assert.equal(result.result.decision.totalCostUsd, 0.11); assert.equal(result.result.decision.costTelemetryComplete, true); assert.equal(validateValidationResult(result.result), result.result); + + for (const [field, invalid, message] of [ + ["packageName", "@example/not-tabellio", /packageName must be/], + ["packageVersion", "1.0.0-alpha..1", /packageVersion must be a semantic version/], + ]) { + const malformed = structuredClone(result.result); + malformed.runner[field] = invalid; + const { integrity: _integrity, ...unsigned } = malformed; + malformed.integrity.digest = digestObject(unsigned); + assert.throws(() => validateValidationResult(malformed), message); + } + + const buildMetadata = structuredClone(result.result); + buildMetadata.runner.packageVersion = "0.6.1+build.7"; + const { integrity: _integrity, ...unsigned } = buildMetadata; + buildMetadata.integrity.digest = digestObject(unsigned); + assert.equal(validateValidationResult(buildMetadata), buildMetadata); }); test("typed validation distinguishes product failure from blocked evidence", async (t) => {