From c707de542c4a98c3cac3314c0e4c1cb6761c886b Mon Sep 17 00:00:00 2001 From: Laith Al-Saadoon Date: Sat, 10 Oct 2026 16:55:25 +0000 Subject: [PATCH 1/6] fix(security): render the OpenVEX suppressions beside the docs site lockfile too osv-scanner applies the osv-scanner.toml in a lockfile's own directory and no other one. A recursive `osv-scanner scan source -r .`, which is what the OpenSSF Scorecard Vulnerabilities check runs, loaded the root copy for uv.lock and reported both npm advisories the ledger argues (GHSA-vfj7-8cjw-p6xm, GHSA-ch52-4w7c-c8xp) against site/pnpm-lock.yaml, so Scorecard scored the check 8 of 10. The renderer takes one output per lockfile directory and writes the same render to each; `security:vex:check` drift-gates both copies, and three tests cover the site copy green, drifted and missing. With site/osv-scanner.toml in place the same recursive scan reports no issues. Co-Authored-By: Claude Opus 5.5 --- mise.toml | 9 ++- osv-scanner.toml | 7 ++ .../atif-cli/tests/test_vex_drift_gate.py | 46 ++++++++++++ scripts/vex_to_osv_config.py | 69 ++++++++++------- site/osv-scanner.toml | 74 +++++++++++++++++++ 5 files changed, 174 insertions(+), 31 deletions(-) create mode 100644 site/osv-scanner.toml diff --git a/mise.toml b/mise.toml index e5ac51d..a162efb 100644 --- a/mise.toml +++ b/mise.toml @@ -336,14 +336,15 @@ depends = [ # --------------------------------------------------------------------------- [tasks."security:vex"] -description = "Regenerate osv-scanner.toml and dependency-review's allow-ghsas from the OpenVEX ledger" +description = "Regenerate both osv-scanner.toml copies (root and site/) and dependency-review's allow-ghsas from the OpenVEX ledger" # NOT in `security`'s depends, deliberately. `osv-scanner.toml` is COMMITTED and drift-gated, so # regenerating it as part of the scan would let a scan run against a suppression surface no gate # has ever seen — the scan would define its own exemptions and then pass. Editing # `security/atif-sql.openvex.json` means running this task and committing what it rewrote: # osv-scanner.toml and the `allow-ghsas:` line of .github/workflows/dependency-review.yml. run = """ -uv run python scripts/vex_to_osv_config.py security/atif-sql.openvex.json osv-scanner.toml \ +uv run python scripts/vex_to_osv_config.py security/atif-sql.openvex.json \ + osv-scanner.toml site/osv-scanner.toml \ --pnpm-lock site/pnpm-lock.yaml --workflow .github/workflows/dependency-review.yml """ @@ -366,6 +367,7 @@ description = "Drift gate: osv-scanner.toml and allow-ghsas must match the ledge sources = [ "security/**/*.json", "osv-scanner.toml", + "site/osv-scanner.toml", "scripts/vex_to_osv_config.py", "uv.lock", "site/pnpm-lock.yaml", @@ -382,7 +384,8 @@ sources = [ # report. run = """ set -eu -uv run python scripts/vex_to_osv_config.py security/atif-sql.openvex.json osv-scanner.toml \ +uv run python scripts/vex_to_osv_config.py security/atif-sql.openvex.json \ + osv-scanner.toml site/osv-scanner.toml \ --pnpm-lock site/pnpm-lock.yaml --workflow .github/workflows/dependency-review.yml --check scratch=$(mktemp -d) trap 'rm -rf "$scratch"' EXIT diff --git a/osv-scanner.toml b/osv-scanner.toml index 2344fab..27e0faa 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -17,6 +17,13 @@ # emitted, and the version scoping is enforced by the generator instead: it asserts every PURL # version in the ledger against uv.lock and FAILS when a bump moves the package, which turns a # silent stale suppression into a red gate. +# +# WHY THERE ARE TWO COPIES +# osv-scanner applies the osv-scanner.toml in a lockfile's own directory and no other one. +# Probed 2026-10-10 with osv-scanner 2.6.0 over `scan source -r .` (what OpenSSF Scorecard's +# Vulnerabilities check runs): the root copy suppressed nothing for site/pnpm-lock.yaml and +# both npm advisories were reported. So the same render is written to the root, beside uv.lock, +# and to site/, beside pnpm-lock.yaml, and each copy reports the other lockfile's ids unused. [[IgnoredVulns]] id = "GHSA-vfj7-8cjw-p6xm" diff --git a/packages/atif-cli/tests/test_vex_drift_gate.py b/packages/atif-cli/tests/test_vex_drift_gate.py index a3bf31b..b2cb9b1 100644 --- a/packages/atif-cli/tests/test_vex_drift_gate.py +++ b/packages/atif-cli/tests/test_vex_drift_gate.py @@ -177,3 +177,49 @@ def test_render_rewrites_the_allow_line_from_the_ledger(tmp_path: Path) -> None: assert (tmp_path / "osv.toml").read_text(encoding="utf-8") == (ROOT / OSV_CONFIG).read_text( encoding="utf-8" ) + + +#: osv-scanner reads the config in a lockfile's own directory only, so the docs site's lockfile +#: needs its own copy of the render: without it Scorecard's recursive scan reports every npm +#: advisory the ledger argued. +SITE_OSV_CONFIG = Path("site/osv-scanner.toml") + + +def _run_both(site_copy: Path = SITE_OSV_CONFIG) -> subprocess.CompletedProcess[str]: + argv = [ + sys.executable, + str(SCRIPT), + str(LEDGER), + str(OSV_CONFIG), + str(site_copy), + "--lock", + str(UV_LOCK), + "--pnpm-lock", + str(PNPM_LOCK), + "--check", + ] + return subprocess.run( # noqa: S603 - fixed interpreter and repository script + argv, capture_output=True, text=True, check=False, cwd=ROOT + ) + + +def test_site_copy_beside_the_pnpm_lock_is_green() -> None: + result = _run_both() + assert result.returncode == 0, result.stderr + assert f"{SITE_OSV_CONFIG} matches {LEDGER}" in result.stdout + assert f"{OSV_CONFIG} matches {LEDGER}" in result.stdout + + +def test_site_copy_that_drifted_is_red(tmp_path: Path) -> None: + drifted = tmp_path / "osv-scanner.toml" + text = (ROOT / SITE_OSV_CONFIG).read_text(encoding="utf-8") + drifted.write_text(text.replace("[[IgnoredVulns]]", "", 1), encoding="utf-8") + result = _run_both(drifted) + assert result.returncode == 1 + assert f"{drifted} has DRIFTED from {LEDGER}" in result.stderr + + +def test_site_copy_that_is_missing_is_red(tmp_path: Path) -> None: + result = _run_both(tmp_path / "osv-scanner.toml") + assert result.returncode == 1 + assert "does not exist but" in result.stderr diff --git a/scripts/vex_to_osv_config.py b/scripts/vex_to_osv_config.py index 9331f9c..1a31702 100755 --- a/scripts/vex_to_osv_config.py +++ b/scripts/vex_to_osv_config.py @@ -4,7 +4,7 @@ Fail when either disagrees with the ledger, or the ledger with the lockfiles. - scripts/vex_to_osv_config.py [--lock uv.lock] + scripts/vex_to_osv_config.py [ ...] [--lock uv.lock] [--pnpm-lock site/pnpm-lock.yaml] [--workflow .github/workflows/dependency-review.yml] scripts/vex_to_osv_config.py [...the same...] --check @@ -172,6 +172,13 @@ # emitted, and the version scoping is enforced by the generator instead: it asserts every PURL # version in the ledger against uv.lock and FAILS when a bump moves the package, which turns a # silent stale suppression into a red gate. +# +# WHY THERE ARE TWO COPIES +# osv-scanner applies the osv-scanner.toml in a lockfile's own directory and no other one. +# Probed 2026-10-10 with osv-scanner 2.6.0 over `scan source -r .` (what OpenSSF Scorecard's +# Vulnerabilities check runs): the root copy suppressed nothing for site/pnpm-lock.yaml and +# both npm advisories were reported. So the same render is written to the root, beside uv.lock, +# and to site/, beside pnpm-lock.yaml, and each copy reports the other lockfile's ids unused. {body}""" #: One decoded JSON value. `json.loads` is typed `Any`, so without a declared shape every @@ -568,7 +575,11 @@ def main(argv: Sequence[str]) -> int: description="Render osv-scanner.toml from an OpenVEX ledger.", ) parser.add_argument("ledger", help="path to the OpenVEX ledger JSON") - parser.add_argument("output", help="path to the osv-scanner.toml to write or check") + parser.add_argument( + "output", + nargs="+", + help="path of each osv-scanner.toml to write or check: one per lockfile directory", + ) parser.add_argument( "--lock", default="uv.lock", help="lockfile the PURL versions are checked against" ) @@ -588,7 +599,7 @@ def main(argv: Sequence[str]) -> int: args = parser.parse_args(argv) ledger_path = Path(args.ledger) - output_path = Path(args.output) + output_paths = [Path(output) for output in args.output] pnpm_lock = Path(args.pnpm_lock) if args.pnpm_lock else None statements = load(ledger_path, Path(args.lock), pnpm_lock) rendered = render(statements, ledger_path.as_posix()) @@ -602,8 +613,9 @@ def main(argv: Sequence[str]) -> int: _fail(f"cannot read {workflow}: {error.strerror or type(error).__name__}") if not args.check: - output_path.write_text(rendered, encoding="utf-8") - sys.stdout.write(f"vex-to-osv-config: wrote {output_path} from {ledger_path}\n") + for output_path in output_paths: + output_path.write_text(rendered, encoding="utf-8") + sys.stdout.write(f"vex-to-osv-config: wrote {output_path} from {ledger_path}\n") if workflow is not None: workflow.write_text( render_workflow(workflow_text, ghsas, workflow.as_posix()), encoding="utf-8" @@ -618,30 +630,31 @@ def main(argv: Sequence[str]) -> int: f"({len(ghsas)} GHSA ids)\n" ) - try: - committed = output_path.read_text(encoding="utf-8") - except OSError: - _fail( - f"{output_path} does not exist but {ledger_path} does — run " - f"`mise run security:vex` and commit the result" - ) - if committed != rendered: - diff = difflib.unified_diff( - committed.splitlines(keepends=True), - rendered.splitlines(keepends=True), - fromfile=f"{output_path} (committed)", - tofile=f"{output_path} (rendered from {ledger_path})", - ) - sys.stderr.writelines(diff) - _fail( - f"{output_path} has DRIFTED from {ledger_path}. A generated suppression surface " - f"nothing verifies is a suppression surface that outlives its reason: run " - f"`mise run security:vex` and commit both files" + for output_path in output_paths: + try: + committed = output_path.read_text(encoding="utf-8") + except OSError: + _fail( + f"{output_path} does not exist but {ledger_path} does — run " + f"`mise run security:vex` and commit the result" + ) + if committed != rendered: + diff = difflib.unified_diff( + committed.splitlines(keepends=True), + rendered.splitlines(keepends=True), + fromfile=f"{output_path} (committed)", + tofile=f"{output_path} (rendered from {ledger_path})", + ) + sys.stderr.writelines(diff) + _fail( + f"{output_path} has DRIFTED from {ledger_path}. A generated suppression surface " + f"nothing verifies is a suppression surface that outlives its reason: run " + f"`mise run security:vex` and commit both files" + ) + sys.stdout.write( + f"vex-to-osv-config: {output_path} matches {ledger_path} " + f"({len(rendered.splitlines())} lines)\n" ) - sys.stdout.write( - f"vex-to-osv-config: {output_path} matches {ledger_path} " - f"({len(rendered.splitlines())} lines)\n" - ) return 0 diff --git a/site/osv-scanner.toml b/site/osv-scanner.toml new file mode 100644 index 0000000..27e0faa --- /dev/null +++ b/site/osv-scanner.toml @@ -0,0 +1,74 @@ +# GENERATED by scripts/vex_to_osv_config.py from security/atif-sql.openvex.json — do not edit. +# +# Edit the ledger, then run `mise run security:vex` and commit both files. +# `mise run security:vex:check` (inside `mise run check`) fails if this file drifts from it. +# +# WHY THIS FILE IS GENERATED +# Probed 2026-08-28: osv-scanner 2.5.0 has no `--vex` flag, so the OpenVEX ledger is rendered +# into the one suppression surface it does have. grype 0.111.1 and trivy 0.70.0 both accept +# `--vex` and read the ledger directly, so they have no generated counterpart. +# +# WHY THE VERSION IS NOT IN THE ENTRIES BELOW +# A VEX statement scopes to a product VERSION through its PURL, so a suppression stops applying +# when the dependency moves. osv-scanner 2.5.0 cannot express that: read from its source at +# v2.5.0, `Config.ShouldIgnore(vulnID)` matches `[[IgnoredVulns]]` on the id alone, and the +# version-aware `[[PackageOverrides]]` carries no vulnerability id, so using it would suppress +# every future advisory against the same pinned version too. So the id-scoped entry is what is +# emitted, and the version scoping is enforced by the generator instead: it asserts every PURL +# version in the ledger against uv.lock and FAILS when a bump moves the package, which turns a +# silent stale suppression into a red gate. +# +# WHY THERE ARE TWO COPIES +# osv-scanner applies the osv-scanner.toml in a lockfile's own directory and no other one. +# Probed 2026-10-10 with osv-scanner 2.6.0 over `scan source -r .` (what OpenSSF Scorecard's +# Vulnerabilities check runs): the root copy suppressed nothing for site/pnpm-lock.yaml and +# both npm advisories were reported. So the same render is written to the root, beside uv.lock, +# and to site/, beside pnpm-lock.yaml, and each copy reports the other lockfile's ids unused. + +[[IgnoredVulns]] +id = "GHSA-vfj7-8cjw-p6xm" +reason = ''' +not_affected: vulnerable_code_cannot_be_controlled_by_adversary + +Products: pkg:npm/braces@3.0.3 + +braces reaches the docs site (site/) on exactly one path, read with `pnpm why braces` on 2026-10-08: starlight-llms-txt 0.12.0 -> micromatch 4.0.8 -> braces 3.0.3. It is a build-time dependency of the static docs site and ships in no atif-sql package or CLI. The vulnerable input is the brace PATTERN, and the only call sites are `micromatch.isMatch(doc.id, pattern)` in starlight-llms-txt's generator.ts, where the doc id is the string matched and never expanded. The only patterns that reach it are the literals `["agents", "index*"]` that site/astro.config.ts passes as `promote`; `include`, `exclude` and `demote` are unset. Those literals are authored in this repository, contain no braces and are reviewed in pull requests, so no outside party chooses a pattern, and a crash would abort a local or CI docs build rather than serve anyone. Revisit when the docs build takes patterns from the environment, a content file or a contributor-supplied config, or when braces publishes a release newer than 3.0.3. + +Recorded in security/atif-sql.openvex.json (OpenVEX statements[0]). Do not edit this file directly; edit the ledger and run `mise run security:vex`. +''' + +[[IgnoredVulns]] +id = "CVE-2026-93687" +reason = ''' +not_affected: vulnerable_code_cannot_be_controlled_by_adversary + +Products: pkg:npm/braces@3.0.3 + +braces reaches the docs site (site/) on exactly one path, read with `pnpm why braces` on 2026-10-08: starlight-llms-txt 0.12.0 -> micromatch 4.0.8 -> braces 3.0.3. It is a build-time dependency of the static docs site and ships in no atif-sql package or CLI. The vulnerable input is the brace PATTERN, and the only call sites are `micromatch.isMatch(doc.id, pattern)` in starlight-llms-txt's generator.ts, where the doc id is the string matched and never expanded. The only patterns that reach it are the literals `["agents", "index*"]` that site/astro.config.ts passes as `promote`; `include`, `exclude` and `demote` are unset. Those literals are authored in this repository, contain no braces and are reviewed in pull requests, so no outside party chooses a pattern, and a crash would abort a local or CI docs build rather than serve anyone. Revisit when the docs build takes patterns from the environment, a content file or a contributor-supplied config, or when braces publishes a release newer than 3.0.3. + +Recorded in security/atif-sql.openvex.json (OpenVEX statements[0]). Do not edit this file directly; edit the ledger and run `mise run security:vex`. +''' + +[[IgnoredVulns]] +id = "GHSA-ch52-4w7c-c8xp" +reason = ''' +not_affected: vulnerable_code_not_in_execute_path + +Products: pkg:npm/http-cache-semantics@4.2.0 + +http-cache-semantics reaches the docs site (site/) on exactly one path, read with `pnpm why http-cache-semantics` on 2026-10-08: astro 7.3.5 -> http-cache-semantics 4.2.0, a build-time dependency that ships in no atif-sql package or CLI. astro's only use is `dist/assets/build/remote.js`, which builds a CachePolicy to decide how long a downloaded REMOTE image may be cached during `astro build`. The docs site has no remote image: site/astro.config.ts sets no `image` option (no `domains`, no `remotePatterns`, so astro's default allows none), and grep over site/src, site/authored and docs/ finds no `astro:assets` image component and no markdown image with an http(s) source on 2026-10-08. The advisory needs a shared cache that answers many users' requests; here the one request is the build's own fetch, built by astro without a max-stale directive, in one process, and the output is a static directory. Nothing serves other users' cached responses. Revisit when the site adds a remote image, an SSR adapter or any shared HTTP cache, or when a release newer than 4.2.0 appears. + +Recorded in security/atif-sql.openvex.json (OpenVEX statements[1]). Do not edit this file directly; edit the ledger and run `mise run security:vex`. +''' + +[[IgnoredVulns]] +id = "CVE-2026-93748" +reason = ''' +not_affected: vulnerable_code_not_in_execute_path + +Products: pkg:npm/http-cache-semantics@4.2.0 + +http-cache-semantics reaches the docs site (site/) on exactly one path, read with `pnpm why http-cache-semantics` on 2026-10-08: astro 7.3.5 -> http-cache-semantics 4.2.0, a build-time dependency that ships in no atif-sql package or CLI. astro's only use is `dist/assets/build/remote.js`, which builds a CachePolicy to decide how long a downloaded REMOTE image may be cached during `astro build`. The docs site has no remote image: site/astro.config.ts sets no `image` option (no `domains`, no `remotePatterns`, so astro's default allows none), and grep over site/src, site/authored and docs/ finds no `astro:assets` image component and no markdown image with an http(s) source on 2026-10-08. The advisory needs a shared cache that answers many users' requests; here the one request is the build's own fetch, built by astro without a max-stale directive, in one process, and the output is a static directory. Nothing serves other users' cached responses. Revisit when the site adds a remote image, an SSR adapter or any shared HTTP cache, or when a release newer than 4.2.0 appears. + +Recorded in security/atif-sql.openvex.json (OpenVEX statements[1]). Do not edit this file directly; edit the ledger and run `mise run security:vex`. +''' From c4bb3335272fbaf4de93599aefa2fa0f6861682d Mon Sep 17 00:00:00 2001 From: Laith Al-Saadoon Date: Sat, 10 Oct 2026 17:05:50 +0000 Subject: [PATCH 2/6] feat(release): sign, attest and attach the release assets with SBOMs A release published to PyPI left nothing on the GitHub release a consumer could verify, so OpenSSF Scorecard scored Signed-Releases as no data and a user had no SBOM to answer "what does this version ship?". publish.yml now stages the release on the tag with `mise run release:assets`: the wheel and sdist uploaded to PyPI, CycloneDX and SPDX SBOMs of the locked runtime closure the gate tested, and SHA256SUMS. A separate job holding the only `attestations: write` signs SLSA v1 build provenance over every asset and an SBOM attestation over the distributions through Sigstore, and writes both bundles into `atif_sql-.intoto.jsonl`. The upload job checks each asset with `gh attestation verify --signer-workflow` against publish.yml, then scripts/verify_release_assets.py --attestations read at the tag, before `gh release upload`; the PyPI job also waits on the attestations. release.yml pushes the bump commit and tag through a write deploy key held in a `release` environment limited to main, so a ruleset on main that requires pull requests can name deploy keys as its automated bypass actor. RELEASING.md documents the assets, the verify commands, the ruleset and the deploy key setup; SECURITY.md points at them, and README shows the Scorecard score badge. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish.yml | 144 ++++++++++- .github/workflows/release.yml | 40 ++- .gitignore | 1 + CONTRIBUTING.md | 2 +- README.md | 1 + RELEASING.md | 116 +++++++-- SECURITY.md | 10 + mise.toml | 34 +++ .../atif-cli/tests/test_release_assets.py | 211 +++++++++++++++ scripts/verify_release_assets.py | 244 ++++++++++++++++++ 10 files changed, 764 insertions(+), 39 deletions(-) create mode 100644 packages/atif-cli/tests/test_release_assets.py create mode 100755 scripts/verify_release_assets.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a766d39..99b2e51 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -50,6 +50,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + outputs: + version: ${{ steps.version.outputs.version }} steps: # The TAG, not the branch. `main` has moved on by the time a maintainer publishes the # draft, and the bytes that reach PyPI have to be the bytes that were released. @@ -85,6 +87,12 @@ jobs: # nothing installs and gives the preflight below seven extra files to reject. - run: uv build --out-dir dist + - id: version + name: Read the version from the tag + env: + RELEASE_TAG: ${{ inputs.tag || github.event.release.tag_name }} + run: echo "version=${RELEASE_TAG#v}" >> "$GITHUB_OUTPUT" + # The preflight, on the property a single-wheel distribution can get wrong. # # Seven module trees reach PyPI inside ONE wheel, and no sibling is on the index, so the @@ -229,6 +237,138 @@ jobs: retention-days: 7 if-no-files-found: error + # What the GitHub release carries beside PyPI: these same two files, a CycloneDX and an + # SPDX SBOM of the locked runtime closure this job just tested, and SHA256SUMS. Staged + # here, in the job that has no `id-token`, so the attest job below signs bytes it did not + # make; `release:assets` ends in scripts/verify_release_assets.py, so a stray file, a + # name that disagrees with the tag or an SBOM for another version fails before signing. + - name: Stage the release assets + env: + RELEASE_TAG: ${{ inputs.tag || github.event.release.tag_name }} + run: RELEASE_VERSION="${RELEASE_TAG#v}" mise run release:assets + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-assets + path: release/ + retention-days: 7 + if-no-files-found: error + + # SLSA build provenance over every release asset, and an SBOM attestation over the two + # distributions, both signed through Sigstore with this workflow's OIDC identity and stored in + # GitHub's attestation API, where `gh attestation verify` finds them by digest. The bundles + # are also written into the release as `atif_sql-.intoto.jsonl`, one per line, so a + # consumer can verify offline and OpenSSF Scorecard's Signed-Releases check sees provenance. + # + # Its own job because it is the only one that holds `attestations: write`: the build job + # cannot sign, and the upload job below cannot mint an identity. + attest: + name: attest the release assets + needs: build + runs-on: ubuntu-latest + permissions: + id-token: write # request the Sigstore signing certificate for this workflow's identity + attestations: write # store the attestations in the repository's attestation API + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-assets + path: release + + - id: provenance + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: release/* + + - id: sbom + name: Attest the SBOM + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: | + release/*.whl + release/*.tar.gz + sbom-path: release/atif_sql-${{ needs.build.outputs.version }}.cdx.json + + # One compact Sigstore bundle per line: the shape `gh attestation verify --bundle` reads. + - name: Write the attestation bundles + env: + VERSION: ${{ needs.build.outputs.version }} + PROVENANCE: ${{ steps.provenance.outputs.bundle-path }} + SBOM: ${{ steps.sbom.outputs.bundle-path }} + run: | + set -euo pipefail + mkdir attestations + jq -c . "$PROVENANCE" "$SBOM" > "attestations/atif_sql-$VERSION.intoto.jsonl" + wc -l "attestations/atif_sql-$VERSION.intoto.jsonl" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: attestations + path: attestations/ + retention-days: 7 + if-no-files-found: error + + # Uploads the attested assets to the GitHub release, after proving each one verifies against + # THIS workflow's identity and that the attestations cover every asset. An asset nobody can + # verify fails the release here rather than shipping beside the ones that can be verified. + # + # No checkout and no `id-token`: the job holds `contents: write` for one `gh release upload`, + # and its inputs are the two artifacts the jobs above produced. The verifier script is read at + # the TAG through the API, so a change on main cannot loosen the gate an old tag passes. + # + # Assets are added to the release after it is published, so this needs immutable releases + # left off in the repository settings. `gh release upload` without `--clobber` refuses to + # replace an asset: a re-dispatch after a partial upload fails loudly rather than swapping + # bytes a consumer may already have verified. + release-assets: + name: upload the signed release assets + needs: [build, attest] + runs-on: ubuntu-latest + permissions: + contents: write # upload assets to the existing release + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-assets + path: release + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: attestations + path: release + + - name: Verify every asset against the signing workflow + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ needs.build.outputs.version }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + bundle="release/atif_sql-$VERSION.intoto.jsonl" + for asset in release/*; do + [ "$asset" = "$bundle" ] && continue + gh attestation verify "$asset" --bundle "$bundle" --repo "$REPO" \ + --signer-workflow "$REPO/.github/workflows/publish.yml" + done + + - name: Check the attestations cover the whole release + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ needs.build.outputs.version }} + TAG: ${{ inputs.tag || github.event.release.tag_name }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + gh api -H "Accept: application/vnd.github.raw+json" \ + "repos/$REPO/contents/scripts/verify_release_assets.py?ref=$TAG" > verify_release_assets.py + python3 verify_release_assets.py release "$VERSION" --attestations + + - name: Upload the assets to the release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ inputs.tag || github.event.release.tag_name }} + REPO: ${{ github.repository }} + run: gh release upload "$TAG" release/* --repo "$REPO" + # WHAT A MAINTAINER CONFIGURES ON PYPI, BEFORE THE FIRST RUN # --------------------------------------------------------- # At (the ACCOUNT-level "Publishing" page, not a @@ -275,7 +415,9 @@ jobs: # single-distribution shape rather than of this file. publish: name: publish atif-sql - needs: build + # `attest` as well as `build`: no version reaches PyPI without its provenance in GitHub's + # attestation API. + needs: [build, attest] runs-on: ubuntu-latest # The human decision, and the reason it sits HERE rather than earlier: publishing the draft # release says "these are the release notes", which is reversible. This says "these bytes diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee3cd7e..06df103 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,6 +45,10 @@ jobs: release: name: cut the release runs-on: ubuntu-latest + # The `release` environment holds RELEASE_DEPLOY_KEY and lets only `main` deploy to it, so + # no other branch's workflow and no pull request can read the one credential that may push + # to `main` past its ruleset. + environment: release permissions: # Pushes the bump commit and the tag, and creates the draft release. `id-token: write` # is deliberately absent: nothing here talks to PyPI, so this job cannot mint a @@ -66,17 +70,33 @@ jobs: # the last tag, and a shallow clone would find no tag and treat every commit in the # repository as unreleased. # - # `persist-credentials: true` — the exception to this repository's default, because this - # is the one job that pushes. The token it leaves in .git/config is the same - # `GITHUB_TOKEN` scoped to `contents: write` above, this job uploads no artifact for it to - # leak into, and every step after the checkout is either a first-party mise task or `git` - # itself. Writing the token into a remote URL instead would put the same secret in the - # same file with none of that reasoning attached, which is why the audit is silenced here - # rather than worked around. + # `persist-credentials: true`, the exception to this repository's default, because this + # is the one job that pushes. With `ssh-key` set, checkout writes the deploy key to a + # runner temp file and points `core.sshCommand` at it; the key can push to this one + # repository and nothing else, this job uploads no artifact for it to leak into, and every + # step after the checkout is either a first-party mise task or `git` itself. That is why + # the audit is silenced here rather than worked around. + # + # The push below goes through a deploy key, the one actor the `main` ruleset lets past its + # pull-request rule: `GITHUB_TOKEN` pushes as `github-actions[bot]`, which no bypass list + # can name. Checked before the checkout, because an empty `ssh-key` makes the checkout fall + # back to the token silently and the run would fail ten minutes later at the push, with + # `GH006`. A dry run pushes nothing and needs no key. + - name: Require the release deploy key + if: ${{ !inputs.dry_run }} + env: + KEY: ${{ secrets.RELEASE_DEPLOY_KEY }} + run: | + if [ -z "$KEY" ]; then + echo "::error::The release environment has no RELEASE_DEPLOY_KEY secret. RELEASING.md, 'The release push and branch protection', has the setup." + exit 1 + fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # zizmor: ignore[artipacked] the one job that pushes needs the credential it checks out with fetch-depth: 0 persist-credentials: true + ssh-key: ${{ secrets.RELEASE_DEPLOY_KEY }} - uses: jdx/mise-action@9149ea85001c7435d5a66bb127d6a1b6227cb0a5 # v5.0.0 with: @@ -215,10 +235,8 @@ jobs: # `--follow-tags` pushes the annotated tag with the commit, in one ref update, so a tag # can never exist without the commit it names. # - # This push is refused with `GH006: Protected branch update failed` when a ruleset on - # `main` requires a pull request, because `GITHUB_TOKEN` pushes as `github-actions[bot]` - # and a ruleset bypass list cannot name it (bypass actors are roles, teams, GitHub Apps, - # and Dependabot). RELEASING.md carries the setting and the local fallback. + # The checkout's deploy key makes this push, and the `main` ruleset names deploy keys as + # its one automated bypass actor. RELEASING.md carries the setting and the local fallback. - name: Push the bump commit and the tag if: ${{ !inputs.dry_run }} run: git push --follow-tags diff --git a/.gitignore b/.gitignore index dd62fc9..99a61a2 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ coverage.xml htmlcov/ .mise/ dist/ +release/ build/ *.egg-info/ .DS_Store diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 08332ab..5c0ffbb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,7 +42,7 @@ that. | `typecheck` | `uv run ty check` | `[tool.ty.rules] all = "error"`, and `[tool.ty.terminal] error-on-warning = true` means a warning is also a failure | | `lint:imports` | `uv run lint-imports` | the import contracts below | | `lint:pnpm-lock` | `python3 scripts/verify_single_yaml_document.py site/pnpm-lock.yaml` | `site/pnpm-lock.yaml` holding more than one YAML document, which GitHub's dependency graph cannot read | -| `security:vex:check` | `uv run python scripts/vex_to_osv_config.py ... --check` | `osv-scanner.toml` or the `allow-ghsas:` line of `.github/workflows/dependency-review.yml` drifting from `security/atif-sql.openvex.json`, or a ledger PURL version that `uv.lock` or `site/pnpm-lock.yaml` no longer locks; `mise run security:vex` re-renders both | +| `security:vex:check` | `uv run python scripts/vex_to_osv_config.py ... --check` | either `osv-scanner.toml` (root, beside `uv.lock`, and `site/`, beside `pnpm-lock.yaml`, since osv-scanner reads only the config in a lockfile's own directory) or the `allow-ghsas:` line of `.github/workflows/dependency-review.yml` drifting from `security/atif-sql.openvex.json`, or a ledger PURL version that `uv.lock` or `site/pnpm-lock.yaml` no longer locks; `mise run security:vex` re-renders both | | `docs:prose` | `python3 scripts/vale_gate.py` | an error-level Vale alert in `docs/**/*.md`, `site/authored/**/*.md`, `README.md`, `AGENTS.md` or `CONTRIBUTING.md`, a run that checked fewer files than that scope holds, or a glob that matched fewer files than its floor in `scripts/vale_gate.py` (`docs/**/*.md` at least 15, `site/authored/**/*.md` at least 2, each root file at least 1), or a rule whose warnings rose past its count in `.vale-baseline.json` (or that warns and is not in it); a baseline that is missing, empty, or does not parse fails as well. `.vale.ini` names the rules and why each one below error sits there, and a new term goes in `.vale/styles/config/vocabularies/atif-sql/accept.txt` | | `test` | `uv run pytest --no-header -q` | `[tool.pytest.ini_options] testpaths = ["packages/*/tests"]` | diff --git a/README.md b/README.md index 49fb500..3ecc101 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![security](https://github.com/laithalsaadoon/atif-sql/actions/workflows/security.yml/badge.svg?branch=main)](https://github.com/laithalsaadoon/atif-sql/actions/workflows/security.yml) [![CodeQL](https://github.com/laithalsaadoon/atif-sql/actions/workflows/codeql.yml/badge.svg?branch=main)](https://github.com/laithalsaadoon/atif-sql/actions/workflows/codeql.yml) [![Scorecard](https://github.com/laithalsaadoon/atif-sql/actions/workflows/scorecard.yml/badge.svg?branch=main)](https://github.com/laithalsaadoon/atif-sql/actions/workflows/scorecard.yml) +[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/laithalsaadoon/atif-sql/badge)](https://scorecard.dev/viewer/?uri=github.com/laithalsaadoon/atif-sql) ![Python 3.13+](https://img.shields.io/badge/python-3.13%2B-blue.svg) [![License Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) diff --git a/RELEASING.md b/RELEASING.md index cab1e32..38d9377 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -251,8 +251,47 @@ litellm`) prints nothing, and `test_dev_only_imports_guard.py` fails if `src/` i 4. **Review the draft and press "Publish release".** That press is what starts `publish.yml` — see below for why it has to be a human and not the workflow. 5. **Approve the `pypi` environment.** `publish.yml` builds the sdist and the wheel, verifies - them, installs them, and then waits. A reviewer approves once and all the upload legs - proceed. + them, installs them, stages the release assets, attests them, and then waits. A reviewer + approves once and all the upload legs proceed. The signed assets reach the GitHub release + without waiting for that approval, so they are on the release whether or not PyPI is. + +## What a GitHub release carries, and how to verify it + +Every release carries the same six assets, staged by `mise run release:assets` on the tag and +uploaded by `publish.yml`: + +| Asset | What it is | +| --- | --- | +| `atif_sql--py3-none-any.whl`, `atif_sql-.tar.gz` | The distributions, byte for byte the ones uploaded to PyPI | +| `atif_sql-.cdx.json`, `atif_sql-.spdx.json` | CycloneDX and SPDX SBOMs of the locked runtime closure the release gate tested (`uv export --no-dev` over `uv.lock`) | +| `SHA256SUMS` | The digest of each of the four files above | +| `atif_sql-.intoto.jsonl` | Two Sigstore bundles, one per line: SLSA v1 build provenance over all five files above, and an SBOM attestation over the two distributions | + +The attestations are signed through Sigstore with `publish.yml`'s own OIDC identity, so the +signer is the workflow file at the tag, not a key someone holds. GitHub's attestation API +stores the same bundles, so verification needs only the file: + +```bash +v=0.2.0 +gh release download "v$v" -R laithalsaadoon/atif-sql -D "atif-sql-$v" +cd "atif-sql-$v" +sha256sum --check SHA256SUMS +for f in atif_sql-$v-py3-none-any.whl atif_sql-$v.tar.gz atif_sql-$v.cdx.json atif_sql-$v.spdx.json SHA256SUMS; do + gh attestation verify "$f" --bundle "atif_sql-$v.intoto.jsonl" -R laithalsaadoon/atif-sql \ + --signer-workflow laithalsaadoon/atif-sql/.github/workflows/publish.yml +done +``` + +Drop `--bundle` to verify against the attestation API instead. A wheel installed from PyPI +verifies the same way, and PyPI shows its own PEP 740 attestation for each file on the +project page. + +`publish.yml` runs the same `gh attestation verify` on every asset, then +`scripts/verify_release_assets.py --attestations`, before `gh release upload`. A stray file, a +name that disagrees with the tag, a digest `SHA256SUMS` does not hold, an SBOM for another +version, or an asset the provenance does not cover fails the upload; its tests plant each +defect. `--clobber` is never passed, so a re-dispatch never replaces an asset someone may have +verified: it fails on the first name that exists. ### Why the release is a draft @@ -334,38 +373,61 @@ the fat-wheel restructure above. Neither is on the table. | **Private vulnerability reporting** enabled | Settings → Advanced Security | `SECURITY.md` and `.github/ISSUE_TEMPLATE/config.yml` both route reports to `/security/advisories/new`, which 404s while this is off. | | **Require review from Code Owners** on `main` | Branch protection or ruleset | `.github/CODEOWNERS` only *requests* review until this is on; with it off the file is a notification list. | | `default_workflow_permissions` = `read` | Settings → Actions | Every workflow here declares its own per-job permissions, so the default needs no write. | -| A push path to `main` for the release workflow | Branch protection or ruleset | See below. | +| Ruleset `main` on the default branch | Settings → Rules → Rulesets | See below: no deletion, no force push, pull requests with two approvals and Code Owner review, stale approvals dismissed, the check jobs required on an up-to-date branch. | +| Environment `release` with secret `RELEASE_DEPLOY_KEY`, deployments from `main` only | Settings → Environments | The private half of the write deploy key `release.yml` pushes the bump commit and tag with. See below. | +| **Immutable releases** off | Settings → General → Releases | `publish.yml` adds the signed assets after the release is published, which an immutable release refuses. | | Pages **Source = GitHub Actions** | Settings → Pages | `docs.yml` deploys through `actions/deploy-pages`, which needs the Actions source rather than a branch. `base: "/atif-sql/"` in `site/astro.config.ts` matches the resulting project-site path. | | The AI-crawler policy, at the ORIGIN root | The `laithalsaadoon.github.io` user-pages repository | `robots.txt` is per-origin (RFC 9309 §2.3), so a project site served from a path segment cannot own one: Astro emits this site's copy to `/atif-sql/robots.txt`, which no crawler fetches. `site/public/robots.txt` is the decided policy and the exact text to install at `https://laithalsaadoon.github.io/robots.txt`. Until it is installed there, these pages inherit whatever that origin already serves — and absent a file, that is fully permissive. Outside this repository's reach, so no gate here can assert it. | ### The release push and branch protection -`release.yml` pushes the bump commit and the tag with `GITHUB_TOKEN`, which acts as -`github-actions[bot]`. A ruleset on `main` that requires a pull request refuses that push with -`GH006: Protected branch update failed`, and **a ruleset bypass list cannot name it** — the -eligible bypass actors are repository/organisation/enterprise admins, the maintain and write -roles or custom roles based on write, teams, GitHub Apps, and Dependabot. `github-actions[bot]` -is none of those. +`main` carries one ruleset, `main`, which OpenSSF Scorecard's Branch-Protection check reads: -The supported resolutions: +- deletion and non-fast-forward pushes are refused, for everyone; +- a change arrives through a pull request with two approvals, Code Owner review, the most + recent push approved, review threads resolved, and stale approvals dismissed on a new push; +- the `check.yml` jobs must pass on a branch that is up to date with `main`. -1. **Grant the push.** Keep the pull-request requirement off `main`, or add a bypass actor the - token can act as. The repository is single-maintainer and every change still goes through a - pull request by convention; the gate that matters is `check.yml`. -2. **Cut the version locally.** The tool is the same one CI runs, from the same lockfile, so - the outcome is identical: +The repository has one maintainer, who cannot approve their own pull request, so the +maintainer's account is a bypass actor and merges without the approvals. The checks still run +on the pull request and the history rules still hold, because a bypass is recorded in the +ruleset's insights and nothing here force-pushes. - ```bash - git switch main && git pull --ff-only - mise run check - uv run cz bump --changelog --check-consistency --annotated-tag - git push --follow-tags - gh release create "v$(uv run cz version --project)" --draft --verify-tag \ - --title "v$(uv run cz version --project)" \ - --notes-file <(uv run cz changelog --dry-run "$(uv run cz version --project)") - ``` +`release.yml` pushes the bump commit and the tag to `main` directly. `GITHUB_TOKEN` pushes as +`github-actions[bot]`, which no ruleset bypass list can name, so the push goes through a +**deploy key** instead, and deploy keys are the ruleset's second bypass actor. Setup, once: + +```bash +ssh-keygen -t ed25519 -N '' -C 'atif-sql release.yml' -f release_deploy_key +gh repo deploy-key add release_deploy_key.pub -R laithalsaadoon/atif-sql \ + --allow-write --title 'release.yml push' +gh api -X PUT repos/laithalsaadoon/atif-sql/environments/release \ + -F 'deployment_branch_policy[protected_branches]=false' \ + -F 'deployment_branch_policy[custom_branch_policies]=true' +gh api -X POST repos/laithalsaadoon/atif-sql/environments/release/deployment-branch-policies \ + -f name=main -f type=branch +gh secret set RELEASE_DEPLOY_KEY --env release -R laithalsaadoon/atif-sql < release_deploy_key +shred -u release_deploy_key release_deploy_key.pub +``` + +Only a run of the `release` job on `main` can read the key, and `release.yml` refuses a +non-dry run without it before it checks anything out. To rotate it, repeat the setup and delete +the old key under Settings → Deploy keys. + +Without the key, cut the version locally; the maintainer's bypass lets the push through, and +the tool is the same one CI runs, from the same lockfile: + +```bash +git switch main && git pull --ff-only +mise run check +uv run cz bump --changelog --check-consistency --annotated-tag +git push --follow-tags +gh release create "v$(uv run cz version --project)" --draft --verify-tag \ + --title "v$(uv run cz version --project)" \ + --notes-file <(uv run cz changelog --dry-run "$(uv run cz version --project)") +``` - From there the flow is identical: review the draft, press Publish, approve `pypi`. +From there the flow is identical: review the draft, press Publish, approve `pypi`. ## Cutting a specific version @@ -404,6 +466,8 @@ an existing filename either way, so this cannot replace bytes. | `mise run lock:check` fails on the release commit | `pre_bump_hooks` is missing from `[tool.commitizen]`, so `uv.lock` still records the previous versions. | | `commitizen found no version-bumping commit since the last tag` | Nothing to release. Land a `feat:`/`fix:`, or dispatch with an explicit `increment`. | | `CurrentVersionNotFoundError: Current version ... is not found in ` | A `version_files` entry points at a file whose version was hand-edited. `--check-consistency` is doing its job. | -| `GH006: Protected branch update failed` | See "The release push and branch protection". | +| `GH006: Protected branch update failed` | The push did not use the deploy key, or the key is not a bypass actor of the `main` ruleset. See "The release push and branch protection". | +| `The release environment has no RELEASE_DEPLOY_KEY secret` | The deploy key setup has not happened yet. Same section. | +| `gh release upload` fails with `already exists` | A re-dispatch of `publish.yml` for a release that already carries its assets. Nothing to do: the assets on the release are the ones first verified. | | 403 from PyPI naming a publisher it cannot find | On a first release, the pending publisher does not exist yet. Afterwards, the publisher does not match the claims. | | `expected one sdist and one wheel for , staged 0` | `uv build --all-packages` did not produce that project — usually a member removed from `[tool.uv.workspace] members`. | diff --git a/SECURITY.md b/SECURITY.md index 29fd6e1..3b61634 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -17,6 +17,16 @@ response time is promised. Expect an acknowledgement and a fix or an explicit "won't fix" when someone gets to it — and please keep the report private until a fix ships or you hear that one is not coming. +## Verifying a release + +Each GitHub release carries the wheel and sdist uploaded to PyPI, CycloneDX and SPDX SBOMs, +`SHA256SUMS`, and `atif_sql-.intoto.jsonl`: SLSA build provenance and an SBOM +attestation, signed through Sigstore by the `publish.yml` workflow itself. `gh attestation +verify -R laithalsaadoon/atif-sql --signer-workflow +laithalsaadoon/atif-sql/.github/workflows/publish.yml` checks any of them; +[RELEASING.md](RELEASING.md#what-a-github-release-carries-and-how-to-verify-it) has the +full procedure. + ## What the security-relevant surface actually is atif-sql is a local analytics tool. It reads the operator's own Claude Code diff --git a/mise.toml b/mise.toml index a162efb..19814c0 100644 --- a/mise.toml +++ b/mise.toml @@ -666,6 +666,40 @@ depends = [ "security:workflows", ] +# --------------------------------------------------------------------------- +# Release assets: what a GitHub release carries beside the PyPI upload. +# +# publish.yml runs this on the tag after `uv build`, attests what it staged, and uploads it to +# the release. It runs the same on a laptop, so what a release will carry is never something +# only CI can show. RELEASING.md has the verify commands a consumer runs. +# --------------------------------------------------------------------------- + +[tasks."release:assets"] +description = "Stage release/: the built wheel and sdist, CycloneDX and SPDX SBOMs of the locked runtime closure, SHA256SUMS" +# Needs RELEASE_VERSION (`0.2.0`, no `v`) and a `uv build --out-dir dist` of that version. +# +# The SBOM describes the LOCKED runtime closure, `uv export --no-dev` over uv.lock: the set +# `mise run check` tested on the tag. A user's `uv tool install` resolves the wheel's ranges +# afresh and can land newer patch versions, which an SBOM of a runner's install would record +# as if they had shipped. `--preview-features sbom-export` names the export's preview state +# instead of silencing it; uv is pinned in mise.lock, so the format moves only with a reviewed +# bump. syft converts that one document to SPDX, so the two formats cannot disagree. +# +# `security:sbom` is a different document: the whole repository, dev tools and the docs site +# included, kept 90 days as a CI artifact for "did we ever ship that?". +run = """ +set -eu +version="${RELEASE_VERSION:?set RELEASE_VERSION to the version being released, such as 0.2.0}" +rm -rf release +mkdir release +cp "dist/atif_sql-$version-py3-none-any.whl" "dist/atif_sql-$version.tar.gz" release/ +uv export --frozen --no-dev --preview-features sbom-export --format cyclonedx1.5 \ + --output-file "release/atif_sql-$version.cdx.json" > /dev/null +syft convert "release/atif_sql-$version.cdx.json" -o "spdx-json=release/atif_sql-$version.spdx.json" +(cd release && sha256sum -- * > SHA256SUMS) +uv run python scripts/verify_release_assets.py release "$version" +""" + # --------------------------------------------------------------------------- # CLI passthrough # --------------------------------------------------------------------------- diff --git a/packages/atif-cli/tests/test_release_assets.py b/packages/atif-cli/tests/test_release_assets.py new file mode 100644 index 0000000..b90c77a --- /dev/null +++ b/packages/atif-cli/tests/test_release_assets.py @@ -0,0 +1,211 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""`scripts/verify_release_assets.py` is the gate a GitHub release passes before upload. + +publish.yml stages the release with `mise run release:assets`, attests it, and uploads it only +when this script accepts the directory. A release asset nobody can verify must fail the release, +so each planted defect below turns the gate red, and a well-formed release is green: the +anti-vacuity proof every gate here carries. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +#: Repository root: `packages/atif-cli/tests/` -> three parents up. +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "verify_release_assets.py" +PUBLISH = ROOT / ".github" / "workflows" / "publish.yml" +VERSION = "0.2.0" +WHEEL = f"atif_sql-{VERSION}-py3-none-any.whl" +SDIST = f"atif_sql-{VERSION}.tar.gz" +CDX = f"atif_sql-{VERSION}.cdx.json" +SPDX = f"atif_sql-{VERSION}.spdx.json" +INTOTO = f"atif_sql-{VERSION}.intoto.jsonl" +SLSA = "https://slsa.dev/provenance/v1" +CYCLONEDX = "https://cyclonedx.org/bom" + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _bundle(predicate_type: str, files: list[Path]) -> str: + statement = { + "_type": "https://in-toto.io/Statement/v1", + "subject": [{"name": f.name, "digest": {"sha256": _sha(f)}} for f in files], + "predicateType": predicate_type, + "predicate": {}, + } + payload = base64.b64encode(json.dumps(statement).encode()).decode() + envelope = {"payloadType": "application/vnd.in-toto+json", "payload": payload} + return json.dumps({"mediaType": "application/vnd.dev.sigstore.bundle.v0.3+json", + "dsseEnvelope": envelope}) # fmt: skip + + +def _write_sums(directory: Path) -> None: + lines = [f"{_sha(directory / name)} {name}" for name in sorted([WHEEL, SDIST, CDX, SPDX])] + (directory / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def _stage(tmp_path: Path, *, attest: bool = True) -> Path: + directory = tmp_path / "release" + directory.mkdir() + (directory / WHEEL).write_bytes(b"wheel bytes") + (directory / SDIST).write_bytes(b"sdist bytes") + cdx = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": {"component": {"name": "atif-sql", "version": VERSION}}, + "components": [{"name": "duckdb", "version": "1.5.6", "purl": "pkg:pypi/duckdb@1.5.6"}], + } + (directory / CDX).write_text(json.dumps(cdx), encoding="utf-8") + spdx = { + "spdxVersion": "SPDX-2.3", + "packages": [ + {"name": "atif-sql", "versionInfo": VERSION}, + {"name": "duckdb", "versionInfo": "1.5.6"}, + ], + } + (directory / SPDX).write_text(json.dumps(spdx), encoding="utf-8") + _write_sums(directory) + if attest: + _attest(directory) + return directory + + +def _attest(directory: Path, *, skip: str = "", sbom: bool = True) -> None: + files = [directory / n for n in sorted([WHEEL, SDIST, CDX, SPDX, "SHA256SUMS"]) if n != skip] + lines = [_bundle(SLSA, files)] + if sbom: + lines.append(_bundle(CYCLONEDX, [directory / WHEEL, directory / SDIST])) + (directory / INTOTO).write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def _run(directory: Path, *, attestations: bool = True) -> subprocess.CompletedProcess[str]: + argv = [sys.executable, str(SCRIPT), str(directory), VERSION] + if attestations: + argv.append("--attestations") + return subprocess.run( # noqa: S603 - fixed interpreter and repository script + argv, capture_output=True, text=True, check=False, cwd=ROOT + ) + + +def test_staged_release_without_attestations_is_green(tmp_path: Path) -> None: + result = _run(_stage(tmp_path, attest=False), attestations=False) + assert result.returncode == 0, result.stderr + assert f"release {VERSION}: 5 assets, 0 findings" in result.stdout + + +def test_attested_release_is_green(tmp_path: Path) -> None: + result = _run(_stage(tmp_path)) + assert result.returncode == 0, result.stderr + assert "6 assets, attestations covered, 0 findings" in result.stdout + + +def test_empty_directory_is_vacuous(tmp_path: Path) -> None: + (tmp_path / "release").mkdir() + result = _run(tmp_path / "release") + assert result.returncode == 2 + assert "EMPTY" in result.stderr + + +def test_stray_file_is_red(tmp_path: Path) -> None: + directory = _stage(tmp_path) + (directory / "helper").write_bytes(b"\x7fELF") + result = _run(directory) + assert result.returncode == 1 + assert "UNLISTED helper" in result.stderr + + +def test_missing_sbom_is_red(tmp_path: Path) -> None: + directory = _stage(tmp_path) + (directory / SPDX).unlink() + result = _run(directory) + assert result.returncode == 1 + assert f"MISSING {SPDX}" in result.stderr + + +def test_version_that_disagrees_with_the_names_is_red(tmp_path: Path) -> None: + directory = _stage(tmp_path) + argv = [sys.executable, str(SCRIPT), str(directory), "0.2.1", "--attestations"] + result = subprocess.run(argv, capture_output=True, text=True, check=False) # noqa: S603 + assert result.returncode == 1 + assert "MISSING atif_sql-0.2.1-py3-none-any.whl" in result.stderr + assert f"UNLISTED {WHEEL}" in result.stderr + + +def test_one_byte_appended_to_the_wheel_is_red(tmp_path: Path) -> None: + directory = _stage(tmp_path) + with (directory / WHEEL).open("ab") as wheel: + wheel.write(b"x") + result = _run(directory) + assert result.returncode == 1 + assert f"DIGEST {WHEEL}" in result.stderr + assert f"NO-PROVENANCE {WHEEL} is not a subject of the provenance" in result.stderr + + +def test_sums_that_omit_an_asset_is_red(tmp_path: Path) -> None: + directory = _stage(tmp_path) + sums = directory / "SHA256SUMS" + kept = [line for line in sums.read_text().splitlines() if not line.endswith(SDIST)] + sums.write_text("\n".join(kept) + "\n") + _attest(directory) + result = _run(directory) + assert result.returncode == 1 + assert f"SHA256SUMS does not list {SDIST}" in result.stderr + + +def test_sbom_for_another_version_is_red(tmp_path: Path) -> None: + directory = _stage(tmp_path, attest=False) + document = json.loads((directory / CDX).read_text()) + document["metadata"]["component"]["version"] = "0.1.0" + (directory / CDX).write_text(json.dumps(document)) + _write_sums(directory) + _attest(directory) + result = _run(directory) + assert result.returncode == 1 + assert f"SBOM {CDX} does not name atif-sql {VERSION}" in result.stderr + + +def test_asset_outside_the_provenance_is_red(tmp_path: Path) -> None: + directory = _stage(tmp_path) + _attest(directory, skip=SPDX) + result = _run(directory) + assert result.returncode == 1 + assert f"NO-PROVENANCE {SPDX} is not a subject of the provenance" in result.stderr + + +def test_missing_sbom_attestation_is_red(tmp_path: Path) -> None: + directory = _stage(tmp_path) + _attest(directory, sbom=False) + result = _run(directory) + assert result.returncode == 1 + assert "NO-SBOM-ATTESTATION" in result.stderr + + +def test_undecodable_attestation_line_is_red(tmp_path: Path) -> None: + directory = _stage(tmp_path) + (directory / INTOTO).write_text('{"dsseEnvelope": {"payload": "not base64!"}}\n') + result = _run(directory) + assert result.returncode == 1 + assert "does not decode" in result.stderr + assert "NO-PROVENANCE" in result.stderr + + +def test_publish_workflow_gates_the_upload_on_this_script() -> None: + # The gate only protects a release if the workflow that uploads runs it, after the signatures + # are checked against the signing workflow, and before `gh release upload`. + text = PUBLISH.read_text(encoding="utf-8") + stage = text.index('run: RELEASE_VERSION="${RELEASE_TAG#v}" mise run release:assets') + verify_signatures = text.index('gh attestation verify "$asset" --bundle') + verify_assets = text.index('verify_release_assets.py release "$VERSION" --attestations') + upload = text.index("run: gh release upload") + assert stage < verify_assets < upload + assert verify_signatures < upload + assert "--signer-workflow" in text diff --git a/scripts/verify_release_assets.py b/scripts/verify_release_assets.py new file mode 100755 index 0000000..bb0f5f9 --- /dev/null +++ b/scripts/verify_release_assets.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Prove a staged GitHub release carries exactly what a consumer needs to verify it. + + scripts/verify_release_assets.py + scripts/verify_release_assets.py --attestations + +`mise run release:assets` stages the release in one directory: the wheel and the sdist that go +to PyPI, a CycloneDX and an SPDX SBOM of the locked runtime closure, and a `SHA256SUMS` +manifest. `publish.yml` then signs Sigstore attestations over those files and adds them as +`atif_sql-.intoto.jsonl`, one bundle per line. This is the release gate both steps +run before anything is uploaded, and its rule is that an asset nobody can verify fails the +release rather than shipping beside the ones that can be: + +- every file is one the release lists by name, and every name carries the version + (a stray file, or a version that disagrees with the tag) +- `SHA256SUMS` names every other asset, and every digest matches its file +- both SBOMs parse, and each names atif-sql at this version as its subject +- with `--attestations`: the `.intoto.jsonl` holds a SLSA provenance statement whose subjects + are every asset's digest, and an SBOM statement whose subjects are the wheel and the sdist + +The signatures themselves are checked by `gh attestation verify` in the workflow, against the +signing workflow's identity; this script checks what was signed. The release is short of +assets in two ways that must not read alike: a missing asset is a finding (exit 1), and an +empty or absent directory means the staging step produced nothing to judge (exit 2). + +stdlib only, matching the other gate helpers: it runs on a runner that has no venv. +""" + +from __future__ import annotations + +import argparse +import base64 +import binascii +import hashlib +import json +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Sequence + +#: One decoded JSON value, as in scripts/vex_to_osv_config.py. +type JsonValue = str | int | float | bool | list["JsonValue"] | dict[str, "JsonValue"] | None + +DISTRIBUTION = "atif-sql" +STEM = "atif_sql" +SUMS = "SHA256SUMS" +SLSA_PROVENANCE = "https://slsa.dev/provenance/v1" +#: The predicate types actions/attest writes for an SBOM, by format. +SBOM_PREDICATES = frozenset({"https://cyclonedx.org/bom", "https://spdx.dev/Document/v2.3"}) + + +def expected_names(version: str, *, attestations: bool) -> dict[str, str]: + """Map each asset the release must carry to what it is.""" + names = { + f"{STEM}-{version}-py3-none-any.whl": "wheel", + f"{STEM}-{version}.tar.gz": "sdist", + f"{STEM}-{version}.cdx.json": "cyclonedx", + f"{STEM}-{version}.spdx.json": "spdx", + SUMS: "sums", + } + if attestations: + names[f"{STEM}-{version}.intoto.jsonl"] = "attestations" + return names + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _load_json(path: Path, problems: list[str]) -> JsonValue: + try: + value: JsonValue = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + problems.append(f"UNPARSABLE {path.name}: {error}") + return None + return value + + +def _check_sums(directory: Path, names: dict[str, str], problems: list[str]) -> None: + listed: dict[str, str] = {} + for number, line in enumerate((directory / SUMS).read_text(encoding="utf-8").splitlines(), 1): + digest, _, name = line.partition(" ") + name = name.removeprefix("*") + if len(digest) != 64 or not name: # noqa: PLR2004 - a sha256 hex digest is 64 characters + problems.append(f"SUMS {SUMS} line {number} is not ' ': {line!r}") + continue + listed[name] = digest + covered = {n for n, kind in names.items() if kind not in {"sums", "attestations"}} + problems.extend(f"SUMS {SUMS} does not list {name}" for name in sorted(covered - listed.keys())) + problems.extend( + f"SUMS {SUMS} lists {name}, which is not a release asset" + for name in sorted(listed.keys() - covered) + ) + for name in sorted(covered & listed.keys()): + actual = _sha256(directory / name) + if listed[name] != actual: + problems.append(f"DIGEST {name} is {actual}, {SUMS} says {listed[name]}") + + +def _check_cyclonedx(path: Path, version: str, problems: list[str]) -> None: + document = _load_json(path, problems) + if not isinstance(document, dict): + if document is not None: + problems.append(f"SBOM {path.name} is not a JSON object") + return + metadata = document.get("metadata") + subject = metadata.get("component") if isinstance(metadata, dict) else None + components = document.get("components") + if document.get("bomFormat") != "CycloneDX": + problems.append(f"SBOM {path.name} bomFormat is {document.get('bomFormat')!r}") + if not isinstance(subject, dict) or (subject.get("name"), subject.get("version")) != ( + DISTRIBUTION, + version, + ): + problems.append(f"SBOM {path.name} does not name {DISTRIBUTION} {version} as its subject") + if not isinstance(components, list) or not components: + problems.append(f"SBOM {path.name} lists no components") + + +def _check_spdx(path: Path, version: str, problems: list[str]) -> None: + document = _load_json(path, problems) + if not isinstance(document, dict): + if document is not None: + problems.append(f"SBOM {path.name} is not a JSON object") + return + spdx_version = document.get("spdxVersion") + packages = document.get("packages") + if not isinstance(spdx_version, str) or not spdx_version.startswith("SPDX-2."): + problems.append(f"SBOM {path.name} spdxVersion is {spdx_version!r}") + if not isinstance(packages, list) or not any( + isinstance(p, dict) and (p.get("name"), p.get("versionInfo")) == (DISTRIBUTION, version) + for p in packages + ): + problems.append(f"SBOM {path.name} has no package {DISTRIBUTION} {version}") + elif len(packages) < 2: # noqa: PLR2004 - the subject plus at least one dependency + problems.append(f"SBOM {path.name} lists no dependency of {DISTRIBUTION}") + + +def _statements(path: Path, problems: list[str]) -> list[dict[str, JsonValue]]: + """Decode the in-toto statement inside every Sigstore bundle line.""" + statements: list[dict[str, JsonValue]] = [] + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + bundle: JsonValue = json.loads(line) + envelope = bundle.get("dsseEnvelope") if isinstance(bundle, dict) else None + payload = envelope.get("payload") if isinstance(envelope, dict) else None + if not isinstance(payload, str): + problems.append(f"ATTESTATION {path.name} line {number} has no DSSE payload") + continue + statement: JsonValue = json.loads(base64.b64decode(payload, validate=True)) + except (json.JSONDecodeError, binascii.Error, UnicodeDecodeError) as error: + problems.append(f"ATTESTATION {path.name} line {number} does not decode: {error}") + continue + if isinstance(statement, dict): + statements.append(statement) + return statements + + +def _subject_digests(statement: dict[str, JsonValue]) -> set[str]: + subjects = statement.get("subject") + digests: set[str] = set() + if isinstance(subjects, list): + for subject in subjects: + digest = subject.get("digest") if isinstance(subject, dict) else None + value = digest.get("sha256") if isinstance(digest, dict) else None + if isinstance(value, str): + digests.add(value) + return digests + + +def _check_attestations( + directory: Path, names: dict[str, str], version: str, problems: list[str] +) -> None: + path = directory / f"{STEM}-{version}.intoto.jsonl" + statements = _statements(path, problems) + provenance = [s for s in statements if s.get("predicateType") == SLSA_PROVENANCE] + sboms = [s for s in statements if s.get("predicateType") in SBOM_PREDICATES] + if not provenance: + problems.append(f"NO-PROVENANCE {path.name} holds no {SLSA_PROVENANCE} statement") + if not sboms: + problems.append(f"NO-SBOM-ATTESTATION {path.name} holds no SBOM statement") + attested = set().union(*(_subject_digests(s) for s in provenance)) + for name, kind in sorted(names.items()): + if kind != "attestations" and _sha256(directory / name) not in attested: + problems.append(f"NO-PROVENANCE {name} is not a subject of the provenance") + sbom_subjects = set().union(*(_subject_digests(s) for s in sboms)) + for name, kind in sorted(names.items()): + if kind in {"wheel", "sdist"} and _sha256(directory / name) not in sbom_subjects: + problems.append(f"NO-SBOM-ATTESTATION {name} is not a subject of an SBOM statement") + + +def verify(directory: Path, version: str, *, attestations: bool) -> tuple[int, list[str]]: + """Return the exit code and every finding for the release staged in `directory`.""" + if not directory.is_dir() or not any(directory.iterdir()): + return 2, [f"EMPTY {directory} holds no release assets: the staging step produced nothing"] + names = expected_names(version, attestations=attestations) + present = {p.name for p in directory.iterdir() if p.is_file()} + problems = [ + f"UNLISTED {name} is not a release asset" for name in sorted(present - names.keys()) + ] + problems += [f"MISSING {name}" for name in sorted(names.keys() - present)] + if problems: + return 1, problems + _check_sums(directory, names, problems) + _check_cyclonedx(directory / f"{STEM}-{version}.cdx.json", version, problems) + _check_spdx(directory / f"{STEM}-{version}.spdx.json", version, problems) + if attestations: + _check_attestations(directory, names, version, problems) + return (1 if problems else 0), problems + + +def main(argv: Sequence[str]) -> int: + """Entry point. Returns the process exit code; 2 also means the ARGUMENTS were wrong.""" + parser = argparse.ArgumentParser( + prog="scripts/verify_release_assets.py", + description="Check a staged release directory before it is uploaded.", + ) + parser.add_argument("directory", help="the staged release directory") + parser.add_argument("version", help="the release version, without the leading v") + parser.add_argument( + "--attestations", + action="store_true", + help="also require the .intoto.jsonl and check what its statements cover", + ) + args = parser.parse_args(argv) + version = str(args.version).removeprefix("v") + directory = Path(args.directory) + code, problems = verify(directory, version, attestations=bool(args.attestations)) + for problem in problems: + sys.stderr.write(f"release {version}: {problem}\n") + if code == 0: + count = len(expected_names(version, attestations=bool(args.attestations))) + checked = "assets, attestations covered" if args.attestations else "assets" + sys.stdout.write(f"release {version}: {count} {checked}, 0 findings\n") + return code + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 32a494fe1efa11cd98bb6ba4183c2f3c12168b74 Mon Sep 17 00:00:00 2001 From: Laith Al-Saadoon Date: Sat, 10 Oct 2026 17:42:19 +0000 Subject: [PATCH 3/6] ci(fuzz): fuzz the transcript converters with Atheris The Claude Code and Codex converters parse transcripts nobody here wrote, and until now only hand-written fixtures exercised that path. Two Atheris harnesses under fuzz/ feed arbitrary bytes, laid out as a session or a rollout on disk, through convert_and_audit and convert_codex_and_audit, the same use cases materialize runs. A DomainError is the converter's documented rejection; any other exception, or a successful conversion whose artifacts the corpus writer could not store, is a finding. The target logic lives in fuzz/transcript_targets.py with no atheris import, so mise run check replays the committed seed corpus through it on every machine. mise run fuzz runs each harness for ATIF_SQL_FUZZ_SECONDS (default 60) and fails on a crash, a timeout, an out-of-memory input, a missing harness, seeds or dictionary, or a run that executed no input. atheris ships only manylinux x86_64 wheels, so it sits in an optional fuzz dependency group behind a platform marker: uv sync on macOS and the universal lock are unchanged. fuzz.yml runs the task weekly, on demand, and on pull requests that change the converter or fuzz/, with the action pins and least-privilege settings the other workflows use, and uploads any crash input on failure. Co-Authored-By: Claude Opus 5.5 --- .editorconfig | 6 + .github/workflows/fuzz.yml | 74 +++++ .gitignore | 3 + .../config/vocabularies/atif-sql/accept.txt | 1 + AGENTS.md | 6 + CONTRIBUTING.md | 26 ++ fuzz/claude_code.dict | 251 ++++++++++++++++ fuzz/codex.dict | 275 ++++++++++++++++++ .../parallel-subagents-image-sidecar | Bin 0 -> 5347 bytes .../corpus/claude_code/tool-call-and-subagent | Bin 0 -> 2489 bytes fuzz/corpus/claude_code/user-and-assistant | 2 + fuzz/corpus/codex/meta-context-messages | 5 + fuzz/corpus/codex/rollout | 18 ++ fuzz/fuzz_claude_code.py | 34 +++ fuzz/fuzz_codex.py | 34 +++ fuzz/run_fuzzers.py | 229 +++++++++++++++ fuzz/test_fuzz_seed_corpus.py | 112 +++++++ fuzz/test_run_fuzzers.py | 175 +++++++++++ fuzz/transcript_targets.py | 156 ++++++++++ mise.toml | 31 +- pyproject.toml | 28 +- uv.lock | 13 + 22 files changed, 1474 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/fuzz.yml create mode 100644 fuzz/claude_code.dict create mode 100644 fuzz/codex.dict create mode 100644 fuzz/corpus/claude_code/parallel-subagents-image-sidecar create mode 100644 fuzz/corpus/claude_code/tool-call-and-subagent create mode 100644 fuzz/corpus/claude_code/user-and-assistant create mode 100644 fuzz/corpus/codex/meta-context-messages create mode 100644 fuzz/corpus/codex/rollout create mode 100644 fuzz/fuzz_claude_code.py create mode 100644 fuzz/fuzz_codex.py create mode 100644 fuzz/run_fuzzers.py create mode 100644 fuzz/test_fuzz_seed_corpus.py create mode 100644 fuzz/test_run_fuzzers.py create mode 100644 fuzz/transcript_targets.py diff --git a/.editorconfig b/.editorconfig index ad5ff32..857c06f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -51,3 +51,9 @@ indent_size = 2 [uv.lock] insert_final_newline = unset trim_trailing_whitespace = unset + +# Fuzzer seed inputs are bytes the harness reads verbatim, NUL separators included. +[fuzz/corpus/**] +charset = unset +insert_final_newline = unset +trim_trailing_whitespace = unset diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 0000000..bd8c21a --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,74 @@ +# Coverage-guided fuzzing of the two transcript converters with Atheris (libFuzzer for +# Python). +# +# The converters read files nobody here wrote: every Claude Code session and Codex rollout on +# the machine. `mise run check` replays the committed seeds through the same targets on every +# push; this job mutates them for a bounded time and fails on any exception that is not the +# converter's own rejection (`DomainError`), on a successful conversion the corpus could not +# write, on a timeout or an out-of-memory input, and on a run that executed no input. +# fuzz/run_fuzzers.py says which is which, and CONTRIBUTING.md how to reproduce a finding. +# +# Weekly as well as on pull requests that touch the converter or the harnesses, because a +# fuzzer finds different inputs on every run: the scheduled run is the one that keeps looking +# when the code does not move. Thursday, offset from the Tuesday and Wednesday scheduled jobs +# so they do not contend for the same runner minute. +# +# No secret and no cloud credential: the harnesses convert bytes in a temporary directory and +# call nothing outside the process. +name: fuzz +on: + pull_request: + paths: + - "packages/atif-converter/**" + - "fuzz/**" + - ".github/workflows/fuzz.yml" + schedule: + - cron: "41 6 * * 4" + workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +permissions: + contents: read +jobs: + fuzz: + name: fuzz the transcript converters + runs-on: ubuntu-latest + # Two targets at 60 s each plus a cold toolchain install; a hang is caught per input by + # libFuzzer's own `-timeout`, so this bound only stops a runner that wedged outright. + timeout-minutes: 30 + permissions: + contents: read # check out the tree the harnesses fuzz + steps: + # `persist-credentials: false`: nothing in this job pushes, and a token left in + # .git/config is readable by every subsequent step, the fuzzed code included. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # `cache: false`, for the reason security.yml gives: a cached tool directory is a tool + # nobody verified, and a cached pipx shebang can point at an interpreter the cache lacks. + - uses: jdx/mise-action@9149ea85001c7435d5a66bb127d6a1b6227cb0a5 # v5.0.0 + with: + # `minimum_release_age` because a mise release exists on GitHub before its assets finish + # uploading: the 2026-09-08 scheduled security run asked for the just-cut v2026.9.3, its + # tarball 404'd five times, and every scanner was skipped. A day-old release has its + # assets. `version` stays unpinned so the runner keeps following mise, gated by that day. + minimum_release_age: 24h + cache: false + - run: mise run lock:check + - run: mise run install:ci + # `UV_LOCKED` makes the `uv run --group fuzz` inside the task install atheris from uv.lock + # as locked, and fail rather than re-resolve when the lock is stale. + - run: mise run fuzz + env: + UV_LOCKED: "1" + # Only on failure: the inputs that crashed, timed out or ran out of memory, and each + # target's whole libFuzzer log, so a finding reproduces from the run that found it. + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: failure() + with: + name: fuzz-findings + path: | + fuzz/out/*/artifacts/ + fuzz/out/*/fuzz.log + retention-days: 30 diff --git a/.gitignore b/.gitignore index 99a61a2..6b4300f 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,6 @@ docs/.packets/ # lychee request cache; links.yml carries it between runs via actions/cache .lycheecache + +# Atheris fuzz runs: the inputs libFuzzer kept, crash artifacts and logs (mise run fuzz). +fuzz/out/ diff --git a/.vale/styles/config/vocabularies/atif-sql/accept.txt b/.vale/styles/config/vocabularies/atif-sql/accept.txt index 7bb6cdb..8b019ba 100644 --- a/.vale/styles/config/vocabularies/atif-sql/accept.txt +++ b/.vale/styles/config/vocabularies/atif-sql/accept.txt @@ -35,6 +35,7 @@ Bedrock Cohere(?:'s)? commitizen(?:'s)? lefthook +[Aa]theris anyio fastapi starlette diff --git a/AGENTS.md b/AGENTS.md index d0cde99..09b4017 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -426,6 +426,12 @@ relative `.md` path as GitHub renders it; the build turns each link into the pag the GitHub blob for a Markdown file the site does not publish. First-time setup: `mise trust && mise install && mise run install`. +Fuzzing sits beside the gate, not in it: `mise run fuzz` runs the Atheris harnesses in `fuzz/` over +both transcript converters for `ATIF_SQL_FUZZ_SECONDS` each (default 60), on Linux x86_64 only, +and fails on any exception other than the converter's `DomainError`. `check` replays the seeds +in `fuzz/corpus/` through the same targets. CONTRIBUTING.md has the contract and the steps +for a finding. + ## Experiments `experiments/` holds numbered experiment protocols (README per experiment). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c0ffbb..c2fbf26 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,6 +76,32 @@ and the pre-push hook runs its link crawl when a push touches either tree. `live-links` job in `docs.yml` runs it after every deploy and every Monday, and fails on any internal link that does not answer 200. +## Fuzzing: `mise run fuzz` + +The two transcript converters read files nobody here wrote, so both are fuzzed with +[Atheris](https://github.com/google/atheris). `fuzz/transcript_targets.py` holds the targets: each +writes one input to disk as a Claude Code session or a Codex rollout and runs the use case +materialize runs (`convert_and_audit`, `convert_codex_and_audit`). A `DomainError` is the +converter's documented rejection of bad input; any other exception is a finding, and so is a +successful conversion whose artifacts the corpus writer could not store. + +| Task | Command | What fails you | +| --- | --- | --- | +| `fuzz` | `uv run --group fuzz python fuzz/run_fuzzers.py` | a crash, a timeout or an out-of-memory input in either target; a target missing its harness, seeds or dictionary; atheris absent; or a run that executed no input | +| `fuzz:claude-code`, `fuzz:codex` | the same, for one target | the same, for that target | + +`ATIF_SQL_FUZZ_SECONDS` sets each target's time bound (default 60). atheris sits in the optional +`fuzz` dependency group, which only installs on Linux x86_64 (it publishes no other wheel), so +`fuzz` stays out of `check`. `check` still replays the committed seeds in `fuzz/corpus/` through +the same targets (`fuzz/test_fuzz_seed_corpus.py`), and `.github/workflows/fuzz.yml` runs `fuzz` +weekly and on pull requests that change the converter or `fuzz/`. + +A finding lands in `fuzz/out//artifacts/` (CI uploads it as `fuzz-findings`). Replay it +with `uv run --group fuzz python fuzz/fuzz_.py `, fix the converter, and add +the input to `fuzz/corpus//` so `check` keeps it fixed. A new target is a function in +`TARGETS`, a `fuzz_.py` harness, a `fuzz/corpus//` seed directory and a +`fuzz/.dict` dictionary; `fuzz` fails until all four exist. + Never reach green by relaxing a gate. Widening an `ignore` list, adding a blanket per-file ignore, or deleting a contract is a change to the project's standards, not a fix to your branch. Raise it in the pull request instead. diff --git a/fuzz/claude_code.dict b/fuzz/claude_code.dict new file mode 100644 index 0000000..2bd6143 --- /dev/null +++ b/fuzz/claude_code.dict @@ -0,0 +1,251 @@ +# SPDX-License-Identifier: Apache-2.0 +# libFuzzer dictionary for fuzz_claude_code.py: JSON structure, then the record keys and +# values the Claude Code transcript converter branches on (string constants in its modules +# and the keys of the seed corpus), each as a quoted JSON string. +"{" +"}" +"[" +"]" +":" +"," +"\"" +"null" +"true" +"false" +"0" +"-1" +"1e309" +"\\u0000" +"\x0a" +"\x00" +"\"ATIF-v1.7\"" +"\"Bash\"" +"\"Err\"" +"\"Task\"" +"\"agent\"" +"\"agentId\"" +"\"agentType\"" +"\"agent_id\"" +"\"agent_ids\"" +"\"agent_step\"" +"\"agent_type\"" +"\"analysis\"" +"\"api_error\"" +"\"application/octet-stream\"" +"\"application/pdf\"" +"\"arguments\"" +"\"assistant\"" +"\"attachment\"" +"\"base64\"" +"\"bin\"" +"\"blob\"" +"\"bytes\"" +"\"cache_creation_input_tokens\"" +"\"cache_creation_total\"" +"\"cache_read_input_tokens\"" +"\"cache_split_partial\"" +"\"call_id\"" +"\"claude_code_cost_state\"" +"\"code\"" +"\"command\"" +"\"compact_boundary\"" +"\"compact_summary_unhandled\"" +"\"compacted\"" +"\"completed\"" +"\"content\"" +"\"context_compacted\"" +"\"cost-state\"" +"\"cost_source\"" +"\"custom_tool_call\"" +"\"custom_tool_call_output\"" +"\"cwd\"" +"\"cwds\"" +"\"data\"" +"\"data:\"" +"\"description\"" +"\"dimensions\"" +"\"document\"" +"\"enrichment_leftover_groups\"" +"\"enrichment_truncated_at_step\"" +"\"enrichment_unattributed_steps\"" +"\"entrypoint\"" +"\"error\"" +"\"event_msg\"" +"\"event_type\"" +"\"exec\"" +"\"exitCode\"" +"\"exit_code\"" +"\"external\"" +"\"extra\"" +"\"failed\"" +"\"file\"" +"\"file_path\"" +"\"final_metrics\"" +"\"function_call\"" +"\"function_call_output\"" +"\"gaps_observed\"" +"\"gif\"" +"\"gitBranch\"" +"\"git_branches\"" +"\"hasUnknownModelCost\"" +"\"height\"" +"\"hook_\"" +"\"id\"" +"\"ignore\"" +"\"image\"" +"\"image/\"" +"\"image/gif\"" +"\"image/jpeg\"" +"\"image/jpg\"" +"\"image/png\"" +"\"image/webp\"" +"\"image_url\"" +"\"images\"" +"\"input\"" +"\"input_image\"" +"\"input_text\"" +"\"input_tokens\"" +"\"interrupted\"" +"\"isCompactSummary\"" +"\"isError\"" +"\"isImage\"" +"\"isMeta\"" +"\"isSidechain\"" +"\"is_compact_summary\"" +"\"is_error\"" +"\"is_sidechain\"" +"\"item_completed\"" +"\"jpg\"" +"\"kind\"" +"\"last-prompt\"" +"\"link_source\"" +"\"media_type\"" +"\"message\"" +"\"message_id\"" +"\"meta\"" +"\"metadata\"" +"\"metrics\"" +"\"mimeType\"" +"\"mime_type\"" +"\"mode\"" +"\"model\"" +"\"model_name\"" +"\"model_refusal_fallback\"" +"\"name\"" +"\"noOutputExpected\"" +"\"non_message_records_dropped\"" +"\"observation\"" +"\"openrouter.reasoning:\"" +"\"operation\"" +"\"originalHeight\"" +"\"originalSize\"" +"\"originalWidth\"" +"\"other\"" +"\"output_tokens\"" +"\"parentAgentId\"" +"\"parentUuid\"" +"\"parent_agent_id\"" +"\"parent_chain_flattened\"" +"\"parent_tool_call_id\"" +"\"parent_uuid\"" +"\"path\"" +"\"payload\"" +"\"payload_bytes\"" +"\"payload_truncated\"" +"\"pdf\"" +"\"permission-mode\"" +"\"png\"" +"\"pr-link\"" +"\"prompt\"" +"\"queue-operation\"" +"\"queued_command\"" +"\"raw_arguments\"" +"\"raw_tool_result\"" +"\"reasoning\"" +"\"record_counts\"" +"\"records_captured\"" +"\"records_converted\"" +"\"records_dropped\"" +"\"redacted_thinking\"" +"\"rendered\"" +"\"replace\"" +"\"reported_cost_has_unknown_model_cost\"" +"\"reported_cost_source\"" +"\"reported_cost_usd\"" +"\"requestId\"" +"\"response_item\"" +"\"result\"" +"\"result_extra\"" +"\"results\"" +"\"role\"" +"\"running\"" +"\"seq\"" +"\"service_tier\"" +"\"service_tiers\"" +"\"sessionId\"" +"\"sha256\"" +"\"slug\"" +"\"source\"" +"\"source_call_id\"" +"\"source_file\"" +"\"source_uuids\"" +"\"spawnDepth\"" +"\"spawn_depth\"" +"\"started\"" +"\"status\"" +"\"stderr\"" +"\"stdout\"" +"\"step_id\"" +"\"steps\"" +"\"stop_hook_summary\"" +"\"stop_reason\"" +"\"stop_sequence\"" +"\"stream_error\"" +"\"subagent_files_convertible\"" +"\"subagent_files_found\"" +"\"subagent_type\"" +"\"subagents\"" +"\"subagents_inlined\"" +"\"subtype\"" +"\"summary\"" +"\"system\"" +"\"text\"" +"\"thinking\"" +"\"timestamp\"" +"\"toolUseID\"" +"\"toolUseId\"" +"\"toolUseResult\"" +"\"tool_call\"" +"\"tool_call_id\"" +"\"tool_calls\"" +"\"tool_name\"" +"\"tool_result\"" +"\"tool_result_is_error\"" +"\"tool_result_metadata\"" +"\"tool_use\"" +"\"tool_use_id\"" +"\"tool_use_ids\"" +"\"tool_use_is_error\"" +"\"tool_use_name\"" +"\"tool_use_result\"" +"\"totalCostUSD\"" +"\"total_cache_creation_input_tokens\"" +"\"total_cache_read_input_tokens\"" +"\"truncated_preview\"" +"\"ts\"" +"\"turn_aborted\"" +"\"turn_id\"" +"\"type\"" +"\"unknown\"" +"\"usage\"" +"\"user\"" +"\"userType\"" +"\"user_type\"" +"\"utf-8\"" +"\"uuid\"" +"\"uuid_not_preserved\"" +"\"version\"" +"\"warning\"" +"\"webp\"" +"\"width\"" +"\"workflow_subagent_files_found\"" diff --git a/fuzz/codex.dict b/fuzz/codex.dict new file mode 100644 index 0000000..9f077e8 --- /dev/null +++ b/fuzz/codex.dict @@ -0,0 +1,275 @@ +# SPDX-License-Identifier: Apache-2.0 +# libFuzzer dictionary for fuzz_codex.py: JSON structure, then the record keys and +# values the Codex rollout converter branches on (string constants in its modules +# and the keys of the seed corpus), each as a quoted JSON string. +"{" +"}" +"[" +"]" +":" +"," +"\"" +"null" +"true" +"false" +"0" +"-1" +"1e309" +"\\u0000" +"\x0a" +"\x00" +"\"ATIF-v1.7\"" +"\"Bash\"" +"\"Err\"" +"\"Task\"" +"\"action\"" +"\"action_type\"" +"\"agent\"" +"\"agentId\"" +"\"agentType\"" +"\"agent_id\"" +"\"agent_type\"" +"\"aggregated_output\"" +"\"api_call_\"" +"\"api_call_1\"" +"\"api_call_id\"" +"\"api_error\"" +"\"application/octet-stream\"" +"\"application/pdf\"" +"\"arguments\"" +"\"assistant\"" +"\"attachment\"" +"\"audio\"" +"\"base64\"" +"\"bin\"" +"\"blob\"" +"\"branch\"" +"\"bundled\"" +"\"bytes\"" +"\"cache_creation_total\"" +"\"cache_write_input_tokens\"" +"\"cached_input_tokens\"" +"\"cached_tokens\"" +"\"call_id\"" +"\"claude_code_cost_state\"" +"\"cli_version\"" +"\"cmd\"" +"\"codex_compaction_count\"" +"\"codex_compaction_unhandled\"" +"\"codex_developer_role_flattened_to_system\"" +"\"codex_item_ids_not_preserved\"" +"\"codex_non_item_records_dropped\"" +"\"codex_reasoning_encrypted_dropped\"" +"\"codex_tool_search_calls_dropped\"" +"\"codex_turn_id\"" +"\"command\"" +"\"commit_hash\"" +"\"compact_boundary\"" +"\"compacted\"" +"\"completed\"" +"\"completed_at_ms\"" +"\"completion_tokens\"" +"\"content\"" +"\"context_compacted\"" +"\"cost-state\"" +"\"cost_usd\"" +"\"custom_tool_call\"" +"\"custom_tool_call_output\"" +"\"cwd\"" +"\"data\"" +"\"data:\"" +"\"description\"" +"\"developer\"" +"\"document\"" +"\"duration\"" +"\"duration_ms\"" +"\"encrypted_content\"" +"\"enrichment_leftover_messages\"" +"\"enrichment_truncated_at_step\"" +"\"enrichment_unattributed_steps\"" +"\"entrypoint\"" +"\"error\"" +"\"event\"" +"\"event:\"" +"\"event_msg\"" +"\"event_type\"" +"\"exec\"" +"\"exec_command\"" +"\"exit_code\"" +"\"extra\"" +"\"failed\"" +"\"file\"" +"\"final_metrics\"" +"\"full\"" +"\"function\"" +"\"function_call\"" +"\"function_call_output\"" +"\"function_name\"" +"\"gif\"" +"\"git\"" +"\"gitBranch\"" +"\"hasUnknownModelCost\"" +"\"height\"" +"\"hook_\"" +"\"id\"" +"\"ignore\"" +"\"image\"" +"\"image/\"" +"\"image/gif\"" +"\"image/jpeg\"" +"\"image/jpg\"" +"\"image/png\"" +"\"image/webp\"" +"\"image_url\"" +"\"images\"" +"\"info\"" +"\"input\"" +"\"input_audio\"" +"\"input_image\"" +"\"input_text\"" +"\"input_tokens\"" +"\"instructions\"" +"\"interrupted\"" +"\"isError\"" +"\"isMeta\"" +"\"isSidechain\"" +"\"is_compact_summary\"" +"\"is_error\"" +"\"is_sidechain\"" +"\"item_completed\"" +"\"item_type\"" +"\"jpg\"" +"\"kind\"" +"\"last_token_usage\"" +"\"link_source\"" +"\"mcp__\"" +"\"media_type\"" +"\"message\"" +"\"message_id\"" +"\"message_parts\"" +"\"meta\"" +"\"metadata\"" +"\"metrics\"" +"\"mimeType\"" +"\"mime_type\"" +"\"mode\"" +"\"model\"" +"\"model_context_window\"" +"\"model_refusal_fallback\"" +"\"name\"" +"\"nanos\"" +"\"nested_in\"" +"\"observation\"" +"\"ordinal\"" +"\"originator\"" +"\"other\"" +"\"output_text\"" +"\"output_tokens\"" +"\"parentAgentId\"" +"\"parentUuid\"" +"\"parent_agent_id\"" +"\"parent_tool_call_id\"" +"\"parent_uuid\"" +"\"payload\"" +"\"payload_bytes\"" +"\"payload_truncated\"" +"\"pdf\"" +"\"permission-mode\"" +"\"png\"" +"\"prompt_tokens\"" +"\"queries\"" +"\"query\"" +"\"queued_command\"" +"\"raw_arguments\"" +"\"reasoning\"" +"\"reasoning_content\"" +"\"reasoning_output_tokens\"" +"\"rendered\"" +"\"replacement_history\"" +"\"reported_cost_has_unknown_model_cost\"" +"\"reported_cost_source\"" +"\"reported_cost_usd\"" +"\"resource\"" +"\"resource_link\"" +"\"response_item\"" +"\"result\"" +"\"results\"" +"\"role\"" +"\"running\"" +"\"secs\"" +"\"seq\"" +"\"server\"" +"\"sessionId\"" +"\"session_id\"" +"\"session_meta\"" +"\"sha256\"" +"\"slug\"" +"\"source\"" +"\"source_call_id\"" +"\"source_file\"" +"\"source_uuids\"" +"\"spawnDepth\"" +"\"spawn_depth\"" +"\"started_at_ms\"" +"\"state\"" +"\"status\"" +"\"step_id\"" +"\"steps\"" +"\"stop_hook_summary\"" +"\"stream_error\"" +"\"subagents\"" +"\"subtype\"" +"\"summary\"" +"\"system\"" +"\"task_complete\"" +"\"task_started\"" +"\"text\"" +"\"timestamp\"" +"\"token_count\"" +"\"token_usage_record\"" +"\"tool\"" +"\"toolUseID\"" +"\"toolUseId\"" +"\"toolUseResult\"" +"\"tool_call\"" +"\"tool_call_details\"" +"\"tool_call_id\"" +"\"tool_calls\"" +"\"tool_metadata\"" +"\"tool_name\"" +"\"tool_order\"" +"\"tool_result\"" +"\"tool_search_call\"" +"\"tool_search_output\"" +"\"tool_use\"" +"\"tool_use_id\"" +"\"tool_use_ids\"" +"\"tools\"" +"\"totalCostUSD\"" +"\"total_cache_write_input_tokens\"" +"\"total_cost\"" +"\"total_token_usage\"" +"\"total_tokens\"" +"\"truncated_preview\"" +"\"ts\"" +"\"turn_aborted\"" +"\"turn_complete\"" +"\"turn_context\"" +"\"turn_id\"" +"\"turn_started\"" +"\"type\"" +"\"unknown\"" +"\"url\"" +"\"usage\"" +"\"user\"" +"\"userType\"" +"\"utf-8\"" +"\"uuid\"" +"\"value\"" +"\"version\"" +"\"via\"" +"\"warning\"" +"\"web_search_call\"" +"\"webp\"" +"\"width\"" +"\"world_state\"" diff --git a/fuzz/corpus/claude_code/parallel-subagents-image-sidecar b/fuzz/corpus/claude_code/parallel-subagents-image-sidecar new file mode 100644 index 0000000000000000000000000000000000000000..78a38f0b92785d4cb069be007ab6e0332403e7ba GIT binary patch literal 5347 zcmcgwQFGcj5WeU86&}A{NgR`;*Ua35Atd2CO(qc1-raOO!d8tYw&by-(3au9->z(9 zY?@36I1nI~C3$zh-EVibu(f8PeP-FZb7a4>Y)PeLf)CA@Tf92>+xRN);UdKnDiujm z+{ks@(~k3@}yH!dHFMN zMEf2NaFQ59dl8x!XYf>`oflx}DZs&%Uv4z`p2i|G`-D^m8B0}xyBDW3V#z?jQrIKS zM3Ne?Vj&(ZqAY?F0H}AcZ@Jx5a`pBnSC6V^#hSu>km@VQtXIBgEZ{5gK)D1%bm1oG z1y-rX+-1$@bjL-RTL>}N+0Eww=!v8wDs^0ds)*94mK^{_F;z4~(GAc!HxNQrBX@-h zmB<5rEBL^pwg}s%KW4c|xL@I({)TV<+MDTj<6{bcK@HK!E^}NN};nLPmCpbPKl(a#!?9l8Ch|(S#y~32-k%K=nZ0o-S=)rbLZn_Bs_Rs zk0-OiB*K3u_%-NzL-;xWd-8es1qaRt@!)WK?NBXP<9chmYyStHLL1y(_x>j^F60qU)HWP3kF)9;qkH{HbSNRtRIw9OtQ7Y3w zZ+pm7C&r!jUkbX%yFv2{J`Ay#0OvIltB__{dBva>%rc!~?mDx+kyuhRoE(~XsYbZC zF?k!Z7Y*{sXvjC?LxXR{hei`7eUM$CE=KT;;At&mo$Q9tmce7x*h}y zo4kPLN&8LKsMckTC^6-BMe^TRO#PH7(-|Mm53ve3!1TR4+oOB8)t5N$cVd&5Lb;I^ zN{3&SMrc2XkcC|oM9)?`pRji6=hcokzIeR2HUmyo?aaMfD0blUrTfQ~KE~VLcyfO; z`gqd+XE^)|lTl5EveXh8ix?;InEga~PXC07A)KCYJQ~4%YhjmDKC{rkZ5qpB*%f8c z4AUKUmTS-0F0i+3vU1<764cuAm3`YQJ@)_xmY((E6X~lppa1YCKL7GWOZ@$ns2f=p x4|E8Zb7=BkS$9IFH5u!!afpDC@$B&to(N1N&5tA2&Z=Ze=KP$dCVGyR{{y;x9aI1S literal 0 HcmV?d00001 diff --git a/fuzz/corpus/claude_code/tool-call-and-subagent b/fuzz/corpus/claude_code/tool-call-and-subagent new file mode 100644 index 0000000000000000000000000000000000000000..a5721bbafdc5695223f502b55fd505c62e627646 GIT binary patch literal 2489 zcmcIm!EW0y4BdHOVfb99j=Ka!kG<`>t(Ogjq1kM#)s|d|S~~>!_mNbb#A%vlz!C&; zL{W_R`1qJ*ihp`YZ;~{4aOnj-gRW(rO?lZX2L|t=(S)`on+NZ;HJ`(o&s$rL#lQ>f>^Qk)pgQ2RcE42cXmepwbYH=Rm3uqz)(R;3*Kwu`;wM^IWR93qmY!=NDz% z8$cN)*V!6c#-(ais3Aj&8dKV0_G1eSHrhODvtV2{-|2JN2ft*C`_hh!{3$G!^uo zF6q(srK4g;4so<0e%R1ObDR?GBN$KNra2w70Y}Ce*(y~vz|rdR)e$}u_bYNh;_SZ9 zEMJXC_PP-ZPHNPM>6&!s;Mdg=elbpW9=?DNZA92(O^`?jX}%_{!8u;XN+7L5kH9sM zmLwLO*ps{b;`+Gwt8{)SR~fmy%Y`jHQ!e)X2wH_mrRuolDaxjSaZT-aBuse them"}]}} +{"timestamp": "2026-09-11T17:27:01.500Z", "type": "response_item", "payload": {"type": "message", "id": "msg_user_1", "role": "user", "content": [{"type": "input_text", "text": "read marker.txt with a shell command"}]}} +{"timestamp": "2026-09-11T17:27:02.000Z", "type": "response_item", "payload": {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "rsn_opaque"}} diff --git a/fuzz/corpus/codex/rollout b/fuzz/corpus/codex/rollout new file mode 100644 index 0000000..a7d946e --- /dev/null +++ b/fuzz/corpus/codex/rollout @@ -0,0 +1,18 @@ +{"timestamp": "2026-09-11T17:27:01.000Z", "type": "session_meta", "payload": {"id": "01a09182-2858-7f42-936b-7f027b341fdf", "session_id": "01a09182-2858-7f42-936b-7f027b341fdf", "timestamp": "2026-09-11T17:27:01.000Z", "cwd": "/home/alice/proj", "originator": "codex-tui", "cli_version": "0.154.0", "source": "cli", "git": {"branch": "main", "commit_hash": "abc1234"}}} +{"timestamp": "2026-09-11T17:27:01.100Z", "type": "event_msg", "payload": {"type": "task_started", "turn_id": "turn-1"}} +{"timestamp": "2026-09-11T17:27:01.200Z", "type": "turn_context", "payload": {"turn_id": "turn-1", "cwd": "/home/alice/proj", "model": "openai/gpt-5.1-codex"}} +{"timestamp": "2026-09-11T17:27:01.300Z", "type": "world_state", "payload": {"full": true, "state": {"model": "openai/gpt-5.1-codex"}}} +{"timestamp": "2026-09-11T17:27:01.400Z", "type": "response_item", "payload": {"type": "message", "id": "msg_dev_1", "role": "developer", "content": [{"type": "input_text", "text": "use them"}]}} +{"timestamp": "2026-09-11T17:27:01.500Z", "type": "response_item", "payload": {"type": "message", "id": "msg_user_1", "role": "user", "content": [{"type": "input_text", "text": "read marker.txt with a shell command"}]}} +{"timestamp": "2026-09-11T17:27:02.000Z", "type": "response_item", "payload": {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "rsn_opaque"}} +{"timestamp": "2026-09-11T17:27:02.100Z", "type": "response_item", "payload": {"type": "message", "id": "msg_agent_empty", "role": "assistant", "content": [{"type": "output_text", "text": ""}]}} +{"timestamp": "2026-09-11T17:27:02.200Z", "type": "response_item", "payload": {"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": "exec_command", "arguments": "{\"cmd\":\"cat marker.txt\"}", "status": "completed"}} +{"timestamp": "2026-09-11T17:27:02.300Z", "type": "response_item", "payload": {"type": "function_call_output", "call_id": "call_1", "output": "{\"output\":\"marker-ok\",\"metadata\":{\"exit_code\":0}}"}} +{"timestamp": "2026-09-11T17:27:02.400Z", "type": "response_item", "payload": {"type": "tool_search_call", "id": "tsc_1", "call_id": "call_search_1", "status": "completed", "arguments": {"query": "outlook email tools"}}} +{"timestamp": "2026-09-11T17:27:02.500Z", "type": "response_item", "payload": {"type": "tool_search_output", "call_id": "call_search_1", "status": "completed", "tools": [{"type": "function", "name": "email_read"}]}} +{"timestamp": "2026-09-11T17:27:02.600Z", "type": "event_msg", "payload": {"type": "token_count", "info": {"total_token_usage": {"input_tokens": 1000, "cached_input_tokens": 0, "cache_write_input_tokens": 900, "output_tokens": 40, "reasoning_output_tokens": 0, "total_tokens": 1040}, "last_token_usage": {"input_tokens": 1000, "cached_input_tokens": 0, "cache_write_input_tokens": 900, "output_tokens": 40, "reasoning_output_tokens": 0, "total_tokens": 1040}, "model_context_window": 258400}}} +{"timestamp": "2026-09-11T17:27:02.700Z", "type": "token_usage_record", "payload": {"turn_id": "turn-1", "usage": {"input_tokens": 1000}}} +{"timestamp": "2026-09-11T17:27:03.000Z", "type": "response_item", "payload": {"type": "message", "id": "msg_agent_final", "role": "assistant", "content": [{"type": "output_text", "text": "marker-ok"}]}} +{"timestamp": "2026-09-11T17:27:03.100Z", "type": "event_msg", "payload": {"type": "token_count", "info": {"total_token_usage": {"input_tokens": 1100, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 8, "reasoning_output_tokens": 0, "total_tokens": 1108}, "last_token_usage": {"input_tokens": 1100, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 8, "reasoning_output_tokens": 0, "total_tokens": 1108}, "model_context_window": 258400}}} +{"timestamp": "2026-09-11T17:27:03.200Z", "type": "compacted", "payload": {"message": "summary of the work so far", "replacement_history": []}} +{"timestamp": "2026-09-11T17:27:03.300Z", "type": "event_msg", "payload": {"type": "task_complete", "turn_id": "turn-1"}} diff --git a/fuzz/fuzz_claude_code.py b/fuzz/fuzz_claude_code.py new file mode 100644 index 0000000..c804078 --- /dev/null +++ b/fuzz/fuzz_claude_code.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Atheris harness: arbitrary bytes as a Claude Code session, through ``convert_and_audit``. + + uv run --group fuzz python fuzz/fuzz_claude_code.py [libFuzzer flags] [CORPUS_DIR ...] + +``mise run fuzz:claude-code`` runs it through ``run_fuzzers.py`` with a time bound, +the seed corpus and the dictionary. The target and its contract live in +``transcript_targets.claude_code_one_input``. +""" + +from __future__ import annotations + +import sys + +# atheris comes from the `fuzz` group (linux x86_64); without it pyright reads typeshed's stub. +import atheris # pyright: ignore[reportMissingModuleSource] +from loguru import logger + +# typeshed's atheris stub does not re-export `instrument_imports`, which atheris itself does. +with atheris.instrument_imports( # pyright: ignore[reportAttributeAccessIssue] + include=["atif_converter", "transcript_targets"] +): + import transcript_targets + + +def main() -> None: + """Silence the converter's debug log, then hand the target to libFuzzer.""" + logger.remove() + atheris.Setup(sys.argv, transcript_targets.claude_code_one_input) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzz/fuzz_codex.py b/fuzz/fuzz_codex.py new file mode 100644 index 0000000..14fd18a --- /dev/null +++ b/fuzz/fuzz_codex.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Atheris harness: arbitrary bytes as a Codex rollout, through ``convert_codex_and_audit``. + + uv run --group fuzz python fuzz/fuzz_codex.py [libFuzzer flags] [CORPUS_DIR ...] + +``mise run fuzz:codex`` runs it through ``run_fuzzers.py`` with a time bound, the +seed corpus and the dictionary. The target and its contract live in +``transcript_targets.codex_one_input``. +""" + +from __future__ import annotations + +import sys + +# atheris comes from the `fuzz` group (linux x86_64); without it pyright reads typeshed's stub. +import atheris # pyright: ignore[reportMissingModuleSource] +from loguru import logger + +# typeshed's atheris stub does not re-export `instrument_imports`, which atheris itself does. +with atheris.instrument_imports( # pyright: ignore[reportAttributeAccessIssue] + include=["atif_converter", "transcript_targets"] +): + import transcript_targets + + +def main() -> None: + """Silence the converter's debug log, then hand the target to libFuzzer.""" + logger.remove() + atheris.Setup(sys.argv, transcript_targets.codex_one_input) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzz/run_fuzzers.py b/fuzz/run_fuzzers.py new file mode 100644 index 0000000..0f4700c --- /dev/null +++ b/fuzz/run_fuzzers.py @@ -0,0 +1,229 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Run each Atheris harness for a bounded time; fail on a finding or on a run that fuzzed nothing. + + uv run --group fuzz python fuzz/run_fuzzers.py [TARGET ...] + +``mise run fuzz`` runs every target, and ``mise run fuzz:claude-code`` or +``mise run fuzz:codex`` one of them. ``ATIF_SQL_FUZZ_SECONDS`` sets each +target's ``-max_total_time`` (default 60). + +A target is a name in ``transcript_targets.TARGETS``. It runs as +``fuzz_.py`` under libFuzzer with the seeds in ``corpus//`` and the +dictionary ``.dict``, and everything it writes lands under +``out//`` (gitignored): ``corpus/``, the inputs libFuzzer found worth +keeping (listed first so libFuzzer writes there and never into the committed +seeds), ``artifacts/``, a crash, timeout or out-of-memory input when there is +one, and ``fuzz.log``, the harness's whole output. + +The run fails, naming the target, when: + +- atheris is not importable the ``fuzz`` group installs it on linux x86_64 only +- a target lacks its harness, its a deleted ``fuzz_.py`` must not leave a run + seeds or its dictionary that fuzzes the other target and passes +- a harness names no target a new harness must be registered, or it never runs +- a requested target is unknown +- no target was selected +- a harness exited non-zero or an uncaught exception, a timeout, an out-of-memory + left an artifact input, or libFuzzer refusing its flags +- libFuzzer executed no input the final ``stat::number_of_executed_units`` is + missing or 0, so nothing was fuzzed + +Each target prints one line, ``fuzz : runs in s, exit ``, +and a failure adds the reason and the artifact paths. +""" + +from __future__ import annotations + +import importlib.util +import os +import re +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from transcript_targets import TARGETS + +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + +FUZZ_DIR = Path(__file__).resolve().parent +SECONDS_ENV = "ATIF_SQL_FUZZ_SECONDS" +DEFAULT_SECONDS = 60 + +#: libFuzzer's per-input limit in seconds: an input slower than this is a finding. +INPUT_TIMEOUT_SECONDS = 30 +#: The largest input libFuzzer generates; the biggest seed is about 5 KB. +MAX_INPUT_BYTES = 16384 + +_EXECUTED = re.compile(r"^stat::number_of_executed_units:\s*(\d+)\s*$", re.MULTILINE) + + +@dataclass(frozen=True, slots=True) +class Target: + """One fuzz target's committed inputs.""" + + name: str + harness: Path + seeds: Path + dictionary: Path + + +@dataclass(frozen=True, slots=True) +class Outcome: + """What one harness run did.""" + + name: str + returncode: int + runs: int | None + seconds: float + artifacts: tuple[Path, ...] + log: Path + + @property + def failure(self) -> str | None: + """Why this run fails the task, or ``None`` when it passed.""" + if self.returncode != 0 or self.artifacts: + return f"harness exited {self.returncode} with {len(self.artifacts)} artifact(s)" + if not self.runs: + return "libFuzzer executed no input" + return None + + +def plan(names: Iterable[str], requested: Sequence[str], fuzz_dir: Path = FUZZ_DIR) -> list[Target]: + """The targets to run, after checking every registered target is complete. + + Raises + ------ + SystemExit: a target or harness is missing, a request is unknown, or none was selected. + """ + registered = sorted(names) + errors: list[str] = [] + targets: dict[str, Target] = {} + for name in registered: + target = Target( + name=name, + harness=fuzz_dir / f"fuzz_{name}.py", + seeds=fuzz_dir / "corpus" / name, + dictionary=fuzz_dir / f"{name}.dict", + ) + if not target.harness.is_file(): + errors.append(f"target {name}: no harness {target.harness.name}") + if not target.seeds.is_dir() or not any(p.is_file() for p in target.seeds.iterdir()): + errors.append(f"target {name}: no seed inputs in corpus/{name}/") + if not target.dictionary.is_file(): + errors.append(f"target {name}: no dictionary {target.dictionary.name}") + targets[name] = target + errors.extend( + f"harness {path.name} names no registered target" + for path in sorted(fuzz_dir.glob("fuzz_*.py")) + if path.stem.removeprefix("fuzz_") not in targets + ) + errors.extend(f"unknown target {name}" for name in requested if name not in targets) + selected = [targets[name] for name in (requested or registered) if name in targets] + if not selected: + errors.append("no fuzz target selected") + if errors: + raise SystemExit("\n".join(f"fuzz: {error}" for error in errors)) + return selected + + +def executed_units(log_text: str) -> int | None: + """LibFuzzer's final count of executed inputs, or ``None`` when it never printed one.""" + counts = _EXECUTED.findall(log_text) + return int(counts[-1]) if counts else None + + +def run_target( + target: Target, seconds: int, out_dir: Path, python: str = sys.executable +) -> Outcome: + """Run one harness under libFuzzer for ``seconds`` and read back what it did.""" + work = out_dir / target.name + corpus = work / "corpus" + artifacts = work / "artifacts" + shutil.rmtree(artifacts, ignore_errors=True) + corpus.mkdir(parents=True, exist_ok=True) + artifacts.mkdir(parents=True) + log = work / "fuzz.log" + command = [ + python, + str(target.harness), + f"-max_total_time={seconds}", + f"-timeout={INPUT_TIMEOUT_SECONDS}", + f"-max_len={MAX_INPUT_BYTES}", + f"-dict={target.dictionary}", + f"-artifact_prefix={artifacts}{os.sep}", + "-print_final_stats=1", + str(corpus), + str(target.seeds), + ] + started = time.monotonic() + with log.open("wb") as sink: + returncode = subprocess.run( # noqa: S603 - our interpreter, our harness, fixed flags + command, stdout=sink, stderr=subprocess.STDOUT, check=False + ).returncode + elapsed = time.monotonic() - started + return Outcome( + name=target.name, + returncode=returncode, + runs=executed_units(log.read_text(encoding="utf-8", errors="replace")), + seconds=elapsed, + artifacts=tuple(sorted(artifacts.iterdir())), + log=log, + ) + + +def report(outcome: Outcome) -> str: + """The lines one outcome prints.""" + runs = "no" if outcome.runs is None else str(outcome.runs) + lines = [ + f"fuzz {outcome.name}: {runs} runs in {outcome.seconds:.0f} s, exit {outcome.returncode}" + ] + if outcome.failure is not None: + lines.append(f"fuzz {outcome.name}: FAILED: {outcome.failure}; log {outcome.log}") + lines.extend(f"fuzz {outcome.name}: artifact {path}" for path in outcome.artifacts) + return "\n".join(lines) + + +def seconds_from_env() -> int: + """``ATIF_SQL_FUZZ_SECONDS`` as a positive whole number of seconds. + + Raises + ------ + SystemExit: the variable is set to anything but a positive integer. + """ + raw = os.environ.get(SECONDS_ENV, "").strip() + if not raw: + return DEFAULT_SECONDS + if not raw.isdigit() or int(raw) < 1: + msg = f"fuzz: {SECONDS_ENV} must be a positive whole number of seconds, got {raw!r}" + raise SystemExit(msg) + return int(raw) + + +def main(argv: Sequence[str]) -> int: + """Run the requested targets (all when none is named); 0 only when each one passed.""" + seconds = seconds_from_env() + targets = plan(TARGETS, argv) + if importlib.util.find_spec("atheris") is None: + msg = ( + "fuzz: atheris is not installed; it installs from the `fuzz` dependency group on " + "linux x86_64 only (uv run --group fuzz)" + ) + raise SystemExit(msg) + outcomes = [run_target(target, seconds, FUZZ_DIR / "out") for target in targets] + sys.stdout.writelines(f"{report(outcome)}\n" for outcome in outcomes) + failed = [outcome.name for outcome in outcomes if outcome.failure is not None] + if failed: + sys.stdout.write(f"fuzz: {len(failed)} of {len(outcomes)} target(s) failed: {failed}\n") + return 1 + total = sum(outcome.runs or 0 for outcome in outcomes) + sys.stdout.write(f"fuzz: {len(outcomes)} target(s) passed, {total} runs\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/fuzz/test_fuzz_seed_corpus.py b/fuzz/test_fuzz_seed_corpus.py new file mode 100644 index 0000000..486a805 --- /dev/null +++ b/fuzz/test_fuzz_seed_corpus.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Replay the committed seed corpus through the fuzz targets, with no fuzzing engine. + +This is the half of the fuzzing that runs in ``mise run check``: every seed +under ``corpus//`` goes through the same function the Atheris harness +drives, so a converter change that makes a seed raise anything but a +``DomainError`` fails here before a fuzzer ever runs. The seeds must also reach +a successful conversion (a corpus the converter rejects outright would start +the fuzzer at the front door), and a planted unexpected exception must escape +the target, which is what makes the harness report a crash. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import transcript_targets +from transcript_targets import TARGETS + +from atif_converter.domain.errors import ConversionError + +CORPUS = Path(__file__).resolve().parent / "corpus" + + +def _seeds() -> list[tuple[str, Path]]: + return [ + (name, seed) + for name in sorted(TARGETS) + for seed in sorted((CORPUS / name).iterdir()) + if seed.is_file() + ] + + +def test_every_target_has_seeds() -> None: + """Each registered target ships at least two seeds, and no seed directory is unregistered.""" + by_target = {name: [seed for target, seed in _seeds() if target == name] for name in TARGETS} + assert all(len(seeds) >= 2 for seeds in by_target.values()), by_target + assert sorted(path.name for path in CORPUS.iterdir() if path.is_dir()) == sorted(TARGETS) + + +@pytest.mark.parametrize( + ("name", "seed"), _seeds(), ids=lambda value: getattr(value, "name", value) +) +def test_seed_replays_without_unexpected_exception(name: str, seed: Path, tmp_path: Path) -> None: + """A seed raises nothing but the converter's own rejection.""" + TARGETS[name](seed.read_bytes(), tmp_path) + + +@pytest.mark.parametrize( + ("name", "seed"), _seeds(), ids=lambda value: getattr(value, "name", value) +) +def test_seed_converts( + name: str, seed: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Every seed converts: none is rejected, so each starts the fuzzer past the reader.""" + monkeypatch.setattr(transcript_targets, "EXPECTED_REJECTIONS", ()) + TARGETS[name](seed.read_bytes(), tmp_path) + + +@pytest.mark.parametrize("name", sorted(TARGETS)) +@pytest.mark.parametrize( + "data", [b"", b"\xff\xfe not utf-8\n", b'[]\n1\n"x"\nnull\n{\n', b"\x00\x00\x00"] +) +def test_malformed_input_is_rejected_or_converted(name: str, data: bytes, tmp_path: Path) -> None: + """Empty, non-UTF-8, non-object and truncated JSONL never escape as a crash.""" + TARGETS[name](data, tmp_path) + + +def test_empty_session_is_an_expected_rejection(tmp_path: Path) -> None: + """An empty transcript is the converter's ``EmptySessionError``, a ``DomainError``.""" + with pytest.raises(transcript_targets.EXPECTED_REJECTIONS): + transcript_targets.convert_and_audit( + transcript_targets.write_claude_code_session(b"", tmp_path) + ) + + +@pytest.mark.parametrize( + ("name", "use_case"), + [("claude_code", "convert_and_audit"), ("codex", "convert_codex_and_audit")], +) +def test_unexpected_exception_escapes( + name: str, use_case: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A planted ``KeyError`` in the use case reaches the caller: the harness reports it.""" + + def explode(*_args: object, **_kwargs: object) -> None: + raise KeyError(use_case) + + monkeypatch.setattr(transcript_targets, use_case, explode) + with pytest.raises(KeyError, match=use_case): + TARGETS[name](b"{}\n", tmp_path) + + +def test_domain_error_is_swallowed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A ``DomainError`` from the use case is the documented rejection, not a finding.""" + + def reject(*_args: object, **_kwargs: object) -> None: + msg = "refused" + raise ConversionError(msg) + + monkeypatch.setattr(transcript_targets, "convert_and_audit", reject) + TARGETS["claude_code"](b"{}\n", tmp_path) + + +def test_claude_code_layout_splits_on_nul(tmp_path: Path) -> None: + """Main transcript, then subagent transcript, then its sidecar, split on NUL.""" + main = transcript_targets.write_claude_code_session(b"a\x00b\x00c", tmp_path) + side = tmp_path / transcript_targets.CLAUDE_SESSION_ID / "subagents" + assert main.read_bytes() == b"a" + assert (side / "agent-fuzz.jsonl").read_bytes() == b"b" + assert (side / "agent-fuzz.meta.json").read_bytes() == b"c" diff --git a/fuzz/test_run_fuzzers.py b/fuzz/test_run_fuzzers.py new file mode 100644 index 0000000..17e0a1c --- /dev/null +++ b/fuzz/test_run_fuzzers.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Planted defects for ``run_fuzzers.py``: each way the task could pass on nothing turns it red. + +The harnesses here are stand-ins that print what libFuzzer prints, so these run +without atheris, in ``mise run check``. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import run_fuzzers +from run_fuzzers import FUZZ_DIR, Outcome, Target, executed_units, plan, run_target +from transcript_targets import TARGETS + +_STATS = "stat::number_of_executed_units: {runs}\nstat::average_exec_per_sec: 1\n" + + +def _tree(root: Path, names: list[str]) -> Path: + for name in names: + (root / f"fuzz_{name}.py").write_text("") + (root / f"{name}.dict").write_text('"x"\n') + (root / "corpus" / name).mkdir(parents=True) + (root / "corpus" / name / "seed").write_bytes(b"{}\n") + return root + + +def _stand_in(tmp_path: Path, body: str) -> Target: + harness = tmp_path / "fuzz_stand_in.py" + harness.write_text(body) + seeds = tmp_path / "seeds" + seeds.mkdir() + (seeds / "seed").write_bytes(b"{}\n") + dictionary = tmp_path / "stand_in.dict" + dictionary.write_text('"x"\n') + return Target(name="stand_in", harness=harness, seeds=seeds, dictionary=dictionary) + + +def test_the_committed_tree_plans_every_target() -> None: + """The repository's own fuzz/ directory is complete for every registered target.""" + assert [target.name for target in plan(TARGETS, [])] == sorted(TARGETS) + + +def test_missing_harness_fails(tmp_path: Path) -> None: + """Deleting one harness fails the plan rather than fuzzing the other target alone.""" + root = _tree(tmp_path, ["one", "two"]) + (root / "fuzz_two.py").unlink() + with pytest.raises(SystemExit, match=re.escape("target two: no harness fuzz_two.py")): + plan(["one", "two"], [], root) + + +def test_empty_seed_directory_fails(tmp_path: Path) -> None: + """A target whose seed directory is empty fails the plan.""" + root = _tree(tmp_path, ["one"]) + (root / "corpus" / "one" / "seed").unlink() + with pytest.raises(SystemExit, match="no seed inputs"): + plan(["one"], [], root) + + +def test_unregistered_harness_fails(tmp_path: Path) -> None: + """A harness file no target names fails the plan, so it can't sit there unrun.""" + root = _tree(tmp_path, ["one"]) + (root / "fuzz_stray.py").write_text("") + with pytest.raises( + SystemExit, match=re.escape("harness fuzz_stray.py names no registered target") + ): + plan(["one"], [], root) + + +def test_zero_targets_fails(tmp_path: Path) -> None: + """With nothing registered, nothing runs, and that is a failure.""" + with pytest.raises(SystemExit, match="no fuzz target selected"): + plan([], [], tmp_path) + + +def test_unknown_request_fails(tmp_path: Path) -> None: + """Asking for a target that does not exist fails rather than running nothing.""" + root = _tree(tmp_path, ["one"]) + with pytest.raises(SystemExit, match="unknown target nope"): + plan(["one"], ["nope"], root) + + +def test_executed_units_reads_the_last_count() -> None: + """The count is libFuzzer's final stats line; no line is ``None``.""" + assert executed_units(_STATS.format(runs=7) + _STATS.format(runs=12345)) == 12345 + assert executed_units("Done 3 runs in 1 second(s)\n") is None + + +def test_a_crash_fails(tmp_path: Path) -> None: + """A harness that exits non-zero and writes an artifact fails, naming the artifact.""" + body = ( + "import sys\n" + "prefix = next(a for a in sys.argv if a.startswith('-artifact_prefix='))\n" + "open(prefix.split('=', 1)[1] + 'crash-0', 'wb').write(b'[]')\n" + f"print({_STATS.format(runs=3)!r})\n" + "sys.exit(1)\n" + ) + outcome = run_target(_stand_in(tmp_path, body), 1, tmp_path / "out") + assert outcome.returncode == 1 + assert [path.name for path in outcome.artifacts] == ["crash-0"] + assert outcome.failure == "harness exited 1 with 1 artifact(s)" + assert "artifact" in run_fuzzers.report(outcome) + + +def test_an_artifact_fails_even_on_exit_zero(tmp_path: Path) -> None: + """An artifact left behind fails the run whatever the exit code says.""" + outcome = Outcome("t", 0, 10, 1.0, (tmp_path / "timeout-0",), tmp_path / "log") + assert outcome.failure is not None + + +def test_a_run_that_executed_nothing_fails(tmp_path: Path) -> None: + """Exit 0 with no executed input (or no stats at all) is not a pass.""" + zero = run_target( + _stand_in(tmp_path, f"print({_STATS.format(runs=0)!r})\n"), 1, tmp_path / "out" + ) + assert zero.failure == "libFuzzer executed no input" + silent = Outcome("t", 0, None, 1.0, (), tmp_path / "log") + assert silent.failure == "libFuzzer executed no input" + + +def test_a_clean_run_passes(tmp_path: Path) -> None: + """A harness that executed inputs, exited 0 and left no artifact passes.""" + outcome = run_target( + _stand_in(tmp_path, f"print({_STATS.format(runs=42)!r})\n"), 1, tmp_path / "out" + ) + assert (outcome.returncode, outcome.runs, outcome.failure) == (0, 42, None) + assert run_fuzzers.report(outcome).startswith("fuzz stand_in: 42 runs in ") + + +def test_stale_artifacts_are_cleared(tmp_path: Path) -> None: + """A previous run's crash file does not fail, or pass, the next run.""" + stale = tmp_path / "out" / "stand_in" / "artifacts" + stale.mkdir(parents=True) + (stale / "crash-old").write_bytes(b"") + outcome = run_target( + _stand_in(tmp_path, f"print({_STATS.format(runs=5)!r})\n"), 1, tmp_path / "out" + ) + assert outcome.artifacts == () + + +@pytest.mark.parametrize("value", ["0", "-5", "ten", "1.5"]) +def test_bad_seconds_fail(value: str, monkeypatch: pytest.MonkeyPatch) -> None: + """``ATIF_SQL_FUZZ_SECONDS`` must be a positive whole number.""" + monkeypatch.setenv(run_fuzzers.SECONDS_ENV, value) + with pytest.raises(SystemExit, match=run_fuzzers.SECONDS_ENV): + run_fuzzers.seconds_from_env() + + +def test_seconds_default_and_override(monkeypatch: pytest.MonkeyPatch) -> None: + """Unset means the default; a positive integer is taken as given.""" + monkeypatch.delenv(run_fuzzers.SECONDS_ENV, raising=False) + assert run_fuzzers.seconds_from_env() == run_fuzzers.DEFAULT_SECONDS + monkeypatch.setenv(run_fuzzers.SECONDS_ENV, "5") + assert run_fuzzers.seconds_from_env() == 5 + + +def test_missing_atheris_fails(monkeypatch: pytest.MonkeyPatch) -> None: + """Without atheris the task refuses rather than reporting a pass for no fuzzing.""" + + def no_spec(_name: str) -> None: + return None + + monkeypatch.setattr(run_fuzzers.importlib.util, "find_spec", no_spec) + with pytest.raises(SystemExit, match="atheris is not installed"): + run_fuzzers.main([]) + + +def test_the_harnesses_import_atheris() -> None: + """Each harness file carries the import Scorecard's Fuzzing check looks for.""" + for name in TARGETS: + assert re.search( + r"^import atheris\b", (FUZZ_DIR / f"fuzz_{name}.py").read_text(), re.MULTILINE + ) diff --git a/fuzz/transcript_targets.py b/fuzz/transcript_targets.py new file mode 100644 index 0000000..4e2d366 --- /dev/null +++ b/fuzz/transcript_targets.py @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Fuzz targets for the two transcript converters, free of any fuzzing engine. + +A target takes one fuzzer input, writes it to disk the way the agent writes a +transcript, and runs the use case materialize runs on that file: +:func:`~atif_converter.application.convert_and_audit.convert_and_audit` for a +Claude Code session, and +:func:`~atif_converter.application.convert_codex.convert_codex_and_audit` for a +Codex rollout. The whole read path is under test: the UTF-8 decode, the JSONL +split, attachment extraction, the conversion, the census, the edges and events +emitters, enrichment and the snapshot re-check. + +THE CONTRACT. The converter's application layer surfaces only +:class:`~atif_converter.domain.errors.DomainError` subtypes (an empty session, +a non-UTF-8 file, a converter failure classified at the seam), and that is the +documented rejection of bad input that atif-cli's ``RealConverter`` relies on to +record a per-session failure and move on. Any other exception escapes the +target and is a finding. A conversion that succeeds must also hand the corpus +writer what it can write: the trajectory and the loss report encode as JSON with +the writer's encoder, every edges and events line decodes to one JSON object, +and every lifted attachment's bytes hash to the sha256 it is filed under. + +INPUT LAYOUT. A NUL byte can't occur in a JSONL line (JSON rejects a raw control +character inside a string and outside one), so it splits one input into files. +For Claude Code the bytes up to the first NUL are the main +``.jsonl``, the bytes up to the second NUL are a subagent +transcript at ``/subagents/agent-fuzz.jsonl``, and the rest is that +subagent's ``agent-fuzz.meta.json`` sidecar. A Codex rollout has no side files, +so the whole input is one ``rollout--.jsonl``. + +The atheris entrypoints beside this module (``fuzz_claude_code.py``, +``fuzz_codex.py``) instrument and drive these functions; ``test_fuzz_seed_corpus.py`` +replays the committed seeds under ``corpus/`` through them in ``mise run check``, +where atheris is not installed. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import tempfile +from functools import cache +from pathlib import Path +from typing import TYPE_CHECKING, Protocol + +from atif_converter.application.convert_and_audit import convert_and_audit +from atif_converter.application.convert_codex import convert_codex_and_audit +from atif_converter.domain.errors import DomainError + +if TYPE_CHECKING: + from atif_converter.domain.fidelity import LossReport + from atif_converter.infrastructure.harbor_adapter import ConversionResult + +#: The exceptions a target lets through as the converter's verdict on bad input. +EXPECTED_REJECTIONS: tuple[type[Exception], ...] = (DomainError,) + +#: The session id the Claude Code input is filed under; any id passes discovery. +CLAUDE_SESSION_ID = "00000000-0000-4000-8000-000000000000" + +#: A Codex rollout filename, which ``codex_session_id`` parses for its trailing uuid. +CODEX_ROLLOUT_NAME = f"rollout-2026-01-01T00-00-00-{CLAUDE_SESSION_ID}.jsonl" + +_SEPARATOR = b"\x00" + +#: The corpus writer's encoder (``atif_corpus.infrastructure.atomic``): compact, +#: and NaN allowed, as the writer allows it. +_ENCODER = json.JSONEncoder(separators=(",", ":")) + + +class FuzzTarget(Protocol): + """One target: libFuzzer calls it with the input alone, the replay test adds a directory.""" + + def __call__(self, data: bytes, workdir: Path | None = None, /) -> None: + """Convert ``data``; raise on anything but an expected rejection.""" + ... + + +class InvariantViolation(AssertionError): # noqa: N818 - names the finding, as the domain errors do + """A conversion succeeded but produced an artifact the corpus could not write.""" + + +@cache +def _scratch_root() -> Path: + """One scratch directory per process, reused by every input.""" + return Path(tempfile.mkdtemp(prefix="atif-fuzz-")) + + +def _fresh(workdir: Path | None, name: str) -> Path: + """An empty directory for one input: ``workdir`` when given, else the process scratch.""" + root = (workdir if workdir is not None else _scratch_root()) / name + if root.exists(): + shutil.rmtree(root) + root.mkdir(parents=True) + return root + + +def _check_artifacts(result: ConversionResult, report: LossReport) -> None: + """Fail when a successful conversion hands the corpus something it can't store.""" + _ENCODER.encode(result.trajectory) + _ENCODER.encode(report.to_json()) + for line in (*result.edges_lines, *result.events_lines): + if not isinstance(json.loads(line), dict): + msg = f"an edges or events line is not a JSON object: {line[:200]!r}" + raise InvariantViolation(msg) + for blob in result.blobs: + if hashlib.sha256(blob.data).hexdigest() != blob.ref.sha256: + msg = f"blob filed under {blob.ref.sha256} hashes to something else" + raise InvariantViolation(msg) + if len(blob.data) != blob.ref.byte_count: + msg = f"blob {blob.ref.sha256} holds {len(blob.data)} bytes, not {blob.ref.byte_count}" + raise InvariantViolation(msg) + + +def write_claude_code_session(data: bytes, root: Path) -> Path: + """Lay one input out as a Claude Code session under ``root``; return the main JSONL.""" + main, _, rest = data.partition(_SEPARATOR) + session_jsonl = root / f"{CLAUDE_SESSION_ID}.jsonl" + session_jsonl.write_bytes(main) + if rest: + subagent, _, sidecar = rest.partition(_SEPARATOR) + side_dir = root / CLAUDE_SESSION_ID / "subagents" + side_dir.mkdir(parents=True) + (side_dir / "agent-fuzz.jsonl").write_bytes(subagent) + if sidecar: + (side_dir / "agent-fuzz.meta.json").write_bytes(sidecar) + return session_jsonl + + +def claude_code_one_input(data: bytes, workdir: Path | None = None) -> None: + """Convert one input as a Claude Code session; raise on anything but a DomainError.""" + session_jsonl = write_claude_code_session(data, _fresh(workdir, "claude_code")) + try: + result, report = convert_and_audit(session_jsonl) + except EXPECTED_REJECTIONS: + return + _check_artifacts(result, report) + + +def codex_one_input(data: bytes, workdir: Path | None = None) -> None: + """Convert one input as a Codex rollout; raise on anything but a DomainError.""" + rollout = _fresh(workdir, "codex") / CODEX_ROLLOUT_NAME + rollout.write_bytes(data) + try: + result, report = convert_codex_and_audit(rollout) + except EXPECTED_REJECTIONS: + return + _check_artifacts(result, report) + + +#: Every target by name: the name is its seed directory under ``corpus/`` and its +#: harness ``fuzz_.py``, and ``run_fuzzers.py`` fails when either is missing. +TARGETS: dict[str, FuzzTarget] = { + "claude_code": claude_code_one_input, + "codex": codex_one_input, +} diff --git a/mise.toml b/mise.toml index 19814c0..36899c3 100644 --- a/mise.toml +++ b/mise.toml @@ -126,12 +126,12 @@ run = "uv run lefthook uninstall" [tasks.lint] description = "Lint with ruff" -sources = ["packages/*/src/**/*.py", "packages/*/tests/**/*.py", "scripts/**/*.py", "pyproject.toml", "packages/*/pyproject.toml"] +sources = ["packages/*/src/**/*.py", "packages/*/tests/**/*.py", "scripts/**/*.py", "fuzz/**/*.py", "pyproject.toml", "packages/*/pyproject.toml"] run = "uv run ruff check ." [tasks.fmt] description = "Check formatting" -sources = ["packages/*/src/**/*.py", "packages/*/tests/**/*.py", "scripts/**/*.py", "pyproject.toml", "packages/*/pyproject.toml"] +sources = ["packages/*/src/**/*.py", "packages/*/tests/**/*.py", "scripts/**/*.py", "fuzz/**/*.py", "pyproject.toml", "packages/*/pyproject.toml"] run = "uv run ruff format --check ." [tasks."fmt:write"] @@ -140,7 +140,7 @@ run = "uv run ruff format ." [tasks.typecheck] description = "Type-check with ty (strict, over every member's src/ + tests/ + scripts/)" -sources = ["packages/*/src/**/*.py", "packages/*/tests/**/*.py", "scripts/**/*.py", "pyproject.toml", "packages/*/pyproject.toml"] +sources = ["packages/*/src/**/*.py", "packages/*/tests/**/*.py", "scripts/**/*.py", "fuzz/**/*.py", "pyproject.toml", "packages/*/pyproject.toml"] run = "uv run ty check" [tasks."typecheck:pyright"] @@ -207,7 +207,7 @@ run = "python3 scripts/verify_single_yaml_document.py site/pnpm-lock.yaml lint:p [tasks.test] description = "Run the full pytest suite across all members" -sources = ["packages/*/src/**/*.py", "packages/*/tests/**/*.py", "pyproject.toml", "packages/*/pyproject.toml"] +sources = ["packages/*/src/**/*.py", "packages/*/tests/**/*.py", "fuzz/**/*.py", "fuzz/corpus/**", "fuzz/*.dict", "pyproject.toml", "packages/*/pyproject.toml"] run = "uv run pytest --no-header -q" [tasks."test:cov"] @@ -217,6 +217,29 @@ description = "The suite under branch coverage; writes coverage.xml and enforces # lives in pyproject.toml, so this task and Codecov read one definition. run = "uv run pytest --no-header -q --cov --cov-report=term-missing --cov-report=xml" +# --------------------------------------------------------------------------- +# Fuzzing (Atheris). Outside `check`: it needs atheris, which the `fuzz` dependency group +# installs on linux x86_64 only, and it spends a minute per target by design. `check` still +# replays the committed seeds through the same targets (fuzz/test_fuzz_seed_corpus.py), and +# .github/workflows/fuzz.yml runs these tasks weekly and on pull requests that touch the +# converter. ATIF_SQL_FUZZ_SECONDS bounds each target (default 60). A crash, a timeout, an +# out-of-memory input, or a run that executed no input fails the task, and the input lands +# in fuzz/out//artifacts/ (gitignored). Reproduce one with +# `uv run --group fuzz python fuzz/fuzz_.py `. +# --------------------------------------------------------------------------- + +[tasks.fuzz] +description = "Fuzz every transcript converter with Atheris, ATIF_SQL_FUZZ_SECONDS each (default 60)" +run = "uv run --group fuzz python fuzz/run_fuzzers.py" + +[tasks."fuzz:claude-code"] +description = "Fuzz the Claude Code transcript converter with Atheris" +run = "uv run --group fuzz python fuzz/run_fuzzers.py claude_code" + +[tasks."fuzz:codex"] +description = "Fuzz the Codex rollout converter with Atheris" +run = "uv run --group fuzz python fuzz/run_fuzzers.py codex" + [tasks.check] description = "All gates: lint + fmt + typecheck (ty + pyright) + imports + workflows + hooks + pnpm lock + vex drift + prose + test" depends = [ diff --git a/pyproject.toml b/pyproject.toml index 08e2cd1..e0e8b4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -166,6 +166,13 @@ dev = [ "ruff>=0.15.11", "ty>=0.0.31", ] +# The Atheris fuzz harnesses under fuzz/ (`mise run fuzz`), outside the default groups so +# `uv sync` never installs it. atheris 3.1.0 publishes only manylinux x86_64 wheels (cp312 to +# cp314) and no sdist, so the marker keeps the group installable, and uv.lock universal, on +# every other platform: there it resolves to nothing and `mise run fuzz` refuses to run. +fuzz = [ + "atheris>=3.1.0; sys_platform == 'linux' and platform_machine == 'x86_64'", +] # --------------------------------------------------------------------------- # ruff — shared across every workspace member. The py313 target follows the @@ -286,6 +293,10 @@ ignore = [ # never imported, so neither is a package. "scripts/**" = ["INP001"] "experiments/**" = ["INP001"] +# fuzz/ is executed by path too (`python fuzz/fuzz_codex.py`), and pytest collects its two +# test modules by rootdir like any tests tree, which is why `assert` is allowed there. +"fuzz/**" = ["INP001"] +"fuzz/test_*.py" = ["S101", "PLR2004", "SLF001"] [tool.ruff.lint.isort] known-first-party = ["atif_converter", "atif_corpus", "atif_duck", "atif_models", "atif_analytics", "atif_embed", "atif_cli", "conftest"] @@ -401,6 +412,8 @@ root = [ "./packages/atif-embed/tests", "./packages/atif-cli/src", "./packages/atif-cli/tests", + # The fuzz harnesses import their target module by its bare name, as libFuzzer runs them. + "./fuzz", ] [tool.ty.src] @@ -418,6 +431,7 @@ include = [ "packages/atif-embed/tests", "packages/atif-cli/src", "packages/atif-cli/tests", + "fuzz", ] # Vendored harbor source, kept byte for byte (see `[tool.ruff] extend-exclude`). # Its own `ty: ignore` comments target an older ty, so checking it would mean @@ -444,11 +458,22 @@ possibly-unresolved-reference = "warn" possibly-missing-attribute = "warn" unused-ignore-comment = "warn" +# atheris installs only from the optional `fuzz` group, and only on linux x86_64, so the +# default environment cannot resolve `import atheris`. The two entrypoints are a few lines +# around that import; the target logic they drive is in fuzz/transcript_targets.py, which +# this override does not cover. A per-line ignore would turn into an unused-ignore error +# in an environment that has the group installed. +[[tool.ty.overrides]] +include = ["fuzz/fuzz_*.py"] + +[tool.ty.overrides.rules] +unresolved-import = "ignore" + # --------------------------------------------------------------------------- # pytest — one run over every member's tests/ tree. # --------------------------------------------------------------------------- [tool.pytest.ini_options] -testpaths = ["packages/*/tests"] +testpaths = ["packages/*/tests", "fuzz"] markers = [ "integration: end-to-end tests through the real converter into a tmp corpus", ] @@ -672,6 +697,7 @@ include = [ "packages/atif-cli/src", "packages/atif-cli/tests", "scripts", + "fuzz", ] # The seven member src/ trees, so a first-party import resolves to the # workspace source rather than to the editable install's redirect. diff --git a/uv.lock b/uv.lock index d70c15b..b2eab4e 100644 --- a/uv.lock +++ b/uv.lock @@ -170,6 +170,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/bd/551ee6af426af84ca33e02622be722925c196608e9127d731ef17c47f06e/argcomplete-3.7.2-py3-none-any.whl", hash = "sha256:6029205678bdd9c1c728a155f5f9ecf5812393f969eef58807641a2bc2aa5b19", size = 43294, upload-time = "2026-08-06T04:53:20.246Z" }, ] +[[package]] +name = "atheris" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/18/e19718c384fd7d801d0da7485407daef9af6194b6d8c8818175bec5efec6/atheris-3.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f8a9f51ce8369026e8eb7b7174835e8c4c85a1a6db5d9add36c15100779d2a39", size = 36800563, upload-time = "2026-06-17T00:04:04.559Z" }, + { url = "https://files.pythonhosted.org/packages/c3/ff/ae7a5bfe99033e510bea4ed09934e636d93777317a48147369bc0dc2b71f/atheris-3.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:315a0b5c819852b1ffe1ca72efc389c7724881f2c33e4aacb8c6bcec49bd5011", size = 36772569, upload-time = "2026-06-17T00:04:07.702Z" }, +] + [[package]] name = "atif-analytics" version = "0.1.0" @@ -403,6 +412,9 @@ dev = [ { name = "ruff" }, { name = "ty" }, ] +fuzz = [ + { name = "atheris", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] [package.metadata] requires-dist = [ @@ -443,6 +455,7 @@ dev = [ { name = "ruff", specifier = ">=0.15.11" }, { name = "ty", specifier = ">=0.0.31" }, ] +fuzz = [{ name = "atheris", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = ">=3.1.0" }] [[package]] name = "attrs" From 5e8bca34e70fd7c835379ca832cc19b35deb783c Mon Sep 17 00:00:00 2001 From: Laith Al-Saadoon Date: Sat, 10 Oct 2026 17:58:55 +0000 Subject: [PATCH 4/6] fix(converter): reject non-string uuids and payload types instead of crashing The Atheris fuzzers found two inputs that raised TypeError past the converter's DomainError contract, so materialize would crash on a session rather than record it as failed and move on: - Claude Code: build_edges sorts on (ts or "", uuid or ""); two records with no timestamp, one with a uuid that is a JSON object, raised "'<' not supported between instances of 'str' and 'dict'". - Codex: _tool_use_ids tested `payload.get("type") in _CALL_ID_ITEM_TYPES`; a type that is a list or object raised "unhashable type". The edges sort retries with non-string values treated as missing only when the original order raised, and the Codex check requires a string type. Every input that converted before keeps byte-identical artifacts, so CONVERTER_SCHEMA_VERSION stays 9 and only the source digest is re-pinned: the inputs whose output changes are the ones that produced no artifact. Both crash inputs are kept under fuzz/regressions/ and replayed in check; each test fails on the unfixed code with the original TypeError. Co-Authored-By: Claude Opus 5.5 --- .editorconfig | 4 +-- fuzz/regressions/claude_code/object-uuid | 4 +++ fuzz/regressions/codex/list-payload-type | 18 ++++++++++ fuzz/test_fuzz_seed_corpus.py | 34 +++++++++++++++++++ mise.toml | 2 +- .../src/atif_converter/domain/codex_edges.py | 5 ++- .../src/atif_converter/domain/edges.py | 14 +++++++- .../tests/test_converter_schema_version.py | 2 +- 8 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 fuzz/regressions/claude_code/object-uuid create mode 100644 fuzz/regressions/codex/list-payload-type diff --git a/.editorconfig b/.editorconfig index 857c06f..c3de3ff 100644 --- a/.editorconfig +++ b/.editorconfig @@ -52,8 +52,8 @@ indent_size = 2 insert_final_newline = unset trim_trailing_whitespace = unset -# Fuzzer seed inputs are bytes the harness reads verbatim, NUL separators included. -[fuzz/corpus/**] +# Fuzzer seeds and crash regressions are bytes the harness reads verbatim, NUL separators included. +[fuzz/{corpus,regressions}/**] charset = unset insert_final_newline = unset trim_trailing_whitespace = unset diff --git a/fuzz/regressions/claude_code/object-uuid b/fuzz/regressions/claude_code/object-uuid new file mode 100644 index 0000000..f9255eb --- /dev/null +++ b/fuzz/regressions/claude_code/object-uuid @@ -0,0 +1,4 @@ +{"type": "user", "uuid": "u-1", "parentUuid": null, "sessionId": "11111111-1111-1111-1111-111111111111", "timestamp": "2026-08-22T00:00:01Z", "cwd": "/tmp/proj", "gitBranch": "main", "version": "2.0.0", "isSidechain": false, "message": {"role": "user", "content": "hello please run a tool"}} +{"type": "assistant", "uuid": "a-2", "parentUuid": "u-2", "sessionId": "11111111-1111-1111-1111-111111111111", "timestamp": "2026-08-22T00:00:06Z", "isSidechain": false, "requestId": "req_002", "message": {"id": "msg_02", "role": "assistant", "model": "claude-test-1", "content": [{"type": "text", "text": "done"}], "stop_reason": "end_turn", "usage": {"input_tokens": 20, "output_tokens": 3, "cache_read_input_tokens": 110, "cache_creation_input_tokens": 0}}} +{"type": "summary", "uuid": {"a": 1}} +{"type": "summary", "uuid": "x"} diff --git a/fuzz/regressions/codex/list-payload-type b/fuzz/regressions/codex/list-payload-type new file mode 100644 index 0000000..4cbc551 --- /dev/null +++ b/fuzz/regressions/codex/list-payload-type @@ -0,0 +1,18 @@ +{"timestamp": "2026-09-11T17:27:01.000Z", "type": "session_meta", "payload": {"id": "01a09182-2858-7f42-936b-7f027b341fdf", "session_id": "01a09182-2858-7f42-936b-7f027b341fdf", "timestamp": "2026-09-11T17:27:01.000Z", "cwd": "/home/alice/proj", "originator": "codex-tui", "cli_version": "0.154.0", "source": "cli", "git": {"branch": "main", "commit_hash": "abc1234"}}} +{"timestamp": "2026-09-11T17:27:01.100Z", "type": "event_msg", "payload": {"type": ["task_started"], "turn_id": "turn-1"}} +{"timestamp": "2026-09-11T17:27:01.200Z", "type": "turn_context", "payload": {"turn_id": "turn-1", "cwd": "/home/alice/proj", "model": "openai/gpt-5.1-codex"}} +{"timestamp": "2026-09-11T17:27:01.300Z", "type": "world_state", "payload": {"full": true, "state": {"model": "openai/gpt-5.1-codex"}}} +{"timestamp": "2026-09-11T17:27:01.400Z", "type": "response_item", "payload": {"type": "message", "id": "msg_dev_1", "role": "developer", "content": [{"type": "input_text", "text": "use them"}]}} +{"timestamp": "2026-09-11T17:27:01.500Z", "type": "response_item", "payload": {"type": "message", "id": "msg_user_1", "role": "user", "content": [{"type": "input_text", "text": "read marker.txt with a shell command"}]}} +{"timestamp": "2026-09-11T17:27:02.000Z", "type": "response_item", "payload": {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "rsn_opaque"}} +{"timestamp": "2026-09-11T17:27:02.100Z", "type": "response_item", "payload": {"type": "message", "id": "msg_agent_empty", "role": "assistant", "content": [{"type": "output_text", "text": ""}]}} +{"timestamp": "2026-09-11T17:27:02.200Z", "type": "response_item", "payload": {"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": "exec_command", "arguments": "{\"cmd\":\"cat marker.txt\"}", "status": "completed"}} +{"timestamp": "2026-09-11T17:27:02.300Z", "type": "response_item", "payload": {"type": "function_call_output", "call_id": "call_1", "output": "{\"output\":\"marker-ok\",\"metadata\":{\"exit_code\":0}}"}} +{"timestamp": "2026-09-11T17:27:02.400Z", "type": "response_item", "payload": {"type": "tool_search_call", "id": "tsc_1", "call_id": "call_search_1", "status": "completed", "arguments": {"query": "outlook email tools"}}} +{"timestamp": "2026-09-11T17:27:02.500Z", "type": "response_item", "payload": {"type": "tool_search_output", "call_id": "call_search_1", "status": "completed", "tools": [{"type": "function", "name": "email_read"}]}} +{"timestamp": "2026-09-11T17:27:02.600Z", "type": "event_msg", "payload": {"type": "token_count", "info": {"total_token_usage": {"input_tokens": 1000, "cached_input_tokens": 0, "cache_write_input_tokens": 900, "output_tokens": 40, "reasoning_output_tokens": 0, "total_tokens": 1040}, "last_token_usage": {"input_tokens": 1000, "cached_input_tokens": 0, "cache_write_input_tokens": 900, "output_tokens": 40, "reasoning_output_tokens": 0, "total_tokens": 1040}, "model_context_window": 258400}}} +{"timestamp": "2026-09-11T17:27:02.700Z", "type": "token_usage_record", "payload": {"turn_id": "turn-1", "usage": {"input_tokens": 1000}}} +{"timestamp": "2026-09-11T17:27:03.000Z", "type": "response_item", "payload": {"type": "message", "id": "msg_agent_final", "role": "assistant", "content": [{"type": "output_text", "text": "marker-ok"}]}} +{"timestamp": "2026-09-11T17:27:03.100Z", "type": "event_msg", "payload": {"type": "token_count", "info": {"total_token_usage": {"input_tokens": 1100, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 8, "reasoning_output_tokens": 0, "total_tokens": 1108}, "last_token_usage": {"input_tokens": 1100, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 8, "reasoning_output_tokens": 0, "total_tokens": 1108}, "model_context_window": 258400}}} +{"timestamp": "2026-09-11T17:27:03.200Z", "type": "compacted", "payload": {"message": "summary of the work so far", "replacement_history": []}} +{"timestamp": "2026-09-11T17:27:03.300Z", "type": "event_msg", "payload": {"type": "task_complete", "turn_id": "turn-1"}} diff --git a/fuzz/test_fuzz_seed_corpus.py b/fuzz/test_fuzz_seed_corpus.py index 486a805..775fb66 100644 --- a/fuzz/test_fuzz_seed_corpus.py +++ b/fuzz/test_fuzz_seed_corpus.py @@ -110,3 +110,37 @@ def test_claude_code_layout_splits_on_nul(tmp_path: Path) -> None: assert main.read_bytes() == b"a" assert (side / "agent-fuzz.jsonl").read_bytes() == b"b" assert (side / "agent-fuzz.meta.json").read_bytes() == b"c" + + +#: Inputs the fuzzer crashed the converters with, kept so each fix stays fixed. They sit +#: outside ``corpus/`` because the converter rejects them, and a seed must convert. +REGRESSIONS = Path(__file__).resolve().parent / "regressions" + + +def _regressions() -> list[tuple[str, Path]]: + return [ + (target.name, crash) + for target in sorted(REGRESSIONS.iterdir()) + if target.is_dir() + for crash in sorted(target.iterdir()) + if crash.is_file() + ] + + +def test_every_regression_names_a_target() -> None: + """Anti-vacuity: the regressions exist, and each sits under a registered target.""" + found = _regressions() + assert len(found) >= 2, found + assert {name for name, _ in found} <= set(TARGETS) + + +@pytest.mark.parametrize( + ("name", "crash"), _regressions(), ids=lambda value: getattr(value, "name", value) +) +def test_fuzzer_crash_stays_fixed(name: str, crash: Path, tmp_path: Path) -> None: + """A crash the fuzzer found is now rejected or converted. + + A non-string ``uuid`` or ``timestamp`` broke the Claude Code edges sort, and an unhashable + payload ``type`` broke the Codex tool-use ids; both raised ``TypeError`` past the converter. + """ + TARGETS[name](crash.read_bytes(), tmp_path) diff --git a/mise.toml b/mise.toml index 36899c3..2490f26 100644 --- a/mise.toml +++ b/mise.toml @@ -207,7 +207,7 @@ run = "python3 scripts/verify_single_yaml_document.py site/pnpm-lock.yaml lint:p [tasks.test] description = "Run the full pytest suite across all members" -sources = ["packages/*/src/**/*.py", "packages/*/tests/**/*.py", "fuzz/**/*.py", "fuzz/corpus/**", "fuzz/*.dict", "pyproject.toml", "packages/*/pyproject.toml"] +sources = ["packages/*/src/**/*.py", "packages/*/tests/**/*.py", "fuzz/**/*.py", "fuzz/corpus/**", "fuzz/regressions/**", "fuzz/*.dict", "pyproject.toml", "packages/*/pyproject.toml"] run = "uv run pytest --no-header -q" [tasks."test:cov"] diff --git a/packages/atif-converter/src/atif_converter/domain/codex_edges.py b/packages/atif-converter/src/atif_converter/domain/codex_edges.py index 7a82d4b..ebb4621 100644 --- a/packages/atif-converter/src/atif_converter/domain/codex_edges.py +++ b/packages/atif-converter/src/atif_converter/domain/codex_edges.py @@ -104,7 +104,10 @@ def _edge_type(record: dict[str, Any]) -> str | None: def _tool_use_ids(record: dict[str, Any]) -> list[str]: """The record's ``call_id``, when it is a tool call or a tool output.""" payload = _payload(record) - if payload.get("type") not in _CALL_ID_ITEM_TYPES: + item_type = payload.get("type") + # A `type` that is an object or a list is unhashable, and the membership test raised + # `TypeError` on it (found by the fuzzer). Only a string can name a call item. + if not isinstance(item_type, str) or item_type not in _CALL_ID_ITEM_TYPES: return [] call_id = payload.get("call_id") return [call_id] if isinstance(call_id, str) and call_id else [] diff --git a/packages/atif-converter/src/atif_converter/domain/edges.py b/packages/atif-converter/src/atif_converter/domain/edges.py index 43001f5..2ecb2de 100644 --- a/packages/atif-converter/src/atif_converter/domain/edges.py +++ b/packages/atif-converter/src/atif_converter/domain/edges.py @@ -86,10 +86,22 @@ def build_edges(records: list[tuple[dict[str, Any], str]]) -> list[dict[str, Any first as empty strings. """ edges = [edge_from_record(record, source_file) for record, source_file in records] - edges.sort(key=lambda e: (e["ts"] or "", e["uuid"] or "")) + try: + edges.sort(key=lambda e: (e["ts"] or "", e["uuid"] or "")) + except TypeError: + # A `timestamp` or `uuid` that is not a string (an object, a list, a number beside a + # string) cannot be compared with the others; the fuzzer found the `TypeError` this + # raised. Such values sort as missing ones do. Only an order that raised takes this + # branch, so every input that converted before keeps its exact order. + edges.sort(key=lambda e: (_sort_text(e["ts"]), _sort_text(e["uuid"]))) return edges +def _sort_text(value: object) -> str: + """A sort key part: the string itself, or ``""`` for a missing or non-string value.""" + return value if isinstance(value, str) else "" + + def edge_to_jsonl_line(edge: dict[str, Any]) -> str: """Serialize one edge as a compact, key-order-stable JSONL line.""" ordered = {key: edge.get(key) for key in EDGE_FIELDS} diff --git a/packages/atif-converter/tests/test_converter_schema_version.py b/packages/atif-converter/tests/test_converter_schema_version.py index 86bec55..83b64e0 100644 --- a/packages/atif-converter/tests/test_converter_schema_version.py +++ b/packages/atif-converter/tests/test_converter_schema_version.py @@ -29,7 +29,7 @@ #: The reviewed pair. Update BOTH in the commit that changes converter code: #: the version only when output can change, the digest always. PINNED_VERSION = 9 -PINNED_SOURCE_DIGEST = "5cc8a5c88c891e5e8289006c784b09eb369395a889d1de46adb9e91c5c8de176" +PINNED_SOURCE_DIGEST = "5ac8625584d3d5d0b8ec67168ab5b4efea324e0b08bf6510c8c5da87d1e36b89" _SRC = Path(__file__).resolve().parents[1] / "src" / "atif_converter" From c0ad2c4afcad25f991bab142ad9a394c04443013 Mon Sep 17 00:00:00 2001 From: Laith Al-Saadoon Date: Sat, 10 Oct 2026 18:31:06 +0000 Subject: [PATCH 5/6] fix(converter): guard every string-set membership test on transcript fields The weekly fuzz job's first pull-request run found a third crash of the same class: extract_codex_blobs tested a response_item payload `type` against a set, and a `type` that is a JSON object raised "unhashable type" before the conversion's error translation, escaping as TypeError. Every membership test outside the harbor conversion that takes a transcript value (the Claude Code block type in blob extraction, the Codex payload and item types in blobs, result signals and enrichment, the tool name and step source) now goes through a `_str` key that is None for a non-string. A non-string never matched these string sets, so every input that converted before keeps byte-identical artifacts: CONVERTER_SCHEMA_VERSION stays 9 and the source digest is re-pinned. The crash input is kept under fuzz/regressions/codex/ and fails on the unfixed code; a further 240 s per harness (742,039 runs) found nothing. Co-Authored-By: Claude Opus 5.5 --- .../codex/dict-payload-type-in-blobs | 21 ++++++++++++++++ .../src/atif_converter/domain/blobs.py | 14 +++++++++-- .../atif_converter/domain/codex_enrichment.py | 16 ++++++++++--- .../atif_converter/domain/result_signals.py | 24 +++++++++++++++---- .../tests/test_converter_schema_version.py | 2 +- 5 files changed, 66 insertions(+), 11 deletions(-) create mode 100644 fuzz/regressions/codex/dict-payload-type-in-blobs diff --git a/fuzz/regressions/codex/dict-payload-type-in-blobs b/fuzz/regressions/codex/dict-payload-type-in-blobs new file mode 100644 index 0000000..00d02cb --- /dev/null +++ b/fuzz/regressions/codex/dict-payload-type-in-blobs @@ -0,0 +1,21 @@ +{"timestamp": "2026-09-11T17:27:01.000Z", "type": "session_meta", "payload": {"id": "01a09182-2858-7f42-936b-7f027b341fdf", "session_id": "01a09182-2858-7f42-936b-7f027b341fdf", "timestamp": "2026-09-11T17:27:01.000Z", "cwd": "/home/alice/proj", "originator": "codex-tui", "cli_version": "0.154.0", "source": "cli", "git": {"branch": "main", "commit_hash": "abc1234"}}} +{"timestamp": "2026-09-11T17:27:01.100Z", "type": "event_msg", "payload": {"type": "task_started", "turn_id": "turn-1"}} +{"timestamp": "2026-09-11T17:27:01.200Z", "type": "turn_context", "payload": {"turn_id": "turn-1", "cwd": "/home/alice/proj", "model": "openai/gpt-5.1-codex"}} +{"timestamp": "2026-09-11T17:27:01.300Z", "type": "world_state", "payload": {"full": true, "state": {"model": "openai/gpt-5.1-codex"}}} +{"timestamp": "2026-09-11T17:27:01.400Z", "type": "response_item", "payload": {"type": "message", "id": "msg_dev_1", "role": "developer", "content": [{"type": "input_text", "text": "use them"}]}} +{"titype": "response_item", "payload": {"type": "message", "id": "msg_dev_1", "role": "developer", "content": [{"type": "input_text", "text": "use them"}]}} +{"timestamp": "2026-09-11T17:27:01.500Z", "type": "response_item", "payload": {"type": "message", "id": "msg_user_1", "role": "~ser", "content": [{"type": "input_text", "text": "read marker.txt with a shell command"}]}} +{"timestamp": "2026-09-11T17:27:02.000Z", "type": "responyload":"observation"ssage", "id": "msg_agent_empty", "role": "assistant", "content": [{"type": "output_text", "text": ""}]}} +{"timestamp": "2026-09-11T17:27:02.200Z", "type": "response_item", "payload": {"type": {"branch": "main", "commit_hash": "abc1234"}}} +{"timestamp": "2026-09-11T17:27:01.100Z", "type": "event_msg", "payload": {"type": "task_started", "turn_id": "turn-1"}} +{"timestamp": "2026-09-11T17:27:01.200Z", "type": "turn_context", "payload": {"turn_id": "turn-1", "cwd": "/home/alice/proj", "model": "openai/gpt-5.1-codex"}} +{"timestamp": "2026-09-11T17:27:01.300Z", "type": "world_state", "payload": {"full": true, "state": {"model": "openai/gpt-5.1-codex"}}} +{"timestamp": "2026-09-11T17:27:01.400Z", "type": "response_item", "payload": {"type": "message", "id": "msg_dev_1", "role": "developer", "content": [{"type": "input_text", "text": "use them"}]}} +{"titype": "response_item", "payload": {"type": "message", "id": "msg_dev_1", "role": "developer", "content": [{"type": "input_text", "text": "use them"}]}} +{"timestamp": "2026-09-11T17:27:01.500Z", "type": "response_item", "payload": {"type": "message", "id": "msg_user_1", "role": "~ser", "content": [{"type": "input_text", "text": "read marker.txt with a shell command"}]}} +{"timestamp": "2026-09-11T17:27:02.000Z", "type": "responyload":"observation"ssage", "id": "msg_agent_empty", "role": "assistant", "content": [{"type": "output_text", "text": ""}]}} +{"timestamp": "2026-09-11T17:27:02.200Z", "type": "response_item", "payload": {"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": "exec_command", "arguments": "{\"cmd\":\"cat marker.txt\"}", "status": "completed"}} +{"timestamp": "2026-09-11T17:27:02.300Z", "type": "response_item", "payload": {"type": "function_call_output", "call_id": "call_1", "output": "{\"output\":\"marker-ok\",\"metadata\":{\"exit_code\":0}}"}} +{"timestamp": "2026-09-11T17:27:02.400Z_token_usage": {"input_tokens": 1100, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "mutput_tokens": 8, "reasoning_output_tokens": 0, "total_tokens": 1108}, "last_token_usage": {"input_tokens": 1100, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 8, "reasoning_output_tokens": 0, "total_tokens": 1108}, "model_context_window": 258400}}} +{"timestamp": "2026-09-11T17:27:03.200Z", "type": "compacted "msg_agent_final", "role": "assistant", "content": [{"type": "output_text", "text": "marker-ok"}]}} +{"timestamp": "2026-09-11T17:27:03.100Z"msg", "payload": {"type": "task_complete", "turn_id": "turn-1"}} diff --git a/packages/atif-converter/src/atif_converter/domain/blobs.py b/packages/atif-converter/src/atif_converter/domain/blobs.py index cc533ae..a76ad14 100644 --- a/packages/atif-converter/src/atif_converter/domain/blobs.py +++ b/packages/atif-converter/src/atif_converter/domain/blobs.py @@ -253,7 +253,7 @@ def _extend_unique(target: list[BlobRef], refs: Iterable[BlobRef]) -> None: def _anthropic_block_ref(block: Any, collector: BlobCollector) -> BlobRef | None: """The ref for an ``image``/``document`` block with a base64 source, else ``None``.""" - if not isinstance(block, dict) or block.get("type") not in {"image", "document"}: + if not isinstance(block, dict) or _str(block.get("type")) not in {"image", "document"}: return None source = block.get("source") if not isinstance(source, dict) or source.get("type") != "base64": @@ -477,7 +477,7 @@ def extract_codex_blobs( refs = _rewrite_codex_items(payload["content"], collector) if record_keys is not None and refs: index.add_record(record_keys[position], refs) - elif payload_type in {"function_call_output", "custom_tool_call_output"}: + elif _str(payload_type) in {"function_call_output", "custom_tool_call_output"}: output = payload.get("output") if isinstance(output, list): index.add_tool_result( @@ -487,6 +487,16 @@ def extract_codex_blobs( return index +def _str(value: object) -> str | None: + """``value`` when it is a string, else ``None``: a set-membership key that cannot raise. + + A transcript field can hold any JSON value, and an object or a list is unhashable, so + testing one against a set raised ``TypeError`` past the converter (found by the fuzzer). + A non-string value never matched one of these string sets, so no result changes. + """ + return value if isinstance(value, str) else None + + __all__ = [ "CODEX_TOOL_IMAGE_MEDIA_TYPES", "Blob", diff --git a/packages/atif-converter/src/atif_converter/domain/codex_enrichment.py b/packages/atif-converter/src/atif_converter/domain/codex_enrichment.py index 828ddf7..4afbdc1 100644 --- a/packages/atif-converter/src/atif_converter/domain/codex_enrichment.py +++ b/packages/atif-converter/src/atif_converter/domain/codex_enrichment.py @@ -194,7 +194,7 @@ def _index_rollout(records: list[tuple[dict[str, Any], str]]) -> _RolloutIndex: pending_reasoning = record_id continue - if item_type in _REASONING_CONSUMERS and pending_reasoning is not None: + if _str(item_type) in _REASONING_CONSUMERS and pending_reasoning is not None: key = payload.get("call_id") if "call" in str(item_type) else record_id index.reasoning_for[str(key) if isinstance(key, str) and key else record_id] = ( pending_reasoning @@ -219,7 +219,7 @@ def _index_rollout(records: list[tuple[dict[str, Any], str]]) -> _RolloutIndex: call_id = payload.get("call_id") if isinstance(call_id, str) and call_id: index.by_call_id.setdefault(call_id, []).append(record_id) - if item_type in {"function_call", "custom_tool_call"}: + if _str(item_type) in {"function_call", "custom_tool_call"}: pending_call_api[call_id] = current_api_call saw_model_output = True @@ -376,7 +376,7 @@ def _attribute_non_agent_messages( if not isinstance(step, dict): continue source = step.get("source") - if source not in {"user", "system"}: + if _str(source) not in {"user", "system"}: continue if cursor >= len(messages): return _step_id(step), 0 @@ -407,3 +407,13 @@ def _record_cache_total(trajectory: dict[str, Any], extra: dict[str, Any]) -> No total = metrics_extra.get("total_cache_write_input_tokens") if isinstance(total, int): extra["cache_creation_total"] = total + + +def _str(value: object) -> str | None: + """``value`` when it is a string, else ``None``: a set-membership key that cannot raise. + + A transcript field can hold any JSON value, and an object or a list is unhashable, so + testing one against a set raised ``TypeError`` past the converter (found by the fuzzer). + A non-string value never matched one of these string sets, so no result changes. + """ + return value if isinstance(value, str) else None diff --git a/packages/atif-converter/src/atif_converter/domain/result_signals.py b/packages/atif-converter/src/atif_converter/domain/result_signals.py index 7549169..533b91e 100644 --- a/packages/atif-converter/src/atif_converter/domain/result_signals.py +++ b/packages/atif-converter/src/atif_converter/domain/result_signals.py @@ -235,7 +235,7 @@ def _index_claude_records( def _claude_exit_code(tool_name: str | None, facts: _ClaudeResult) -> int | None: - if tool_name not in _EXIT_CODE_TOOLS or not facts.is_error: + if _str(tool_name) not in _EXIT_CODE_TOOLS or not facts.is_error: return None for text in (facts.first_text, facts.tool_use_result): if isinstance(text, str): @@ -319,7 +319,11 @@ def _subagent_entries( if not isinstance(tool_use_result, dict): continue agent_id = tool_use_result.get("agentId") - if isinstance(agent_id, str) and agent_id and tool_names.get(call_id) in {"Task", "Agent"}: + if ( + isinstance(agent_id, str) + and agent_id + and _str(tool_names.get(call_id)) in {"Task", "Agent"} + ): from_results.setdefault(agent_id, (call_id, tool_use_result)) entries: list[dict[str, Any]] = [] @@ -429,7 +433,7 @@ def _item_outcome(item: dict[str, Any]) -> _CodexOutcome | None: return _command_outcome(item) if item_type == "McpToolCall": return _mcp_outcome(item) - if item_type in _STATUS_ITEM_TYPES: + if _str(item_type) in _STATUS_ITEM_TYPES: return _status_outcome(item) return None @@ -604,9 +608,9 @@ def _index_codex_records(records: Iterable[Any]) -> _CodexIndex: record_type = record.get("type") if record_type == "event_msg" and payload_type == "item_completed": index.item(payload) - elif record_type == "response_item" and payload_type in _CODEX_CALL_TYPES: + elif record_type == "response_item" and _str(payload_type) in _CODEX_CALL_TYPES: index.call(payload) - elif record_type == "response_item" and payload_type in _CODEX_OUTPUT_TYPES: + elif record_type == "response_item" and _str(payload_type) in _CODEX_OUTPUT_TYPES: index.output(payload) return index @@ -754,6 +758,16 @@ def annotate_codex_trajectory( return trajectory +def _str(value: object) -> str | None: + """``value`` when it is a string, else ``None``: a set-membership key that cannot raise. + + A transcript field can hold any JSON value, and an object or a list is unhashable, so + testing one against a set raised ``TypeError`` past the converter (found by the fuzzer). + A non-string value never matched one of these string sets, so no result changes. + """ + return value if isinstance(value, str) else None + + __all__ = [ "annotate_claude_code_trajectory", "annotate_codex_trajectory", diff --git a/packages/atif-converter/tests/test_converter_schema_version.py b/packages/atif-converter/tests/test_converter_schema_version.py index 83b64e0..6d583a2 100644 --- a/packages/atif-converter/tests/test_converter_schema_version.py +++ b/packages/atif-converter/tests/test_converter_schema_version.py @@ -29,7 +29,7 @@ #: The reviewed pair. Update BOTH in the commit that changes converter code: #: the version only when output can change, the digest always. PINNED_VERSION = 9 -PINNED_SOURCE_DIGEST = "5ac8625584d3d5d0b8ec67168ab5b4efea324e0b08bf6510c8c5da87d1e36b89" +PINNED_SOURCE_DIGEST = "438a7d67f55f14b853242a682335aa2d89e378988d6997fa2a860dc5f9c04e1d" _SRC = Path(__file__).resolve().parents[1] / "src" / "atif_converter" From 435884e5ea18d2fa3d2b9e90856c7406b6cbd13c Mon Sep 17 00:00:00 2001 From: Laith Al-Saadoon Date: Sat, 10 Oct 2026 19:30:41 +0000 Subject: [PATCH 6/6] docs(release): describe the three main rulesets and the release push RELEASING.md described one ruleset named main and a maintainer bypass that let a local update through. The repository now has three rulesets on the default branch: main history (no bypass), main checks (deploy key only) and main review (deploy key, plus the admin role for pull requests only). Describe them, why they are split, how the maintainer merges, and the accurate fallback when the release deploy key is unavailable: cut the bump commit through a pull request and publish only the tag. Co-Authored-By: Claude Sonnet 5.5 --- RELEASING.md | 85 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 21 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 38d9377..0ae71be 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -371,9 +371,11 @@ the fat-wheel restructure above. Neither is on the table. | --- | --- | --- | | Environment `pypi` with **required reviewers** | Settings → Environments | The gate on the irreversible step. Without reviewers the environment exists, the OIDC claim matches, and the gate is decorative. Its name must match the `Environment name` on the trusted publisher. | | **Private vulnerability reporting** enabled | Settings → Advanced Security | `SECURITY.md` and `.github/ISSUE_TEMPLATE/config.yml` both route reports to `/security/advisories/new`, which 404s while this is off. | -| **Require review from Code Owners** on `main` | Branch protection or ruleset | `.github/CODEOWNERS` only *requests* review until this is on; with it off the file is a notification list. | +| **Require review from Code Owners** in the `main review` ruleset | Settings → Rules → Rulesets | `.github/CODEOWNERS` only *requests* review until this is on; with it off the file is a notification list. | | `default_workflow_permissions` = `read` | Settings → Actions | Every workflow here declares its own per-job permissions, so the default needs no write. | -| Ruleset `main` on the default branch | Settings → Rules → Rulesets | See below: no deletion, no force push, pull requests with two approvals and Code Owner review, stale approvals dismissed, the check jobs required on an up-to-date branch. | +| Ruleset `main history` on the default branch | Settings → Rules → Rulesets | Refuses deletion and non-fast-forward pushes. No bypass actor. See below. | +| Ruleset `main checks` on the default branch | Settings → Rules → Rulesets | The check jobs required on an up-to-date branch. The deploy key is the only bypass actor. See below. | +| Ruleset `main review` on the default branch | Settings → Rules → Rulesets | Pull requests with two approvals, Code Owner review, the latest push approved, review threads resolved, stale approvals dismissed. Bypass actors: the deploy key, and the repository admin role for pull requests only. See below. | | Environment `release` with secret `RELEASE_DEPLOY_KEY`, deployments from `main` only | Settings → Environments | The private half of the write deploy key `release.yml` pushes the bump commit and tag with. See below. | | **Immutable releases** off | Settings → General → Releases | `publish.yml` adds the signed assets after the release is published, which an immutable release refuses. | | Pages **Source = GitHub Actions** | Settings → Pages | `docs.yml` deploys through `actions/deploy-pages`, which needs the Actions source rather than a branch. `base: "/atif-sql/"` in `site/astro.config.ts` matches the resulting project-site path. | @@ -381,21 +383,40 @@ the fat-wheel restructure above. Neither is on the table. ### The release push and branch protection -`main` carries one ruleset, `main`, which OpenSSF Scorecard's Branch-Protection check reads: +The goal: every change to `main` is reviewed and checked, the history cannot be rewritten, and +a release still lands without a person pressing a button for the bump commit. `main` carries +three rulesets, which OpenSSF Scorecard's Branch-Protection check reads: -- deletion and non-fast-forward pushes are refused, for everyone; -- a change arrives through a pull request with two approvals, Code Owner review, the most - recent push approved, review threads resolved, and stale approvals dismissed on a new push; -- the `check.yml` jobs must pass on a branch that is up to date with `main`. +| Ruleset | Rules | Bypass | +| --- | --- | --- | +| `main history` | Deletion and non-fast-forward pushes are refused. | None. | +| `main checks` | `quality gates`, `coverage`, `codeql analysis`, `dependency diff review`, `build and gate the docs site` and `scanners` must pass on a branch that is up to date with `main`. | The deploy key, always. | +| `main review` | A pull request with two approvals, Code Owner review, the most recent push approved, review threads resolved, and stale approvals dismissed on a new push. | The deploy key, always. The repository admin role, for pull requests only. | + +The rulesets are split because a bypass skips every rule in its ruleset. The repository has +one maintainer, who cannot approve their own pull request, so the maintainer's bypass sits on +`main review` alone and works only while merging a pull request. `main checks` has no +maintainer bypass: CI stays required for everyone except the release push. Merge a pull +request as the maintainer with `gh pr merge --admin`, or the bypass checkbox in the web UI, +once the required checks are green. + +`fuzz` and the external link crawl are not required checks. `fuzz` runs only for pull +requests that touch the converter, the harnesses or its own workflow, so a required `fuzz` +would block every other pull request waiting on a check that never starts. The link crawl +depends on third-party hosts being reachable. -The repository has one maintainer, who cannot approve their own pull request, so the -maintainer's account is a bypass actor and merges without the approvals. The checks still run -on the pull request and the history rules still hold, because a bypass is recorded in the -ruleset's insights and nothing here force-pushes. +To read what is enforced on `main`, without changing anything: + +```bash +gh api repos/laithalsaadoon/atif-sql/rules/branches/main --jq '[.[].type]' +# required_status_checks, deletion, non_fast_forward, pull_request +``` `release.yml` pushes the bump commit and the tag to `main` directly. `GITHUB_TOKEN` pushes as `github-actions[bot]`, which no ruleset bypass list can name, so the push goes through a -**deploy key** instead, and deploy keys are the ruleset's second bypass actor. Setup, once: +**deploy key** instead. Deploy keys can be a bypass actor, so the key is one on `main checks` and +`main review`, and never on `main history`: even the release push cannot delete `main` or rewrite +its history. Setup, once: ```bash ssh-keygen -t ed25519 -N '' -C 'atif-sql release.yml' -f release_deploy_key @@ -414,19 +435,41 @@ Only a run of the `release` job on `main` can read the key, and `release.yml` re non-dry run without it before it checks anything out. To rotate it, repeat the setup and delete the old key under Settings → Deploy keys. -Without the key, cut the version locally; the maintainer's bypass lets the push through, and -the tool is the same one CI runs, from the same lockfile: +Without the key, `release.yml` cannot push, and a direct push from the maintainer meets +`main checks` and `main review` with no bypass. Either restore the key with the preceding +setup block, or cut the bump commit through a pull request and push only the tag. Tags are not +covered by these branch rulesets. The tool is the same one CI runs, from the same lockfile; +`--version-files-only` rewrites the version files and the changelog and runs the +`pre_bump_hooks`, which re-lock `uv.lock`, without making a commit or a tag: ```bash git switch main && git pull --ff-only -mise run check -uv run cz bump --changelog --check-consistency --annotated-tag -git push --follow-tags -gh release create "v$(uv run cz version --project)" --draft --verify-tag \ - --title "v$(uv run cz version --project)" \ - --notes-file <(uv run cz changelog --dry-run "$(uv run cz version --project)") +git switch -c release-prep +uv run cz bump --version-files-only --changelog --check-consistency --yes +VERSION="$(uv run cz version --project)" +git add -u && git add CHANGELOG.md +git commit -m "chore(release): $VERSION" +git push -u origin release-prep +gh pr create --fill ``` +When the checks are green, merge with `gh pr merge --admin`, then tag the merged commit and +open the draft release: + +```bash +git switch main && git pull --ff-only +VERSION="$(uv run cz version --project)" +git log -1 --format=%s # the release commit, "chore(release): " +git tag -a "v$VERSION" -m "chore(release): $VERSION" +git push origin "v$VERSION" +gh release create "v$VERSION" --draft --verify-tag \ + --title "v$VERSION" \ + --notes-file <(uv run cz changelog --dry-run "$VERSION") +``` + +Create the tag after the merge, not before: a squash or rebase merge gives the change a new +hash, and the tag must name the commit that is on `main`. + From there the flow is identical: review the draft, press Publish, approve `pypi`. ## Cutting a specific version @@ -466,7 +509,7 @@ an existing filename either way, so this cannot replace bytes. | `mise run lock:check` fails on the release commit | `pre_bump_hooks` is missing from `[tool.commitizen]`, so `uv.lock` still records the previous versions. | | `commitizen found no version-bumping commit since the last tag` | Nothing to release. Land a `feat:`/`fix:`, or dispatch with an explicit `increment`. | | `CurrentVersionNotFoundError: Current version ... is not found in ` | A `version_files` entry points at a file whose version was hand-edited. `--check-consistency` is doing its job. | -| `GH006: Protected branch update failed` | The push did not use the deploy key, or the key is not a bypass actor of the `main` ruleset. See "The release push and branch protection". | +| `GH006: Protected branch update failed` | The push did not use the deploy key, or the key is not a bypass actor of the `main checks` and `main review` rulesets. A maintainer's direct push gets this too: their bypass covers merging a pull request only. See "The release push and branch protection". | | `The release environment has no RELEASE_DEPLOY_KEY secret` | The deploy key setup has not happened yet. Same section. | | `gh release upload` fails with `already exists` | A re-dispatch of `publish.yml` for a release that already carries its assets. Nothing to do: the assets on the release are the ones first verified. | | 403 from PyPI naming a publisher it cannot find | On a first release, the pending publisher does not exist yet. Afterwards, the publisher does not match the claims. |