From 810fa034e264426252b5cb12d03778e4c27568da Mon Sep 17 00:00:00 2001 From: "Simplicio, Wesley (ext)" Date: Tue, 14 Jul 2026 07:25:33 -0300 Subject: [PATCH 1/6] ci: enforce distribution quality gate Refs #10 --- .coveragerc | 17 ++ .github/pull_request_template.md | 26 +++ .github/workflows/quality.yml | 254 +++++++++++++++++++++ .github/workflows/release.yml | 22 +- CHANGELOG.md | 15 ++ Formula/simplicio.rb | 2 +- QUALITY.md | 73 ++++++ SIMPLICIO_ECOSYSTEM.md | 4 +- VERSION.md | 6 +- benchmarks/baseline.json | 13 ++ npm/simplicio-installer/install.js | 51 +++-- npm/simplicio-installer/package.json | 2 +- npm/simplicio-unscoped/install.js | 51 +++-- npm/simplicio-unscoped/package.json | 2 +- npm/simplicio/install.js | 51 +++-- npm/simplicio/package.json | 2 +- pypi/simplicio/pyproject.toml | 4 +- scripts/__init__.py | 1 + scripts/benchmark_distribution.py | 110 +++++++++ scripts/flaky_check.py | 77 +++++++ scripts/quality_policy.py | 82 +++++++ scripts/run_python_tests.py | 67 ++++++ scripts/security_scan.py | 98 ++++++++ scripts/verify_distribution_consistency.py | 212 +++++++++-------- simplicio-update-manifest.json | 2 +- tests/node/installer-e2e.test.cjs | 23 ++ tests/node/installer-unit.test.cjs | 63 +++++ tests/test_distribution_consistency.py | 127 +++++++++++ tests/test_quality_helpers.py | 203 ++++++++++++++++ version.txt | 2 +- 30 files changed, 1477 insertions(+), 185 deletions(-) create mode 100644 .coveragerc create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/quality.yml create mode 100644 QUALITY.md create mode 100644 benchmarks/baseline.json create mode 100644 scripts/__init__.py create mode 100644 scripts/benchmark_distribution.py create mode 100644 scripts/flaky_check.py create mode 100644 scripts/quality_policy.py create mode 100644 scripts/run_python_tests.py create mode 100644 scripts/security_scan.py create mode 100644 tests/node/installer-e2e.test.cjs create mode 100644 tests/node/installer-unit.test.cjs create mode 100644 tests/test_distribution_consistency.py create mode 100644 tests/test_quality_helpers.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..45b26bb --- /dev/null +++ b/.coveragerc @@ -0,0 +1,17 @@ +[run] +branch = True +source = scripts +omit = + scripts/run_python_tests.py + +[report] +show_missing = True +skip_covered = False +exclude_also = + if __name__ == .__main__.: + +[xml] +output = artifacts/unit/python-coverage.xml + +[json] +output = artifacts/unit/python-coverage.json diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..9d93603 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,26 @@ +## Summary + + + +## Tracking issue + + + +## Quality evidence + +- [ ] `quality-gate` is green. +- [ ] A bug fix includes a regression test that fails before the fix. +- [ ] Coverage remains >=85% globally and >=90% for critical release-integrity code. +- [ ] Installer/package behavior was exercised in dry-run mode on the supported matrix. +- [ ] No test is skipped. If an external dependency requires a skip, the adjacent + `JUSTIFICATION:` comment links a time-boxed repository issue. +- [ ] Benchmark changes are within `benchmarks/baseline.json`, or this PR updates the baseline with + measured rationale. + +## Regression test + + + +## Exceptions + + diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml new file mode 100644 index 0000000..d644ebe --- /dev/null +++ b/.github/workflows/quality.yml @@ -0,0 +1,254 @@ +name: quality-gate + +on: + pull_request: + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: quality-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: lint + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Python syntax + run: python -m compileall -q scripts pypi/simplicio/simplicio tests + - name: Node syntax + run: | + node --check npm/simplicio/install.js + node --check npm/simplicio-installer/install.js + node --check npm/simplicio-unscoped/install.js + - name: Shell syntax + run: shellcheck install.sh scripts/tami-loop.sh + - name: PowerShell syntax + shell: pwsh + run: | + $tokens = $null + $errors = $null + [System.Management.Automation.Language.Parser]::ParseFile((Resolve-Path install.ps1), [ref]$tokens, [ref]$errors) | Out-Null + if ($errors.Count) { $errors | ForEach-Object { Write-Error $_.Message }; exit 1 } + - name: Ignored-test policy + run: python scripts/quality_policy.py --junit artifacts/lint/skip-policy.xml + - name: Upload lint evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: lint-reports + path: artifacts/lint + retention-days: 14 + + unit: + name: unit + runs-on: ubuntu-latest + timeout-minutes: 8 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Install coverage tool + run: python -m pip install coverage==7.6.12 + - name: Python tests with JUnit and coverage + run: | + coverage run --rcfile=.coveragerc scripts/run_python_tests.py --junit artifacts/unit/python-junit.xml + coverage report --fail-under=85 + coverage report --include=scripts/verify_distribution_consistency.py --fail-under=90 + coverage xml + coverage json + - name: Node installer coverage + run: >- + node --test --experimental-test-coverage + --test-coverage-include='npm/*/install.js' + --test-coverage-lines=85 --test-coverage-functions=90 + tests/node/installer-unit.test.cjs + - name: Node JUnit + if: always() + run: >- + node --test --test-reporter=junit + --test-reporter-destination=artifacts/unit/node-junit.xml + tests/node/installer-unit.test.cjs + - name: Upload unit evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: unit-reports + path: artifacts/unit + retention-days: 14 + + integration: + name: integration (${{ matrix.os }}, py${{ matrix.python }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 12 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python: ["3.11", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Strict distribution regression audit + run: >- + python scripts/verify_distribution_consistency.py --strict + --junit artifacts/integration/distribution-junit.xml + - name: Build Python distribution + run: | + python -m pip install build==1.2.2.post1 + python -m build --sdist --wheel --outdir artifacts/integration/dist pypi/simplicio + - name: Validate npm package manifests + run: | + npm pack --dry-run ./npm/simplicio + npm pack --dry-run ./npm/simplicio-installer + npm pack --dry-run ./npm/simplicio-unscoped + - name: Upload integration evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-${{ matrix.os }}-py${{ matrix.python }} + path: artifacts/integration + retention-days: 14 + + e2e: + name: e2e (${{ matrix.os }}, node${{ matrix.node }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + node: [20, 22, 24] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - name: Prepare report directory + run: >- + node -e "require('fs').mkdirSync('artifacts/e2e', { recursive: true })" + - name: Exercise real npm entrypoints without network + run: >- + node --test --test-reporter=junit + --test-reporter-destination=artifacts/e2e/junit.xml + tests/node/installer-e2e.test.cjs + - name: Upload E2E evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-${{ matrix.os }}-node${{ matrix.node }} + path: artifacts/e2e + retention-days: 14 + + security: + name: security + runs-on: ubuntu-latest + timeout-minutes: 8 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Deterministic secret and dangerous-code scan + run: python scripts/security_scan.py --junit artifacts/security/secret-scan-junit.xml + - name: Upload security evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: security-reports + path: artifacts/security + retention-days: 14 + + benchmark: + name: benchmark + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Enforce versioned regression budget + run: >- + python scripts/benchmark_distribution.py + --junit artifacts/benchmark/junit.xml + --json artifacts/benchmark/results.json + - name: Upload benchmark evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-reports + path: artifacts/benchmark + retention-days: 14 + + flaky: + name: flaky + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Repeat Python unit suite + run: >- + python scripts/flaky_check.py --attempts 3 + --junit artifacts/flaky/python-junit.xml -- + python -m unittest discover -s tests -p test_*.py + - name: Repeat Node unit suite + run: >- + python scripts/flaky_check.py --attempts 3 + --junit artifacts/flaky/node-junit.xml -- + node --test tests/node/installer-unit.test.cjs + - name: Upload flaky evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: flaky-reports + path: artifacts/flaky + retention-days: 14 + + quality-gate: + name: quality-gate + if: always() + runs-on: ubuntu-latest + needs: [lint, unit, integration, e2e, security, benchmark, flaky] + steps: + - name: Reject any missing or failed mandatory gate + if: >- + needs.lint.result != 'success' || + needs.unit.result != 'success' || + needs.integration.result != 'success' || + needs.e2e.result != 'success' || + needs.security.result != 'success' || + needs.benchmark.result != 'success' || + needs.flaky.result != 'success' + run: exit 1 + - name: Confirm merge evidence + run: echo "All mandatory quality gates passed." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b060ccd..4a5e8e5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,22 +27,24 @@ jobs: shell: pwsh run: echo "version=$((Get-Content simplicio-update-manifest.json | ConvertFrom-Json).version)" >> $env:GITHUB_OUTPUT - name: Stage release assets + shell: pwsh run: | - mkdir -p dist - cp simplicio.exe dist/simplicio-windows-x86_64.exe - cp simplicio dist/simplicio-macos-arm64 - if (Test-Path simplicio-darwin-x64) { cp simplicio-darwin-x64 dist/simplicio-darwin-x64 } - if (Test-Path simplicio-linux-x64) { cp simplicio-linux-x64 dist/simplicio-linux-x64 } - cp simplicio-update-manifest.json dist/ - cp SHA256SUMS dist/ || true + New-Item -ItemType Directory -Force -Path dist | Out-Null + Copy-Item simplicio.exe dist/simplicio-windows-x86_64.exe + Copy-Item simplicio dist/simplicio-macos-arm64 + if (Test-Path simplicio-darwin-x64) { Copy-Item simplicio-darwin-x64 dist/simplicio-darwin-x64 } + if (Test-Path simplicio-linux-x64) { Copy-Item simplicio-linux-x64 dist/simplicio-linux-x64 } + Copy-Item simplicio-update-manifest.json dist/ + Get-ChildItem dist -File | Sort-Object Name | ForEach-Object { + "$(($_ | Get-FileHash -Algorithm SHA256).Hash.ToLower()) *$($_.Name)" + } | Set-Content -Encoding ascii dist/SHA256SUMS - name: Create or update release uses: softprops/action-gh-release@v2 with: tag_name: v${{ steps.ver.outputs.version }} - name: "v${{ steps.ver.outputs.version }} — Public Beta (free until 2026-06-30)" + name: "v${{ steps.ver.outputs.version }} — Public Beta" body: | - Free public beta. All features unlocked until 2026-06-30; the - subscription gate re-engages automatically afterwards. + Free public beta. All features remain unlocked during the public-beta phase. Windows: `irm https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.ps1 | iex` macOS/Linux: `curl -fsSL https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.sh | sh` diff --git a/CHANGELOG.md b/CHANGELOG.md index a11e1b2..2fe21bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [Unreleased] + +### Added + +- Mandatory pull-request quality gate with separate lint, unit, integration, + E2E, security, benchmark, and flaky-test jobs; JUnit, coverage, package, and + failure evidence is retained as workflow artifacts. +- Regression coverage for distribution-version drift and safe network-free npm + installer dry runs across the supported OS and runtime matrix. + +### Fixed + +- Aligned public wrapper metadata with the canonical `3.5.2` update manifest + and made release checksums derive from the staged artifacts. + ## [1.6.1] - 2026-07-01 ### Fixed diff --git a/Formula/simplicio.rb b/Formula/simplicio.rb index 38aa28f..017dd1f 100644 --- a/Formula/simplicio.rb +++ b/Formula/simplicio.rb @@ -4,7 +4,7 @@ class Simplicio < Formula desc "AI coding agent that saves up to 96% on tokens" homepage "https://simpleti.com.br/simplicio/#start" - version "1.2.0" + version "3.5.2" license "Proprietary" on_macos do diff --git a/QUALITY.md b/QUALITY.md new file mode 100644 index 0000000..09d06ed --- /dev/null +++ b/QUALITY.md @@ -0,0 +1,73 @@ +# CI quality gate + +Issue #10 is enforced by `.github/workflows/quality.yml`. Every pull request and push to +`master` runs independent lint, unit, integration, end-to-end, security, benchmark, and flaky-test +jobs. The `quality-gate` aggregate job fails unless every mandatory job succeeds; configure branch +protection to require that exact check name. + +## Required evidence + +| Job | Contract | Expected duration | +| --- | --- | ---: | +| `lint` | Python/Node/PowerShell/shell syntax plus ignored-test policy | 2 minutes | +| `unit` | Python and Node unit tests, JUnit, global coverage >=85%, critical functions >=90% | 4 minutes | +| `integration` | Strict distribution audit and package builds on Linux, macOS, Windows; Python 3.11/3.13 | 8 minutes | +| `e2e` | Real npm CLI entrypoints in network-free dry-run mode; Node 20/22/24 on all three OSes | 6 minutes | +| `security` | High-confidence secret scan plus dynamic code/shell detection | 4 minutes | +| `benchmark` | Installer payload and audit-time budgets against `benchmarks/baseline.json` | 2 minutes | +| `flaky` | Three clean repetitions of Python and Node unit suites | 5 minutes | + +Matrix jobs run concurrently. The expected wall-clock target is 15 minutes. A run above 20 minutes +for three consecutive successful PRs requires a tracking issue and a baseline review. + +JUnit, coverage, benchmark, security, package-build, and flaky-attempt reports are uploaded with +`if: always()` and retained for 14 days, including on failure. + +## Coverage and regression policy + +- Global line/branch coverage over testable first-party Python quality code must stay at or above + 85%. `scripts/verify_distribution_consistency.py`, the critical release-integrity surface, must + stay at or above 90%. +- Node installer lines must stay at or above 85% and functions at or above 90%. +- Every bug fix must add a test that fails before the fix. The version-source and expired-beta + regressions that triggered issue #10 are frozen in `tests/test_distribution_consistency.py`. +- A test may be skipped only with an adjacent `JUSTIFICATION:` comment and a full + `https://github.com/wesleysimplicio/simplicio/issues/` URL. The linked issue must name an + owner, external dependency, and removal date no later than 30 days. `quality_policy.py` blocks + unregistered skips. +- Retries never hide failure: `flaky_check.py` records every attempt, labels mixed outcomes FLAKY, + and returns non-zero if any attempt fails. + +## Benchmark exceptions + +`benchmarks/baseline.json` is reviewed code. A deliberate payload/performance increase must update +the baseline in the same PR with measurements and rationale. Emergency diagnosis may set +`SIMPLICIO_BENCHMARK_TOLERANCE_PERCENT`, but merges still require a linked exception issue and an +updated baseline; the environment override is not configured on protected branches. + +## Local parity + +```text +python scripts/verify_distribution_consistency.py --strict +python scripts/quality_policy.py +coverage run --rcfile=.coveragerc scripts/run_python_tests.py --junit artifacts/unit/python-junit.xml +coverage report --fail-under=85 +coverage report --include=scripts/verify_distribution_consistency.py --fail-under=90 +node --test --experimental-test-coverage --test-coverage-lines=85 --test-coverage-functions=90 tests/node/installer-unit.test.cjs +node --test tests/node/installer-e2e.test.cjs +python scripts/security_scan.py +python scripts/benchmark_distribution.py +``` + +## Repository settings required after merge + +The workflow file cannot enable its own control plane. An administrator must: + +1. Enable GitHub Actions for `wesleysimplicio/simplicio`. +2. Update the active `master` ruleset to require pull requests and the `quality-gate` status check, + require branches to be up to date, block force-push/deletion, and disallow bypass. +3. Confirm one PR exercises all jobs, artifacts are downloadable, and direct merge is rejected when + any mandatory job fails. + +Exceptions are never made by disabling a job or lowering thresholds. Use a linked, time-boxed issue; +keep `quality-gate` required and fix or formally revise the versioned contract through review. diff --git a/SIMPLICIO_ECOSYSTEM.md b/SIMPLICIO_ECOSYSTEM.md index d975727..3ea8c83 100644 --- a/SIMPLICIO_ECOSYSTEM.md +++ b/SIMPLICIO_ECOSYSTEM.md @@ -7,11 +7,11 @@ Consumidores finais via `npx @wesleysimplicio/simplicio`. - [simplicio-runtime](https://github.com/wesleysimplicio/simplicio-runtime) — binário compilado incluído no release ## Versão atual -1.2.0 (package.json) +3.5.2 (simplicio-update-manifest.json) ## Versão mínima esperada pelos dependentes Nenhuma — repo é ponto de entrada para usuários finais. --- -_Last updated: 2026-06-30_ +_Last updated: 2026-07-14_ diff --git a/VERSION.md b/VERSION.md index 5d4e172..f74e113 100644 --- a/VERSION.md +++ b/VERSION.md @@ -8,10 +8,10 @@ This is the **public distribution repo** for [Simplicio](https://github.com/wesl **Not the source code.** The Rust runtime source lives in the private [`simplicio-runtime`](https://github.com/wesleysimplicio/simplicio-runtime) repo. -## Current Version: v3.0.2 +## Current Version: v3.5.2 -- **Release:** v3.0.2 — Omnicoder HBP fabric, Guardians CLI, Parakeet STT, provider cleanup -- **Previous release:** v1.2.0 — Universal Adapter + 58 tests + Hermes parity +- **Release:** v3.5.2 — signed macOS artifact and verified update manifest +- **Previous release:** v3.0.2 — Omnicoder HBP fabric, Guardians CLI, Parakeet STT, provider cleanup - **Default branch:** `master` - **Last release asset:** macOS (ARM/x86_64), Linux (x86_64), Windows (x86_64) diff --git a/benchmarks/baseline.json b/benchmarks/baseline.json new file mode 100644 index 0000000..d2ef9bc --- /dev/null +++ b/benchmarks/baseline.json @@ -0,0 +1,13 @@ +{ + "schema": "simplicio.distribution-benchmark/v1", + "metrics": { + "installer_payload_bytes": { + "baseline": 16522, + "max_regression_percent": 10 + }, + "audit_median_ms": { + "baseline": 25, + "max_regression_percent": 400 + } + } +} diff --git a/npm/simplicio-installer/install.js b/npm/simplicio-installer/install.js index b8353fe..5f0c4ed 100644 --- a/npm/simplicio-installer/install.js +++ b/npm/simplicio-installer/install.js @@ -2,27 +2,36 @@ const { execSync } = require('child_process'); const os = require('os'); -const platform = os.platform(); -const isWin = platform === 'win32'; +function installerCommand(platform = os.platform()) { + return platform === 'win32' + ? 'powershell -c "irm https://simpleti.com.br/simplicio/install.ps1 | iex"' + : 'curl -fsSL https://simpleti.com.br/simplicio/install.sh | sh'; +} -try { - if (isWin) { - console.log('Downloading and running Simplicio installer for Windows...'); - execSync( - 'powershell -c "irm https://simpleti.com.br/simplicio/install.ps1 | iex"', - { stdio: 'inherit' } - ); - } else { - console.log('Downloading and running Simplicio installer for macOS/Linux...'); - execSync( - 'curl -fsSL https://simpleti.com.br/simplicio/install.sh | sh', - { stdio: 'inherit' } - ); +function main({ + platform = os.platform(), + execute = execSync, + dryRun = process.env.SIMPLICIO_INSTALL_DRY_RUN === '1', + log = console, +} = {}) { + const command = installerCommand(platform); + log.log(`Downloading and running Simplicio installer for ${platform === 'win32' ? 'Windows' : 'macOS/Linux'}...`); + if (dryRun) { + log.log(`[DRY RUN] ${command}`); + return 0; + } + try { + execute(command, { stdio: 'inherit' }); + log.log('\n✅ Simplicio installed successfully!'); + log.log(' Run: simplicio chat'); + log.log(' Docs: https://simpleti.com.br/simplicio/#start'); + return 0; + } catch (err) { + log.error('\n❌ Installation failed:', err.message); + return 1; } - console.log('\n✅ Simplicio installed successfully!'); - console.log(' Run: simplicio chat'); - console.log(' Docs: https://simpleti.com.br/simplicio/#start'); -} catch (err) { - console.error('\n❌ Installation failed:', err.message); - process.exit(1); } + +if (require.main === module) process.exitCode = main(); + +module.exports = { installerCommand, main }; diff --git a/npm/simplicio-installer/package.json b/npm/simplicio-installer/package.json index b595c70..b88fc1f 100644 --- a/npm/simplicio-installer/package.json +++ b/npm/simplicio-installer/package.json @@ -1,6 +1,6 @@ { "name": "simplicio-installer", - "version": "3.0.2", + "version": "3.5.2", "description": "Simplicio installer wrapper for npm.", "bin": { "simplicio-installer": "./install.js" diff --git a/npm/simplicio-unscoped/install.js b/npm/simplicio-unscoped/install.js index 7369e37..5f0c4ed 100644 --- a/npm/simplicio-unscoped/install.js +++ b/npm/simplicio-unscoped/install.js @@ -2,27 +2,36 @@ const { execSync } = require('child_process'); const os = require('os'); -const platform = os.platform(); -const IS_WIN = platform === 'win32'; +function installerCommand(platform = os.platform()) { + return platform === 'win32' + ? 'powershell -c "irm https://simpleti.com.br/simplicio/install.ps1 | iex"' + : 'curl -fsSL https://simpleti.com.br/simplicio/install.sh | sh'; +} -try { - if (IS_WIN) { - console.log('Downloading and running Simplicio installer for Windows...'); - execSync( - 'powershell -c "irm https://simpleti.com.br/simplicio/install.ps1 | iex"', - { stdio: 'inherit' } - ); - } else { - console.log('Downloading and running Simplicio installer for macOS/Linux...'); - execSync( - 'curl -fsSL https://simpleti.com.br/simplicio/install.sh | sh', - { stdio: 'inherit' } - ); +function main({ + platform = os.platform(), + execute = execSync, + dryRun = process.env.SIMPLICIO_INSTALL_DRY_RUN === '1', + log = console, +} = {}) { + const command = installerCommand(platform); + log.log(`Downloading and running Simplicio installer for ${platform === 'win32' ? 'Windows' : 'macOS/Linux'}...`); + if (dryRun) { + log.log(`[DRY RUN] ${command}`); + return 0; + } + try { + execute(command, { stdio: 'inherit' }); + log.log('\n✅ Simplicio installed successfully!'); + log.log(' Run: simplicio chat'); + log.log(' Docs: https://simpleti.com.br/simplicio/#start'); + return 0; + } catch (err) { + log.error('\n❌ Installation failed:', err.message); + return 1; } - console.log('\n✅ Simplicio installed successfully!'); - console.log(' Run: simplicio chat'); - console.log(' Docs: https://simpleti.com.br/simplicio/#start'); -} catch (err) { - console.error('\n❌ Installation failed:', err.message); - process.exit(1); } + +if (require.main === module) process.exitCode = main(); + +module.exports = { installerCommand, main }; diff --git a/npm/simplicio-unscoped/package.json b/npm/simplicio-unscoped/package.json index 50fffb6..c196e02 100644 --- a/npm/simplicio-unscoped/package.json +++ b/npm/simplicio-unscoped/package.json @@ -1,6 +1,6 @@ { "name": "simplicio", - "version": "3.0.2", + "version": "3.5.2", "description": "Simplicio \u2014 up to 96% token savings on any LLM. One-command install.", "bin": { "simplicio": "./install.js" diff --git a/npm/simplicio/install.js b/npm/simplicio/install.js index 7369e37..5f0c4ed 100644 --- a/npm/simplicio/install.js +++ b/npm/simplicio/install.js @@ -2,27 +2,36 @@ const { execSync } = require('child_process'); const os = require('os'); -const platform = os.platform(); -const IS_WIN = platform === 'win32'; +function installerCommand(platform = os.platform()) { + return platform === 'win32' + ? 'powershell -c "irm https://simpleti.com.br/simplicio/install.ps1 | iex"' + : 'curl -fsSL https://simpleti.com.br/simplicio/install.sh | sh'; +} -try { - if (IS_WIN) { - console.log('Downloading and running Simplicio installer for Windows...'); - execSync( - 'powershell -c "irm https://simpleti.com.br/simplicio/install.ps1 | iex"', - { stdio: 'inherit' } - ); - } else { - console.log('Downloading and running Simplicio installer for macOS/Linux...'); - execSync( - 'curl -fsSL https://simpleti.com.br/simplicio/install.sh | sh', - { stdio: 'inherit' } - ); +function main({ + platform = os.platform(), + execute = execSync, + dryRun = process.env.SIMPLICIO_INSTALL_DRY_RUN === '1', + log = console, +} = {}) { + const command = installerCommand(platform); + log.log(`Downloading and running Simplicio installer for ${platform === 'win32' ? 'Windows' : 'macOS/Linux'}...`); + if (dryRun) { + log.log(`[DRY RUN] ${command}`); + return 0; + } + try { + execute(command, { stdio: 'inherit' }); + log.log('\n✅ Simplicio installed successfully!'); + log.log(' Run: simplicio chat'); + log.log(' Docs: https://simpleti.com.br/simplicio/#start'); + return 0; + } catch (err) { + log.error('\n❌ Installation failed:', err.message); + return 1; } - console.log('\n✅ Simplicio installed successfully!'); - console.log(' Run: simplicio chat'); - console.log(' Docs: https://simpleti.com.br/simplicio/#start'); -} catch (err) { - console.error('\n❌ Installation failed:', err.message); - process.exit(1); } + +if (require.main === module) process.exitCode = main(); + +module.exports = { installerCommand, main }; diff --git a/npm/simplicio/package.json b/npm/simplicio/package.json index d0e5a36..9f0b1b8 100644 --- a/npm/simplicio/package.json +++ b/npm/simplicio/package.json @@ -1,6 +1,6 @@ { "name": "@wesleysimplicio/simplicio", - "version": "3.0.2", + "version": "3.5.2", "description": "Simplicio CLI \u2014 up to 96% token savings on any LLM. One command install.", "bin": { "simplicio": "./install.js" diff --git a/pypi/simplicio/pyproject.toml b/pypi/simplicio/pyproject.toml index 3ef5bff..8b47f8f 100644 --- a/pypi/simplicio/pyproject.toml +++ b/pypi/simplicio/pyproject.toml @@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta" [project] name = "simplicio-installer" -version = "3.0.2" +version = "3.5.2" description = "Simplicio — AI coding agent that saves up to 96% on tokens" readme = "README.md" requires-python = ">=3.7" -license = {text = "Proprietary"} +license = "LicenseRef-Proprietary" keywords = ["ai", "coding-agent", "cli", "token-savings"] [project.urls] diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e38ad13 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""CI quality-gate helpers.""" diff --git a/scripts/benchmark_distribution.py b/scripts/benchmark_distribution.py new file mode 100644 index 0000000..0eb9e59 --- /dev/null +++ b/scripts/benchmark_distribution.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Compare deterministic distribution metrics with a versioned baseline.""" + +from __future__ import annotations + +import argparse +import json +import os +import statistics +import sys +import time +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping, Sequence + +try: + from .verify_distribution_consistency import run_audit +except ImportError: # direct script execution + from verify_distribution_consistency import run_audit + +ROOT = Path(__file__).resolve().parents[1] +PAYLOAD_FILES = ( + "install.sh", + "install.ps1", + "npm/simplicio/install.js", + "npm/simplicio-installer/install.js", + "npm/simplicio-unscoped/install.js", + "pypi/simplicio/simplicio/__main__.py", +) + + +@dataclass(frozen=True) +class MetricResult: + name: str + measured: float + maximum: float + passed: bool + + +def payload_bytes(root: Path = ROOT) -> int: + return sum((root / relative).stat().st_size for relative in PAYLOAD_FILES) + + +def audit_median_ms(root: Path = ROOT, repetitions: int = 5) -> float: + samples: list[float] = [] + for _ in range(repetitions): + started = time.perf_counter() + findings = run_audit(root) + if any(item.level in {"ERROR", "WARN"} for item in findings): + raise RuntimeError("distribution audit is not clean") + samples.append((time.perf_counter() - started) * 1000) + return statistics.median(samples) + + +def evaluate(measured: Mapping[str, float], baseline: Mapping[str, dict], override: float | None = None) -> list[MetricResult]: + results: list[MetricResult] = [] + for name, value in measured.items(): + contract = baseline[name] + tolerance = float(override if override is not None else contract["max_regression_percent"]) + maximum = float(contract["baseline"]) * (1 + tolerance / 100) + results.append(MetricResult(name, float(value), maximum, float(value) <= maximum)) + return results + + +def write_junit(path: Path, results: Sequence[MetricResult]) -> None: + suite = ET.Element( + "testsuite", + name="distribution-benchmark", + tests=str(len(results)), + failures=str(sum(not result.passed for result in results)), + ) + for result in results: + case = ET.SubElement(suite, "testcase", name=result.name) + detail = f"measured={result.measured:.3f}; maximum={result.maximum:.3f}" + if result.passed: + ET.SubElement(case, "system-out").text = detail + else: + ET.SubElement(case, "failure", message="regression budget exceeded").text = detail + path.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--baseline", type=Path, default=ROOT / "benchmarks/baseline.json") + parser.add_argument("--repetitions", type=int, default=5) + parser.add_argument("--junit", type=Path) + parser.add_argument("--json", type=Path) + args = parser.parse_args(argv) + contract = json.loads(args.baseline.read_text(encoding="utf-8"))["metrics"] + measured = { + "installer_payload_bytes": payload_bytes(args.root), + "audit_median_ms": audit_median_ms(args.root, args.repetitions), + } + override_value = os.environ.get("SIMPLICIO_BENCHMARK_TOLERANCE_PERCENT") + results = evaluate(measured, contract, float(override_value) if override_value else None) + payload = [result.__dict__ for result in results] + print(json.dumps(payload, indent=2)) + if args.json: + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + if args.junit: + write_junit(args.junit, results) + return int(any(not result.passed for result in results)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/flaky_check.py b/scripts/flaky_check.py new file mode 100644 index 0000000..77c072d --- /dev/null +++ b/scripts/flaky_check.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Repeat a test command; any inconsistent or failed attempt blocks the gate.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import time +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + + +@dataclass(frozen=True) +class Attempt: + number: int + returncode: int + duration_seconds: float + output: str + + +def run_attempts(command: Sequence[str], attempts: int) -> list[Attempt]: + results: list[Attempt] = [] + for number in range(1, attempts + 1): + started = time.perf_counter() + process = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) + results.append(Attempt(number, process.returncode, time.perf_counter() - started, process.stdout)) + return results + + +def classification(results: Sequence[Attempt]) -> str: + codes = {result.returncode for result in results} + if codes == {0}: + return "PASS" + if 0 in codes: + return "FLAKY" + return "FAIL" + + +def write_junit(path: Path, results: Sequence[Attempt], verdict: str) -> None: + suite = ET.Element("testsuite", name="flaky-check", tests=str(len(results)), failures=str(verdict != "PASS")) + for result in results: + case = ET.SubElement( + suite, + "testcase", + name=f"attempt-{result.number}", + time=f"{result.duration_seconds:.6f}", + ) + ET.SubElement(case, "system-out").text = result.output + if result.returncode: + ET.SubElement(case, "failure", message=f"exit {result.returncode}") + path.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--attempts", type=int, default=3) + parser.add_argument("--junit", type=Path, required=True) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args(argv) + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if args.attempts < 2 or not command: + parser.error("provide a command and at least two attempts") + results = run_attempts(command, args.attempts) + verdict = classification(results) + write_junit(args.junit, results, verdict) + for result in results: + print(f"attempt={result.number} exit={result.returncode} duration={result.duration_seconds:.3f}s") + print(f"flaky-check: {verdict}") + return int(verdict != "PASS") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/quality_policy.py b/scripts/quality_policy.py new file mode 100644 index 0000000..be0020f --- /dev/null +++ b/scripts/quality_policy.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Reject ignored tests unless an issue-linked justification is adjacent.""" + +from __future__ import annotations + +import argparse +import re +import sys +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Sequence + +ROOT = Path(__file__).resolve().parents[1] +ISSUE_URL = "https://github.com/wesleysimplicio/simplicio/issues/" +SKIP_PATTERNS = ( + re.compile(r"@(?:unittest\.)?skip(?:If|Unless)?\b"), + re.compile(r"pytest\.skip\s*\("), + re.compile(r"pytest\.mark\.skip"), + re.compile(r"\b(?:test|it|describe)\.skip\s*\("), +) + + +@dataclass(frozen=True) +class Violation: + path: str + line: int + text: str + + +def test_files(root: Path) -> Iterable[Path]: + tests = root / "tests" + if not tests.exists(): + return [] + return ( + path + for path in tests.rglob("*") + if path.is_file() and path.suffix.lower() in {".py", ".js", ".cjs", ".mjs", ".ts"} + ) + + +def find_violations(root: Path = ROOT) -> list[Violation]: + violations: list[Violation] = [] + for path in test_files(root): + lines = path.read_text(encoding="utf-8").splitlines() + for index, line in enumerate(lines): + if not any(pattern.search(line) for pattern in SKIP_PATTERNS): + continue + context = "\n".join(lines[max(0, index - 2) : index + 1]) + if "JUSTIFICATION:" not in context or ISSUE_URL not in context: + violations.append(Violation(path.relative_to(root).as_posix(), index + 1, line.strip())) + return violations + + +def write_junit(path: Path, violations: Sequence[Violation]) -> None: + suite = ET.Element("testsuite", name="skip-policy", tests="1", failures=str(bool(violations))) + case = ET.SubElement(suite, "testcase", name="all-skips-have-issue-linked-justification") + if violations: + detail = "\n".join(f"{item.path}:{item.line}: {item.text}" for item in violations) + ET.SubElement(case, "failure", message="unjustified ignored tests").text = detail + path.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--junit", type=Path) + args = parser.parse_args(argv) + violations = find_violations(args.root) + if args.junit: + write_junit(args.junit, violations) + if violations: + for item in violations: + print(f"[ERROR] {item.path}:{item.line}: ignored test lacks JUSTIFICATION + issue URL") + return 1 + print("skip-policy: PASS (no unjustified ignored tests)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_python_tests.py b/scripts/run_python_tests.py new file mode 100644 index 0000000..200305d --- /dev/null +++ b/scripts/run_python_tests.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Run unittest discovery and emit a dependency-free JUnit report.""" + +from __future__ import annotations + +import argparse +import sys +import time +import unittest +import xml.etree.ElementTree as ET +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +class RecordingResult(unittest.TextTestResult): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.successes: list[unittest.case.TestCase] = [] + + def addSuccess(self, test): # noqa: N802 - unittest API + self.successes.append(test) + super().addSuccess(test) + + +def write_junit(path: Path, result: RecordingResult, duration: float) -> None: + suite = ET.Element( + "testsuite", + name="python-unit", + tests=str(result.testsRun), + failures=str(len(result.failures)), + errors=str(len(result.errors)), + skipped=str(len(result.skipped)), + time=f"{duration:.6f}", + ) + outcomes = {str(test): ("failure", detail) for test, detail in result.failures} + outcomes.update({str(test): ("error", detail) for test, detail in result.errors}) + skipped = {str(test): reason for test, reason in result.skipped} + all_tests = list(result.successes) + [test for test, _ in result.failures + result.errors + result.skipped] + for test in all_tests: + case = ET.SubElement(suite, "testcase", name=str(test)) + if str(test) in outcomes: + kind, detail = outcomes[str(test)] + ET.SubElement(case, kind, message=kind).text = detail + elif str(test) in skipped: + ET.SubElement(case, "skipped", message=skipped[str(test)]) + path.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--start-directory", default="tests") + parser.add_argument("--pattern", default="test_*.py") + parser.add_argument("--junit", type=Path, required=True) + args = parser.parse_args() + suite = unittest.defaultTestLoader.discover(args.start_directory, pattern=args.pattern) + started = time.perf_counter() + result = unittest.TextTestRunner(verbosity=2, resultclass=RecordingResult).run(suite) + write_junit(args.junit, result, time.perf_counter() - started) + return int(not result.wasSuccessful()) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/security_scan.py b/scripts/security_scan.py new file mode 100644 index 0000000..819965e --- /dev/null +++ b/scripts/security_scan.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Small deterministic secret scanner for tracked text distribution files.""" + +from __future__ import annotations + +import argparse +import ast +import re +import subprocess +import sys +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Sequence + +ROOT = Path(__file__).resolve().parents[1] +TEXT_SUFFIXES = {".js", ".json", ".md", ".ps1", ".py", ".rb", ".sh", ".toml", ".txt", ".yml", ".yaml"} +PATTERNS = ( + ("aws-access-key", re.compile(r"AKIA[0-9A-Z]{16}")), + ("github-token", re.compile(r"(?:ghp|github_pat)_[A-Za-z0-9_]{30,}")), + ("openai-key", re.compile(r"sk-[A-Za-z0-9]{32,}")), + ("private-key", re.compile("-" * 5 + r"BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY")), +) + + +@dataclass(frozen=True) +class SecretFinding: + path: str + line: int + kind: str + + +def tracked_text_files(root: Path = ROOT) -> Iterable[Path]: + output = subprocess.check_output( + ["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"], cwd=root + ) + for raw in output.split(b"\0"): + if not raw: + continue + relative = Path(raw.decode("utf-8")) + if relative.parts and relative.parts[0] in {".simplicio", "artifacts"}: + continue + path = root / relative + if path.suffix.lower() in TEXT_SUFFIXES and path.is_file(): + yield path + + +def scan(root: Path = ROOT) -> list[SecretFinding]: + findings: list[SecretFinding] = [] + for path in tracked_text_files(root): + text = path.read_text(encoding="utf-8", errors="replace") + for number, line in enumerate(text.splitlines(), 1): + for kind, pattern in PATTERNS: + if pattern.search(line): + findings.append(SecretFinding(path.relative_to(root).as_posix(), number, kind)) + if path.suffix.lower() == ".py": + tree = ast.parse(text, filename=str(path)) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if isinstance(node.func, ast.Name) and node.func.id in {"eval", "exec"}: + findings.append(SecretFinding(path.relative_to(root).as_posix(), node.lineno, "dynamic-code-execution")) + shell_enabled = any( + keyword.arg == "shell" and isinstance(keyword.value, ast.Constant) and keyword.value.value is True + for keyword in node.keywords + ) + if shell_enabled and node.args and not isinstance(node.args[0], ast.Constant): + findings.append(SecretFinding(path.relative_to(root).as_posix(), node.lineno, "dynamic-shell-command")) + return findings + + +def write_junit(path: Path, findings: Sequence[SecretFinding]) -> None: + suite = ET.Element("testsuite", name="secret-scan", tests="1", failures=str(bool(findings))) + case = ET.SubElement(suite, "testcase", name="tracked-text-has-no-high-confidence-secrets") + if findings: + detail = "\n".join(f"{item.path}:{item.line}: {item.kind}" for item in findings) + ET.SubElement(case, "failure", message="potential secrets found").text = detail + path.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--junit", type=Path) + args = parser.parse_args(argv) + findings = scan(args.root) + if args.junit: + write_junit(args.junit, findings) + for finding in findings: + print(f"[ERROR] {finding.path}:{finding.line}: {finding.kind}") + if not findings: + print("secret-scan: PASS") + return int(bool(findings)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/verify_distribution_consistency.py b/scripts/verify_distribution_consistency.py index f9c3094..49ed7d2 100644 --- a/scripts/verify_distribution_consistency.py +++ b/scripts/verify_distribution_consistency.py @@ -1,47 +1,35 @@ #!/usr/bin/env python3 -"""Low-risk integrity checks for the public Simplicio distribution repo. - -Focuses on verifiable packaging/distribution drift: -- canonical branch install URLs (`master` in this repo) -- release/version source-of-truth mismatches -- stale or contradictory public-beta claims - -Exit code: -- 0: no hard failures (warnings may still be printed) -- 1: at least one hard failure -""" +"""Fail-closed integrity audit for the public Simplicio distribution.""" from __future__ import annotations +import argparse import json import re import sys -from dataclasses import dataclass +import xml.etree.ElementTree as ET +from dataclasses import asdict, dataclass from datetime import date from pathlib import Path -from typing import Iterable +from typing import Iterable, Sequence ROOT = Path(__file__).resolve().parents[1] CANONICAL_BRANCH = "master" MAIN_INSTALL_RE = re.compile( r"https://raw\.githubusercontent\.com/wesleysimplicio/simplicio/main/install\.(?:sh|ps1)" ) -VERSION_RE = re.compile(r'"version"\s*:\s*"([^"]+)"') FORMULA_VERSION_RE = re.compile(r'version\s+"([^"]+)"') BETA_NO_END_RE = re.compile(r"public beta with no end date", re.IGNORECASE) ECOSYSTEM_VERSION_RE = re.compile(r"## Versão atual\s+([^\n]+)", re.MULTILINE) +CURRENT_VERSION_RE = re.compile(r"## Current Version:\s*v([^\s]+)") -@dataclass +@dataclass(frozen=True) class Finding: - level: str # ERROR | WARN | OK + level: str message: str -def rel(path: Path) -> str: - return str(path.relative_to(ROOT)) - - def read_text(path: Path) -> str: return path.read_text(encoding="utf-8") @@ -51,7 +39,7 @@ def load_json(path: Path) -> dict: def version_from_package_json(path: Path) -> str: - return load_json(path)["version"] + return str(load_json(path)["version"]) def version_from_formula(path: Path) -> str: @@ -62,127 +50,153 @@ def version_from_formula(path: Path) -> str: def version_from_pyproject(path: Path) -> str: - text = read_text(path) - match = re.search(r'^version\s*=\s*"([^"]+)"', text, re.MULTILINE) + match = re.search(r'^version\s*=\s*"([^"]+)"', read_text(path), re.MULTILINE) if not match: raise ValueError(f"could not parse pyproject version from {path}") return match.group(1) -def iter_install_reference_files() -> Iterable[Path]: - yield ROOT / "README.md" - yield ROOT / "INSTALL.md" - yield ROOT / "install.sh" - yield ROOT / "install.ps1" - yield ROOT / ".github/workflows/release.yml" - yield ROOT / "pypi/simplicio/simplicio/__main__.py" - for path in sorted((ROOT / "READMEs").glob("README*.md")): - yield path +def iter_install_reference_files(root: Path) -> Iterable[Path]: + fixed = ( + "README.md", + "INSTALL.md", + "install.sh", + "install.ps1", + ".github/workflows/release.yml", + "pypi/simplicio/simplicio/__main__.py", + ) + yield from (root / relative for relative in fixed) + yield from sorted((root / "READMEs").glob("README*.md")) -def main() -> int: +def run_audit(root: Path = ROOT, *, today: date | None = None) -> list[Finding]: + today = today or date.today() findings: list[Finding] = [] - version_md = read_text(ROOT / "VERSION.md") - if "Use `master` branch only" not in version_md: + version_document = read_text(root / "VERSION.md") + if "Use `master` branch only" not in version_document: findings.append(Finding("ERROR", "VERSION.md no longer declares `master` as canonical branch.")) - offenders = [] - for path in iter_install_reference_files(): - text = read_text(path) - if MAIN_INSTALL_RE.search(text): - offenders.append(rel(path)) + offenders = [ + str(path.relative_to(root)) + for path in iter_install_reference_files(root) + if MAIN_INSTALL_RE.search(read_text(path)) + ] if offenders: findings.append( Finding( "ERROR", - "install references still point at `/main/` instead of canonical `/master/`: " + ", ".join(offenders), + "install references point at `/main/` instead of `/master/`: " + ", ".join(offenders), ) ) else: findings.append(Finding("OK", "all public install references use the canonical `master` branch.")) - version_txt = read_text(ROOT / "version.txt").strip() - manifest = load_json(ROOT / "simplicio-update-manifest.json") - manifest_version = manifest["version"] + version_txt = read_text(root / "version.txt").strip() + manifest = load_json(root / "simplicio-update-manifest.json") + manifest_version = str(manifest["version"]) if manifest_version != version_txt: findings.append( - Finding( - "ERROR", - f"version mismatch: version.txt={version_txt} but simplicio-update-manifest.json={manifest_version}", - ) + Finding("ERROR", f"version mismatch: version.txt={version_txt} but manifest={manifest_version}") ) else: findings.append(Finding("OK", f"release version sources agree on {version_txt}.")) - wrapper_versions = { - "Formula/simplicio.rb": version_from_formula(ROOT / "Formula/simplicio.rb"), - "npm/simplicio/package.json": version_from_package_json(ROOT / "npm/simplicio/package.json"), - "npm/simplicio-installer/package.json": version_from_package_json(ROOT / "npm/simplicio-installer/package.json"), - "npm/simplicio-unscoped/package.json": version_from_package_json(ROOT / "npm/simplicio-unscoped/package.json"), - "pypi/simplicio/pyproject.toml": version_from_pyproject(ROOT / "pypi/simplicio/pyproject.toml"), + documented = CURRENT_VERSION_RE.search(version_document) + if not documented or documented.group(1) != manifest_version: + value = documented.group(1) if documented else "missing" + findings.append(Finding("WARN", f"VERSION.md advertises {value} while manifest is {manifest_version}.")) + else: + findings.append(Finding("OK", "VERSION.md matches the release manifest.")) + + wrappers = { + "Formula/simplicio.rb": version_from_formula(root / "Formula/simplicio.rb"), + "npm/simplicio/package.json": version_from_package_json(root / "npm/simplicio/package.json"), + "npm/simplicio-installer/package.json": version_from_package_json( + root / "npm/simplicio-installer/package.json" + ), + "npm/simplicio-unscoped/package.json": version_from_package_json( + root / "npm/simplicio-unscoped/package.json" + ), + "pypi/simplicio/pyproject.toml": version_from_pyproject(root / "pypi/simplicio/pyproject.toml"), } - drift = {path: version for path, version in wrapper_versions.items() if version != manifest_version} + drift = {path: version for path, version in wrappers.items() if version != manifest_version} if drift: - joined = ", ".join(f"{path}={version}" for path, version in drift.items()) - findings.append( - Finding( - "WARN", - f"wrapper/package versions lag manifest {manifest_version}: {joined}", - ) - ) + details = ", ".join(f"{path}={version}" for path, version in drift.items()) + findings.append(Finding("WARN", f"wrapper versions lag manifest {manifest_version}: {details}")) else: findings.append(Finding("OK", "wrapper/package versions match the release manifest.")) - ecosystem_text = read_text(ROOT / "SIMPLICIO_ECOSYSTEM.md") - eco_match = ECOSYSTEM_VERSION_RE.search(ecosystem_text) - if eco_match and manifest_version not in eco_match.group(1): + ecosystem = read_text(root / "SIMPLICIO_ECOSYSTEM.md") + ecosystem_match = ECOSYSTEM_VERSION_RE.search(ecosystem) + if not ecosystem_match or manifest_version not in ecosystem_match.group(1): + advertised = ecosystem_match.group(1).strip() if ecosystem_match else "missing" findings.append( - Finding( - "WARN", - f"SIMPLICIO_ECOSYSTEM.md advertises `{eco_match.group(1).strip()}` while manifest is `{manifest_version}`.", - ) + Finding("WARN", f"SIMPLICIO_ECOSYSTEM.md advertises `{advertised}` while manifest is `{manifest_version}`.") ) + else: + findings.append(Finding("OK", "ecosystem documentation matches the release manifest.")) beta_until = manifest.get("entitlement", {}).get("beta_until") if beta_until: try: - beta_until_date = date.fromisoformat(beta_until) - if beta_until_date < date.today(): - findings.append( - Finding( - "WARN", - f"public-beta date is stale: beta_until={beta_until} is before today ({date.today().isoformat()}).", - ) - ) + beta_date = date.fromisoformat(str(beta_until)) + if beta_date < today: + findings.append(Finding("WARN", f"public-beta date {beta_until} is before {today.isoformat()}.")) except ValueError: findings.append(Finding("WARN", f"could not parse beta_until date: {beta_until}")) - - readme_text = read_text(ROOT / "README.md") - if beta_until and BETA_NO_END_RE.search(readme_text): - findings.append( - Finding( - "WARN", - f"README says 'public beta with no end date' but manifest still carries beta_until={beta_until}.", + if BETA_NO_END_RE.search(read_text(root / "README.md")): + findings.append( + Finding("WARN", f"README has no-end-date claim but manifest carries beta_until={beta_until}.") ) - ) - errors = [f for f in findings if f.level == "ERROR"] - warnings = [f for f in findings if f.level == "WARN"] - oks = [f for f in findings if f.level == "OK"] + return findings - print("Simplicio distribution consistency audit") - print(f"repo: {ROOT}") - print("") - for bucket in (errors, warnings, oks): - for finding in bucket: - prefix = {"ERROR": "[ERROR]", "WARN": "[WARN]", "OK": "[OK]"}[finding.level] - print(f"{prefix} {finding.message}") - print("") - print(f"summary: {len(errors)} error(s), {len(warnings)} warning(s), {len(oks)} ok") - return 1 if errors else 0 +def write_junit(path: Path, findings: Sequence[Finding]) -> None: + failures = [item for item in findings if item.level in {"ERROR", "WARN"}] + suite = ET.Element( + "testsuite", + name="distribution-consistency", + tests=str(len(findings)), + failures=str(len(failures)), + ) + for index, finding in enumerate(findings, 1): + case = ET.SubElement(suite, "testcase", name=f"finding-{index}-{finding.level.lower()}") + if finding.level in {"ERROR", "WARN"}: + ET.SubElement(case, "failure", message=finding.message).text = finding.message + else: + ET.SubElement(case, "system-out").text = finding.message + path.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) + + +def print_human(root: Path, findings: Sequence[Finding]) -> None: + print("Simplicio distribution consistency audit") + print(f"repo: {root}") + for finding in sorted(findings, key=lambda item: {"ERROR": 0, "WARN": 1, "OK": 2}[item.level]): + print(f"[{finding.level}] {finding.message}") + counts = {level: sum(item.level == level for item in findings) for level in ("ERROR", "WARN", "OK")} + print(f"summary: {counts['ERROR']} error(s), {counts['WARN']} warning(s), {counts['OK']} ok") + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--strict", action="store_true", help="treat warnings as failures") + parser.add_argument("--junit", type=Path) + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + findings = run_audit(args.root) + if args.junit: + write_junit(args.junit, findings) + if args.json: + print(json.dumps([asdict(item) for item in findings], indent=2)) + else: + print_human(args.root, findings) + blocked_levels = {"ERROR", "WARN"} if args.strict else {"ERROR"} + return int(any(item.level in blocked_levels for item in findings)) if __name__ == "__main__": - raise SystemExit(main()) + sys.exit(main()) diff --git a/simplicio-update-manifest.json b/simplicio-update-manifest.json index d2666c6..2954b52 100644 --- a/simplicio-update-manifest.json +++ b/simplicio-update-manifest.json @@ -10,7 +10,7 @@ }, "entitlement": { "current_phase": "public_beta", - "beta_until": "2026-06-30", + "beta_until": null, "updates_allowed": true }, "artifacts": [ diff --git a/tests/node/installer-e2e.test.cjs b/tests/node/installer-e2e.test.cjs new file mode 100644 index 0000000..2a7eb81 --- /dev/null +++ b/tests/node/installer-e2e.test.cjs @@ -0,0 +1,23 @@ +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const path = require('node:path'); +const test = require('node:test'); + +const installers = [ + 'npm/simplicio/install.js', + 'npm/simplicio-installer/install.js', + 'npm/simplicio-unscoped/install.js', +]; + +for (const relative of installers) { + test(`${relative} executes its real CLI entrypoint without network in dry-run mode`, () => { + const result = spawnSync(process.execPath, [path.resolve(relative)], { + cwd: path.resolve('.'), + env: { ...process.env, SIMPLICIO_INSTALL_DRY_RUN: '1' }, + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /\[DRY RUN\]/); + assert.match(result.stdout, process.platform === 'win32' ? /powershell/ : /curl/); + }); +} diff --git a/tests/node/installer-unit.test.cjs b/tests/node/installer-unit.test.cjs new file mode 100644 index 0000000..17e9e47 --- /dev/null +++ b/tests/node/installer-unit.test.cjs @@ -0,0 +1,63 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); + +const modules = [ + '../../npm/simplicio/install.js', + '../../npm/simplicio-installer/install.js', + '../../npm/simplicio-unscoped/install.js', +]; + +function logger() { + const messages = []; + return { + messages, + log: { log: (...parts) => messages.push(parts.join(' ')), error: (...parts) => messages.push(parts.join(' ')) }, + }; +} + +for (const path of modules) { + const installer = require(path); + + test(`${path} routes Windows and Unix installers`, () => { + assert.match(installer.installerCommand('win32'), /powershell.*install\.ps1/); + assert.match(installer.installerCommand('linux'), /curl.*install\.sh/); + assert.match(installer.installerCommand('darwin'), /curl.*install\.sh/); + }); + + test(`${path} dry-run never executes`, () => { + const capture = logger(); + const code = installer.main({ + platform: 'linux', + dryRun: true, + execute: () => assert.fail('execute must not run'), + log: capture.log, + }); + assert.equal(code, 0); + assert.ok(capture.messages.some((line) => line.includes('[DRY RUN]'))); + }); + + test(`${path} returns success after execution`, () => { + const capture = logger(); + let command; + const code = installer.main({ + platform: 'win32', + dryRun: false, + execute: (value) => { command = value; }, + log: capture.log, + }); + assert.equal(code, 0); + assert.match(command, /powershell/); + }); + + test(`${path} converts execution errors into a non-zero result`, () => { + const capture = logger(); + const code = installer.main({ + platform: 'linux', + dryRun: false, + execute: () => { throw new Error('synthetic failure'); }, + log: capture.log, + }); + assert.equal(code, 1); + assert.ok(capture.messages.some((line) => line.includes('synthetic failure'))); + }); +} diff --git a/tests/test_distribution_consistency.py b/tests/test_distribution_consistency.py new file mode 100644 index 0000000..fc2fd85 --- /dev/null +++ b/tests/test_distribution_consistency.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import contextlib +import io +import json +import tempfile +import unittest +from datetime import date +from pathlib import Path + +from scripts import verify_distribution_consistency as audit + + +class DistributionFixture: + def __init__(self, root: Path): + self.root = root + self.version = "3.5.2" + self.write() + + def put(self, relative: str, value: str) -> None: + path = self.root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(value, encoding="utf-8") + + def write(self) -> None: + self.put("VERSION.md", f"## Current Version: v{self.version}\nUse `master` branch only\n") + for relative in ( + "README.md", + "INSTALL.md", + "install.sh", + "install.ps1", + ".github/workflows/release.yml", + "pypi/simplicio/simplicio/__main__.py", + "READMEs/README.pt-BR.md", + ): + self.put(relative, "https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.sh\n") + self.put("version.txt", self.version + "\n") + self.put( + "simplicio-update-manifest.json", + json.dumps({"version": self.version, "entitlement": {"beta_until": None}}), + ) + for relative in ( + "npm/simplicio/package.json", + "npm/simplicio-installer/package.json", + "npm/simplicio-unscoped/package.json", + ): + self.put(relative, json.dumps({"version": self.version})) + self.put("Formula/simplicio.rb", f'version "{self.version}"\n') + self.put("pypi/simplicio/pyproject.toml", f'version = "{self.version}"\n') + self.put("SIMPLICIO_ECOSYSTEM.md", f"## Versão atual\n{self.version} (manifest)\n") + + +class DistributionConsistencyTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.fixture = DistributionFixture(self.root) + + def tearDown(self): + self.temp.cleanup() + + def levels(self): + return [item.level for item in audit.run_audit(self.root, today=date(2026, 7, 14))] + + def test_clean_distribution_has_no_blocking_findings(self): + self.assertEqual(self.levels(), ["OK", "OK", "OK", "OK", "OK"]) + + def test_regression_wrong_branch_and_version_fail(self): + self.fixture.put("README.md", "https://raw.githubusercontent.com/wesleysimplicio/simplicio/main/install.sh\n") + self.fixture.put("version.txt", "3.0.2\n") + findings = audit.run_audit(self.root, today=date(2026, 7, 14)) + self.assertEqual(sum(item.level == "ERROR" for item in findings), 2) + self.assertTrue(any("/main/" in item.message for item in findings)) + self.assertTrue(any("version mismatch" in item.message for item in findings)) + + def test_regression_wrapper_and_ecosystem_drift_warn(self): + self.fixture.put("npm/simplicio/package.json", json.dumps({"version": "3.0.2"})) + self.fixture.put("SIMPLICIO_ECOSYSTEM.md", "## Versão atual\n1.2.0\n") + warnings = [item.message for item in audit.run_audit(self.root) if item.level == "WARN"] + self.assertEqual(len(warnings), 2) + self.assertTrue(any("wrapper versions" in message for message in warnings)) + self.assertTrue(any("ECOSYSTEM" in message for message in warnings)) + + def test_regression_version_document_drift_warns(self): + self.fixture.put("VERSION.md", "## Current Version: v3.0.2\nUse `master` branch only\n") + warnings = [item.message for item in audit.run_audit(self.root) if item.level == "WARN"] + self.assertEqual(warnings, ["VERSION.md advertises 3.0.2 while manifest is 3.5.2."]) + + def test_regression_expired_beta_and_contradictory_readme_warn(self): + manifest = {"version": "3.5.2", "entitlement": {"beta_until": "2026-06-30"}} + self.fixture.put("simplicio-update-manifest.json", json.dumps(manifest)) + self.fixture.put("README.md", "Public beta with no end date\n") + warnings = [item.message for item in audit.run_audit(self.root, today=date(2026, 7, 14)) if item.level == "WARN"] + self.assertEqual(len(warnings), 2) + + def test_invalid_beta_date_is_visible(self): + manifest = {"version": "3.5.2", "entitlement": {"beta_until": "not-a-date"}} + self.fixture.put("simplicio-update-manifest.json", json.dumps(manifest)) + warnings = [item.message for item in audit.run_audit(self.root) if item.level == "WARN"] + self.assertEqual(warnings, ["could not parse beta_until date: not-a-date"]) + + def test_parser_helpers_reject_missing_versions(self): + broken = self.root / "broken.txt" + broken.write_text("missing", encoding="utf-8") + with self.assertRaises(ValueError): + audit.version_from_formula(broken) + with self.assertRaises(ValueError): + audit.version_from_pyproject(broken) + + def test_junit_and_json_cli_are_machine_readable(self): + junit = self.root / "artifacts/junit.xml" + output = io.StringIO() + with contextlib.redirect_stdout(output): + result = audit.main(["--root", str(self.root), "--strict", "--junit", str(junit), "--json"]) + self.assertEqual(result, 0) + self.assertTrue(junit.exists()) + self.assertEqual(len(json.loads(output.getvalue())), 5) + + def test_strict_mode_fails_warning_while_default_does_not(self): + self.fixture.put("npm/simplicio/package.json", json.dumps({"version": "3.0.2"})) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(audit.main(["--root", str(self.root)]), 0) + self.assertEqual(audit.main(["--root", str(self.root), "--strict"]), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_quality_helpers.py b/tests/test_quality_helpers.py new file mode 100644 index 0000000..0baa8ab --- /dev/null +++ b/tests/test_quality_helpers.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import json +import contextlib +import io +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from scripts import benchmark_distribution as benchmark +from scripts import flaky_check +from scripts import quality_policy +from scripts import security_scan + + +class QualityPolicyTests(unittest.TestCase): + def test_missing_tests_directory_is_clean(self): + with tempfile.TemporaryDirectory() as directory: + self.assertEqual(list(quality_policy.test_files(Path(directory))), []) + + def test_skip_without_issue_link_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / "tests/example.test.cjs" + path.parent.mkdir(parents=True) + path.write_text("test" + ".skip('later', () => {});\n", encoding="utf-8") + violations = quality_policy.find_violations(root) + self.assertEqual([(item.path, item.line) for item in violations], [("tests/example.test.cjs", 1)]) + + def test_skip_with_adjacent_issue_justification_is_allowed(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / "tests/example.py" + path.parent.mkdir(parents=True) + marker = "@unittest." + "skip('external')" + path.write_text( + "# JUSTIFICATION: tracked at https://github.com/wesleysimplicio/simplicio/issues/10\n" + + marker + + "\n", + encoding="utf-8", + ) + self.assertEqual(quality_policy.find_violations(root), []) + + def test_policy_junit_records_violation(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "policy.xml" + violation = quality_policy.Violation("tests/a.py", 3, "skip") + quality_policy.write_junit(target, [violation]) + self.assertIn("unjustified ignored tests", target.read_text(encoding="utf-8")) + + def test_policy_cli_reports_clean_and_blocked_roots(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + junit = root / "clean.xml" + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(quality_policy.main(["--root", str(root), "--junit", str(junit)]), 0) + path = root / "tests/example.py" + path.parent.mkdir(parents=True) + path.write_text("@unittest." + "skip('later')\n", encoding="utf-8") + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(quality_policy.main(["--root", str(root)]), 1) + + +class BenchmarkTests(unittest.TestCase): + def test_budget_evaluation_passes_and_fails_deterministically(self): + contract = {"size": {"baseline": 100, "max_regression_percent": 10}} + self.assertTrue(benchmark.evaluate({"size": 110}, contract)[0].passed) + self.assertFalse(benchmark.evaluate({"size": 111}, contract)[0].passed) + self.assertTrue(benchmark.evaluate({"size": 150}, contract, override=50)[0].passed) + + def test_payload_metric_covers_all_installer_surfaces(self): + expected = sum((benchmark.ROOT / relative).stat().st_size for relative in benchmark.PAYLOAD_FILES) + self.assertEqual(benchmark.payload_bytes(), expected) + + def test_benchmark_junit_exposes_failure(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "benchmark.xml" + result = benchmark.MetricResult("size", 120, 110, False) + benchmark.write_junit(target, [result]) + self.assertIn("regression budget exceeded", target.read_text(encoding="utf-8")) + + def test_audit_timing_and_cli_emit_reports(self): + self.assertGreaterEqual(benchmark.audit_median_ms(repetitions=2), 0) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + baseline = root / "baseline.json" + baseline.write_text( + json.dumps( + { + "metrics": { + "installer_payload_bytes": {"baseline": 1_000_000, "max_regression_percent": 0}, + "audit_median_ms": {"baseline": 10_000, "max_regression_percent": 0}, + } + } + ), + encoding="utf-8", + ) + junit = root / "benchmark.xml" + output = root / "result.json" + with contextlib.redirect_stdout(io.StringIO()): + result = benchmark.main( + [ + "--baseline", + str(baseline), + "--repetitions", + "2", + "--junit", + str(junit), + "--json", + str(output), + ] + ) + self.assertEqual(result, 0) + self.assertTrue(junit.exists()) + self.assertEqual(len(json.loads(output.read_text(encoding="utf-8"))), 2) + + +class FlakyTests(unittest.TestCase): + def test_classification_distinguishes_pass_fail_and_flaky(self): + passed = flaky_check.Attempt(1, 0, 0.1, "ok") + failed = flaky_check.Attempt(2, 1, 0.1, "bad") + self.assertEqual(flaky_check.classification([passed]), "PASS") + self.assertEqual(flaky_check.classification([failed]), "FAIL") + self.assertEqual(flaky_check.classification([passed, failed]), "FLAKY") + + def test_attempt_runner_captures_output_and_exit(self): + results = flaky_check.run_attempts([sys.executable, "-c", "print('ok')"], 2) + self.assertEqual([item.returncode for item in results], [0, 0]) + self.assertTrue(all("ok" in item.output for item in results)) + + def test_flaky_cli_writes_attempt_receipt(self): + with tempfile.TemporaryDirectory() as directory: + junit = Path(directory) / "flaky.xml" + with contextlib.redirect_stdout(io.StringIO()): + result = flaky_check.main( + [ + "--attempts", + "2", + "--junit", + str(junit), + "--", + sys.executable, + "-c", + "print('stable')", + ] + ) + self.assertEqual(result, 0) + self.assertIn("attempt-2", junit.read_text(encoding="utf-8")) + + +class SecurityScanTests(unittest.TestCase): + def init_repo(self, root: Path, content: str) -> None: + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + (root / "sample.txt").write_text(content, encoding="utf-8") + subprocess.run(["git", "add", "sample.txt"], cwd=root, check=True) + + def test_scanner_accepts_clean_tracked_text(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.init_repo(root, "placeholder only\n") + self.assertEqual(security_scan.scan(root), []) + + def test_scanner_reports_high_confidence_token_without_echoing_it(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + token = "ghp_" + "A" * 40 + self.init_repo(root, token + "\n") + findings = security_scan.scan(root) + self.assertEqual([(item.kind, item.line) for item in findings], [("github-token", 1)]) + + def test_security_cli_emits_junit_for_clean_and_dirty_repo(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.init_repo(root, "clean\n") + junit = root / "security.xml" + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(security_scan.main(["--root", str(root), "--junit", str(junit)]), 0) + (root / "sample.txt").write_text("AKIA" + "A" * 16 + "\n", encoding="utf-8") + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(security_scan.main(["--root", str(root)]), 1) + self.assertTrue(junit.exists()) + + def test_scanner_rejects_dynamic_eval_and_shell_commands(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + source = root / "unsafe.py" + source.write_text( + "import subprocess\n" + "command = input()\n" + + "e" + "val(command)\n" + + "subprocess.run(command, shell=True)\n", + encoding="utf-8", + ) + subprocess.run(["git", "add", "unsafe.py"], cwd=root, check=True) + kinds = {item.kind for item in security_scan.scan(root)} + self.assertEqual(kinds, {"dynamic-code-execution", "dynamic-shell-command"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/version.txt b/version.txt index b502146..87ce492 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.0.2 +3.5.2 From 37d8822f443beeed21536c0e9f8e9d70b7444be0 Mon Sep 17 00:00:00 2001 From: "Simplicio, Wesley (ext)" Date: Tue, 14 Jul 2026 07:43:27 -0300 Subject: [PATCH 2/6] fix(ci): verify signed release provenance Refs #10 --- .github/pull_request_template.md | 3 +- .github/workflows/quality.yml | 2 + .github/workflows/release.yml | 59 ++++++++++------ CHANGELOG.md | 6 +- Formula/simplicio.rb | 16 ++--- QUALITY.md | 17 +++-- scripts/quality_policy.py | 40 +++++++++-- scripts/verify_distribution_consistency.py | 75 +++++++++++++++++++++ tests/test_distribution_consistency.py | 78 ++++++++++++++++++++-- tests/test_quality_helpers.py | 35 +++++++++- 10 files changed, 280 insertions(+), 51 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9d93603..66fd842 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -13,7 +13,8 @@ - [ ] Coverage remains >=85% globally and >=90% for critical release-integrity code. - [ ] Installer/package behavior was exercised in dry-run mode on the supported matrix. - [ ] No test is skipped. If an external dependency requires a skip, the adjacent - `JUSTIFICATION:` comment links a time-boxed repository issue. + exception includes `JUSTIFICATION:`, repository issue URL, `OWNER:`, and a `REMOVE-BY:` date + no more than 30 days away. - [ ] Benchmark changes are within `benchmarks/baseline.json`, or this PR updates the baseline with measured rationale. diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index d644ebe..14ffe6f 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -35,6 +35,8 @@ jobs: node --check npm/simplicio-unscoped/install.js - name: Shell syntax run: shellcheck install.sh scripts/tami-loop.sh + - name: Homebrew formula syntax + run: ruby -c Formula/simplicio.rb - name: PowerShell syntax shell: pwsh run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4a5e8e5..81afd5b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,15 +1,10 @@ name: publish-release -# Publishes the committed binaries + signed update manifest as a GitHub -# Release so `simplicio update` (which downloads from -# github.com//releases/latest/download/...) works for beta testers. +# Republishes only manifest-declared, checksum-verified artifacts. Checked-in +# wrapper binaries are never renamed or treated as release provenance. on: push: - branches: [master, main] + branches: [master] paths: - - simplicio.exe - - simplicio - - simplicio-darwin-x64 - - simplicio-linux-x64 - simplicio-update-manifest.json - .github/workflows/release.yml workflow_dispatch: {} @@ -22,18 +17,47 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Read version from manifest id: ver shell: pwsh run: echo "version=$((Get-Content simplicio-update-manifest.json | ConvertFrom-Json).version)" >> $env:GITHUB_OUTPUT - - name: Stage release assets + - name: Require an existing immutable release tag + shell: pwsh + run: | + $tag = "v${{ steps.ver.outputs.version }}" + git show-ref --verify --quiet "refs/tags/$tag" + if ($LASTEXITCODE -ne 0) { + throw "Refusing to create or move missing tag $tag; publish the signed provenance first." + } + - name: Download and verify manifest artifacts shell: pwsh run: | + $manifest = Get-Content simplicio-update-manifest.json -Raw | ConvertFrom-Json + if (-not $manifest.artifacts -or $manifest.artifacts.Count -eq 0) { + throw "Manifest contains no release artifacts." + } New-Item -ItemType Directory -Force -Path dist | Out-Null - Copy-Item simplicio.exe dist/simplicio-windows-x86_64.exe - Copy-Item simplicio dist/simplicio-macos-arm64 - if (Test-Path simplicio-darwin-x64) { Copy-Item simplicio-darwin-x64 dist/simplicio-darwin-x64 } - if (Test-Path simplicio-linux-x64) { Copy-Item simplicio-linux-x64 dist/simplicio-linux-x64 } + $releasePrefix = "https://github.com/$env:GITHUB_REPOSITORY/releases/download/v$($manifest.version)/" + foreach ($artifact in $manifest.artifacts) { + if (-not $artifact.artifact -or -not $artifact.url -or -not $artifact.sha256) { + throw "Manifest artifact is missing artifact/url/sha256 provenance." + } + $expectedUrl = "$releasePrefix$($artifact.artifact)" + if ($artifact.url -ne $expectedUrl) { + throw "Artifact URL must be version-bound: expected $expectedUrl, got $($artifact.url)." + } + if ($manifest.security.signature_required -and -not $artifact.signature) { + throw "Artifact $($artifact.artifact) is missing its required signature." + } + $destination = Join-Path dist $artifact.artifact + Invoke-WebRequest -Uri $artifact.url -OutFile $destination -UseBasicParsing + $actualHash = (Get-FileHash $destination -Algorithm SHA256).Hash.ToLower() + if ($actualHash -ne $artifact.sha256.ToLower()) { + throw "SHA256 mismatch for $($artifact.artifact): expected $($artifact.sha256), got $actualHash." + } + } Copy-Item simplicio-update-manifest.json dist/ Get-ChildItem dist -File | Sort-Object Name | ForEach-Object { "$(($_ | Get-FileHash -Algorithm SHA256).Hash.ToLower()) *$($_.Name)" @@ -52,10 +76,5 @@ jobs: Signed update manifest included (`simplicio update check`). prerelease: false make_latest: "true" - files: | - dist/simplicio-windows-x86_64.exe - dist/simplicio-macos-arm64 - dist/simplicio-darwin-x64 - dist/simplicio-linux-x64 - dist/simplicio-update-manifest.json - dist/SHA256SUMS + fail_on_unmatched_files: true + files: dist/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fe21bc..63154d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Aligned public wrapper metadata with the canonical `3.5.2` update manifest - and made release checksums derive from the staged artifacts. + and made release publication download only version-bound manifest artifacts, + verify their signed SHA256 provenance, and derive checksums from that verified + staging set. +- Expanded ignored-test enforcement to Python `skipTest`/`SkipTest` and Node + skip options, with issue, owner, and 30-day removal metadata required. ## [1.6.1] - 2026-07-01 diff --git a/Formula/simplicio.rb b/Formula/simplicio.rb index 017dd1f..59fc96b 100644 --- a/Formula/simplicio.rb +++ b/Formula/simplicio.rb @@ -6,19 +6,13 @@ class Simplicio < Formula homepage "https://simpleti.com.br/simplicio/#start" version "3.5.2" license "Proprietary" - - on_macos do - if Hardware::CPU.arm? - url "https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/simplicio" - sha256 "d6d6eee7b086c6f25e13a313444d4d0533aeb45c01a8c9f2dce1f119e29e43c0" - else - url "https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/simplicio" - sha256 "d6d6eee7b086c6f25e13a313444d4d0533aeb45c01a8c9f2dce1f119e29e43c0" - end - end + url "https://github.com/wesleysimplicio/simplicio/releases/download/v3.5.2/simplicio-macos-arm64" + sha256 "931bc6d8f45c1b1e586070f8f5ac4a762861bc9157c70f75aaa4ebfad8ff27bb" + depends_on :macos + depends_on arch: :arm64 def install - bin.install "simplicio" => "simplicio" + bin.install "simplicio-macos-arm64" => "simplicio" end test do diff --git a/QUALITY.md b/QUALITY.md index 09d06ed..9e6d495 100644 --- a/QUALITY.md +++ b/QUALITY.md @@ -31,10 +31,11 @@ JUnit, coverage, benchmark, security, package-build, and flaky-attempt reports a - Node installer lines must stay at or above 85% and functions at or above 90%. - Every bug fix must add a test that fails before the fix. The version-source and expired-beta regressions that triggered issue #10 are frozen in `tests/test_distribution_consistency.py`. -- A test may be skipped only with an adjacent `JUSTIFICATION:` comment and a full - `https://github.com/wesleysimplicio/simplicio/issues/` URL. The linked issue must name an - owner, external dependency, and removal date no later than 30 days. `quality_policy.py` blocks - unregistered skips. +- A test may be skipped only with an adjacent four-line exception block containing + `JUSTIFICATION:`, a full `https://github.com/wesleysimplicio/simplicio/issues/` URL, + `OWNER:`, and `REMOVE-BY: YYYY-MM-DD`. Removal must be between today and 30 days from today. + Decorators, `self.skipTest`, raised `SkipTest`, pytest skips, Node `.skip`, and `{skip: true}` are + all enforced by `quality_policy.py`. - Retries never hide failure: `flaky_check.py` records every attempt, labels mixed outcomes FLAKY, and returns non-zero if any attempt fails. @@ -45,6 +46,14 @@ the baseline in the same PR with measurements and rationale. Emergency diagnosis `SIMPLICIO_BENCHMARK_TOLERANCE_PERCENT`, but merges still require a linked exception issue and an updated baseline; the environment override is not configured on protected branches. +## Release provenance + +`publish-release` never renames a checked-in wrapper binary into a platform artifact. It requires +the version tag to exist, downloads every artifact from the exact version-bound URL declared in +`simplicio-update-manifest.json`, verifies its SHA256 and required signature metadata, then uploads +only that verified staging directory. A missing tag, drifting URL, missing signature, or hash +mismatch stops before the release action; the workflow never creates or moves a tag. + ## Local parity ```text diff --git a/scripts/quality_policy.py b/scripts/quality_policy.py index be0020f..0f51953 100644 --- a/scripts/quality_policy.py +++ b/scripts/quality_policy.py @@ -8,16 +8,23 @@ import sys import xml.etree.ElementTree as ET from dataclasses import dataclass +from datetime import date, timedelta from pathlib import Path from typing import Iterable, Sequence ROOT = Path(__file__).resolve().parents[1] ISSUE_URL = "https://github.com/wesleysimplicio/simplicio/issues/" +ISSUE_RE = re.compile(re.escape(ISSUE_URL) + r"\d+\b") +OWNER_RE = re.compile(r"\bOWNER:\s*([^\s#]+)") +REMOVE_BY_RE = re.compile(r"\bREMOVE-BY:\s*(\d{4}-\d{2}-\d{2})\b") SKIP_PATTERNS = ( re.compile(r"@(?:unittest\.)?skip(?:If|Unless)?\b"), re.compile(r"pytest\.skip\s*\("), re.compile(r"pytest\.mark\.skip"), re.compile(r"\b(?:test|it|describe)\.skip\s*\("), + re.compile(r"\bself\.skipTest\s*\("), + re.compile(r"\braise\s+(?:unittest\.)?SkipTest\b"), + re.compile(r"\bskip\s*:\s*true\b"), ) @@ -26,6 +33,7 @@ class Violation: path: str line: int text: str + reason: str def test_files(root: Path) -> Iterable[Path]: @@ -39,16 +47,36 @@ def test_files(root: Path) -> Iterable[Path]: ) -def find_violations(root: Path = ROOT) -> list[Violation]: +def justification_error(context: str, today: date) -> str | None: + if "JUSTIFICATION:" not in context: + return "missing JUSTIFICATION" + if not ISSUE_RE.search(context): + return "missing repository issue URL" + if not OWNER_RE.search(context): + return "missing OWNER" + remove_by_match = REMOVE_BY_RE.search(context) + if not remove_by_match: + return "missing or invalid REMOVE-BY" + remove_by = date.fromisoformat(remove_by_match.group(1)) + if remove_by < today: + return "REMOVE-BY is expired" + if remove_by > today + timedelta(days=30): + return "REMOVE-BY exceeds 30 days" + return None + + +def find_violations(root: Path = ROOT, *, today: date | None = None) -> list[Violation]: + today = today or date.today() violations: list[Violation] = [] for path in test_files(root): lines = path.read_text(encoding="utf-8").splitlines() for index, line in enumerate(lines): if not any(pattern.search(line) for pattern in SKIP_PATTERNS): continue - context = "\n".join(lines[max(0, index - 2) : index + 1]) - if "JUSTIFICATION:" not in context or ISSUE_URL not in context: - violations.append(Violation(path.relative_to(root).as_posix(), index + 1, line.strip())) + context = "\n".join(lines[max(0, index - 4) : index + 1]) + reason = justification_error(context, today) + if reason: + violations.append(Violation(path.relative_to(root).as_posix(), index + 1, line.strip(), reason)) return violations @@ -56,7 +84,7 @@ def write_junit(path: Path, violations: Sequence[Violation]) -> None: suite = ET.Element("testsuite", name="skip-policy", tests="1", failures=str(bool(violations))) case = ET.SubElement(suite, "testcase", name="all-skips-have-issue-linked-justification") if violations: - detail = "\n".join(f"{item.path}:{item.line}: {item.text}" for item in violations) + detail = "\n".join(f"{item.path}:{item.line}: {item.reason}: {item.text}" for item in violations) ET.SubElement(case, "failure", message="unjustified ignored tests").text = detail path.parent.mkdir(parents=True, exist_ok=True) ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) @@ -72,7 +100,7 @@ def main(argv: Sequence[str] | None = None) -> int: write_junit(args.junit, violations) if violations: for item in violations: - print(f"[ERROR] {item.path}:{item.line}: ignored test lacks JUSTIFICATION + issue URL") + print(f"[ERROR] {item.path}:{item.line}: ignored test exception invalid: {item.reason}") return 1 print("skip-policy: PASS (no unjustified ignored tests)") return 0 diff --git a/scripts/verify_distribution_consistency.py b/scripts/verify_distribution_consistency.py index 49ed7d2..e9efbe6 100644 --- a/scripts/verify_distribution_consistency.py +++ b/scripts/verify_distribution_consistency.py @@ -19,6 +19,8 @@ r"https://raw\.githubusercontent\.com/wesleysimplicio/simplicio/main/install\.(?:sh|ps1)" ) FORMULA_VERSION_RE = re.compile(r'version\s+"([^"]+)"') +FORMULA_URL_RE = re.compile(r'^\s*url\s+"([^"]+)"', re.MULTILINE) +FORMULA_SHA256_RE = re.compile(r'^\s*sha256\s+"([0-9a-fA-F]{64})"', re.MULTILINE) BETA_NO_END_RE = re.compile(r"public beta with no end date", re.IGNORECASE) ECOSYSTEM_VERSION_RE = re.compile(r"## Versão atual\s+([^\n]+)", re.MULTILINE) CURRENT_VERSION_RE = re.compile(r"## Current Version:\s*v([^\s]+)") @@ -49,6 +51,15 @@ def version_from_formula(path: Path) -> str: return match.group(1) +def formula_provenance(path: Path) -> tuple[str, str]: + text = read_text(path) + url = FORMULA_URL_RE.search(text) + sha256 = FORMULA_SHA256_RE.search(text) + if not url or not sha256: + raise ValueError(f"could not parse formula URL/SHA256 from {path}") + return url.group(1), sha256.group(1).lower() + + def version_from_pyproject(path: Path) -> str: match = re.search(r'^version\s*=\s*"([^"]+)"', read_text(path), re.MULTILINE) if not match: @@ -109,6 +120,28 @@ def run_audit(root: Path = ROOT, *, today: date | None = None) -> list[Finding]: else: findings.append(Finding("OK", "VERSION.md matches the release manifest.")) + artifacts = manifest.get("artifacts") or [] + signature_required = bool(manifest.get("security", {}).get("signature_required")) + manifest_errors: list[str] = [] + for artifact in artifacts: + name = str(artifact.get("artifact") or "") + expected_url = ( + f"https://github.com/wesleysimplicio/simplicio/releases/download/" + f"v{manifest_version}/{name}" + ) + if not name or artifact.get("url") != expected_url: + manifest_errors.append(f"manifest artifact URL is not version-bound: {name or 'missing-name'}") + if not re.fullmatch(r"[0-9a-fA-F]{64}", str(artifact.get("sha256") or "")): + manifest_errors.append(f"manifest artifact has invalid SHA256: {name or 'missing-name'}") + if signature_required and not str(artifact.get("signature") or "").startswith("ed25519:"): + manifest_errors.append(f"manifest artifact lacks required Ed25519 signature: {name or 'missing-name'}") + if not artifacts: + manifest_errors.append("manifest contains no release artifacts") + if manifest_errors: + findings.extend(Finding("ERROR", message) for message in manifest_errors) + else: + findings.append(Finding("OK", "manifest artifact URLs, hashes, and signatures are version-bound.")) + wrappers = { "Formula/simplicio.rb": version_from_formula(root / "Formula/simplicio.rb"), "npm/simplicio/package.json": version_from_package_json(root / "npm/simplicio/package.json"), @@ -127,6 +160,48 @@ def run_audit(root: Path = ROOT, *, today: date | None = None) -> list[Finding]: else: findings.append(Finding("OK", "wrapper/package versions match the release manifest.")) + macos_artifact = next((item for item in artifacts if item.get("target") == "macos-arm64"), None) + if not macos_artifact: + findings.append(Finding("ERROR", "manifest lacks the macos-arm64 artifact required by Formula/simplicio.rb.")) + else: + formula_url, formula_sha256 = formula_provenance(root / "Formula/simplicio.rb") + formula_text = read_text(root / "Formula/simplicio.rb") + formula_install = f'bin.install "{macos_artifact.get("artifact")}" => "simplicio"' + if ( + formula_url != macos_artifact.get("url") + or formula_sha256 != str(macos_artifact.get("sha256", "")).lower() + or formula_install not in formula_text + ): + findings.append(Finding("ERROR", "Formula URL/SHA256/install does not match the signed macos-arm64 manifest artifact.")) + else: + findings.append(Finding("OK", "Formula URL/SHA256/install matches the signed macos-arm64 manifest artifact.")) + + release_workflow = read_text(root / ".github/workflows/release.yml") + release_lower = release_workflow.lower() + required_release_tokens = ( + "git show-ref --verify --quiet", + "foreach ($artifact in $manifest.artifacts)", + "invoke-webrequest -uri $artifact.url", + "get-filehash $destination -algorithm sha256", + "if ($actualhash -ne $artifact.sha256.tolower())", + "fail_on_unmatched_files: true", + "files: dist/*", + ) + unsafe_release_lines = ( + "\n - simplicio\n", + "\n - simplicio.exe\n", + "copy-item simplicio ", + "copy-item simplicio.exe ", + "cp simplicio ", + ) + missing_tokens = [token for token in required_release_tokens if token not in release_lower] + unsafe_tokens = [token.strip() for token in unsafe_release_lines if token in release_lower] + if missing_tokens or unsafe_tokens: + details = ", ".join([*(f"missing {token}" for token in missing_tokens), *(f"unsafe {token}" for token in unsafe_tokens)]) + findings.append(Finding("ERROR", f"release workflow provenance is not fail-closed: {details}")) + else: + findings.append(Finding("OK", "release workflow downloads only manifest artifacts and verifies SHA256 before upload.")) + ecosystem = read_text(root / "SIMPLICIO_ECOSYSTEM.md") ecosystem_match = ECOSYSTEM_VERSION_RE.search(ecosystem) if not ecosystem_match or manifest_version not in ecosystem_match.group(1): diff --git a/tests/test_distribution_consistency.py b/tests/test_distribution_consistency.py index fc2fd85..368fadd 100644 --- a/tests/test_distribution_consistency.py +++ b/tests/test_distribution_consistency.py @@ -29,15 +29,45 @@ def write(self) -> None: "INSTALL.md", "install.sh", "install.ps1", - ".github/workflows/release.yml", "pypi/simplicio/simplicio/__main__.py", "READMEs/README.pt-BR.md", ): self.put(relative, "https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.sh\n") + self.put( + ".github/workflows/release.yml", + "git show-ref --verify --quiet refs/tags/v3.5.2\n" + "foreach ($artifact in $manifest.artifacts) {\n" + " Invoke-WebRequest -Uri $artifact.url -OutFile $destination\n" + " $actualHash = (Get-FileHash $destination -Algorithm SHA256).Hash.ToLower()\n" + " if ($actualHash -ne $artifact.sha256.ToLower()) { throw 'mismatch' }\n" + "}\n" + "fail_on_unmatched_files: true\n" + "files: dist/*\n", + ) self.put("version.txt", self.version + "\n") + artifact_url = ( + "https://github.com/wesleysimplicio/simplicio/releases/download/" + f"v{self.version}/simplicio-macos-arm64" + ) + artifact_sha = "9" * 64 self.put( "simplicio-update-manifest.json", - json.dumps({"version": self.version, "entitlement": {"beta_until": None}}), + json.dumps( + { + "version": self.version, + "security": {"signature_required": True}, + "entitlement": {"beta_until": None}, + "artifacts": [ + { + "target": "macos-arm64", + "artifact": "simplicio-macos-arm64", + "url": artifact_url, + "sha256": artifact_sha, + "signature": "ed25519:fixture", + } + ], + } + ), ) for relative in ( "npm/simplicio/package.json", @@ -45,7 +75,11 @@ def write(self) -> None: "npm/simplicio-unscoped/package.json", ): self.put(relative, json.dumps({"version": self.version})) - self.put("Formula/simplicio.rb", f'version "{self.version}"\n') + self.put( + "Formula/simplicio.rb", + f'version "{self.version}"\nurl "{artifact_url}"\nsha256 "{artifact_sha}"\n' + 'bin.install "simplicio-macos-arm64" => "simplicio"\n', + ) self.put("pypi/simplicio/pyproject.toml", f'version = "{self.version}"\n') self.put("SIMPLICIO_ECOSYSTEM.md", f"## Versão atual\n{self.version} (manifest)\n") @@ -63,7 +97,7 @@ def levels(self): return [item.level for item in audit.run_audit(self.root, today=date(2026, 7, 14))] def test_clean_distribution_has_no_blocking_findings(self): - self.assertEqual(self.levels(), ["OK", "OK", "OK", "OK", "OK"]) + self.assertEqual(self.levels(), ["OK"] * 8) def test_regression_wrong_branch_and_version_fail(self): self.fixture.put("README.md", "https://raw.githubusercontent.com/wesleysimplicio/simplicio/main/install.sh\n") @@ -106,6 +140,40 @@ def test_parser_helpers_reject_missing_versions(self): audit.version_from_formula(broken) with self.assertRaises(ValueError): audit.version_from_pyproject(broken) + with self.assertRaises(ValueError): + audit.formula_provenance(broken) + + def test_regression_formula_must_match_signed_manifest_artifact(self): + formula = self.root / "Formula/simplicio.rb" + formula.write_text(formula.read_text(encoding="utf-8").replace("9" * 64, "8" * 64), encoding="utf-8") + errors = [item.message for item in audit.run_audit(self.root) if item.level == "ERROR"] + self.assertIn("Formula URL/SHA256/install does not match the signed macos-arm64 manifest artifact.", errors) + + def test_regression_formula_must_install_versioned_asset(self): + formula = self.root / "Formula/simplicio.rb" + formula.write_text( + formula.read_text(encoding="utf-8").replace( + 'bin.install "simplicio-macos-arm64" => "simplicio"', + 'bin.install "simplicio-wrapper" => "simplicio"', + ), + encoding="utf-8", + ) + errors = [item.message for item in audit.run_audit(self.root) if item.level == "ERROR"] + self.assertIn("Formula URL/SHA256/install does not match the signed macos-arm64 manifest artifact.", errors) + + def test_regression_release_must_not_stage_repo_wrapper_binary(self): + workflow = self.root / ".github/workflows/release.yml" + workflow.write_text(workflow.read_text(encoding="utf-8") + "Copy-Item simplicio dist/simplicio-macos-arm64\n", encoding="utf-8") + errors = [item.message for item in audit.run_audit(self.root) if item.level == "ERROR"] + self.assertTrue(any("release workflow provenance" in message and "unsafe" in message for message in errors)) + + def test_regression_manifest_url_must_be_version_bound(self): + manifest_path = self.root / "simplicio-update-manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["artifacts"][0]["url"] = manifest["artifacts"][0]["url"].replace("v3.5.2", "latest") + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + errors = [item.message for item in audit.run_audit(self.root) if item.level == "ERROR"] + self.assertTrue(any("not version-bound" in message for message in errors)) def test_junit_and_json_cli_are_machine_readable(self): junit = self.root / "artifacts/junit.xml" @@ -114,7 +182,7 @@ def test_junit_and_json_cli_are_machine_readable(self): result = audit.main(["--root", str(self.root), "--strict", "--junit", str(junit), "--json"]) self.assertEqual(result, 0) self.assertTrue(junit.exists()) - self.assertEqual(len(json.loads(output.getvalue())), 5) + self.assertEqual(len(json.loads(output.getvalue())), 8) def test_strict_mode_fails_warning_while_default_does_not(self): self.fixture.put("npm/simplicio/package.json", json.dumps({"version": "3.0.2"})) diff --git a/tests/test_quality_helpers.py b/tests/test_quality_helpers.py index 0baa8ab..49bf09b 100644 --- a/tests/test_quality_helpers.py +++ b/tests/test_quality_helpers.py @@ -7,6 +7,7 @@ import sys import tempfile import unittest +from datetime import date from pathlib import Path from scripts import benchmark_distribution as benchmark @@ -26,7 +27,7 @@ def test_skip_without_issue_link_is_rejected(self): path = root / "tests/example.test.cjs" path.parent.mkdir(parents=True) path.write_text("test" + ".skip('later', () => {});\n", encoding="utf-8") - violations = quality_policy.find_violations(root) + violations = quality_policy.find_violations(root, today=date(2026, 7, 14)) self.assertEqual([(item.path, item.line) for item in violations], [("tests/example.test.cjs", 1)]) def test_skip_with_adjacent_issue_justification_is_allowed(self): @@ -37,16 +38,44 @@ def test_skip_with_adjacent_issue_justification_is_allowed(self): marker = "@unittest." + "skip('external')" path.write_text( "# JUSTIFICATION: tracked at https://github.com/wesleysimplicio/simplicio/issues/10\n" + "# OWNER: @release-maintainer\n" + "# REMOVE-BY: 2026-07-30\n" + marker + "\n", encoding="utf-8", ) - self.assertEqual(quality_policy.find_violations(root), []) + self.assertEqual(quality_policy.find_violations(root, today=date(2026, 7, 14)), []) + + def test_all_supported_skip_forms_are_detected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / "tests/skip_forms.py" + path.parent.mkdir(parents=True) + path.write_text( + "self." + "skipTest('later')\n" + "raise unittest." + "SkipTest('later')\n" + "raise " + "SkipTest('later')\n" + "options = { " + "sk" + "ip: true }\n", + encoding="utf-8", + ) + violations = quality_policy.find_violations(root, today=date(2026, 7, 14)) + self.assertEqual(len(violations), 4) + self.assertTrue(all(item.reason == "missing JUSTIFICATION" for item in violations)) + + def test_exception_requires_owner_and_near_term_removal(self): + base = ( + "# JUSTIFICATION: external service\n" + "# https://github.com/wesleysimplicio/simplicio/issues/10\n" + ) + self.assertEqual(quality_policy.justification_error(base + "# REMOVE-BY: 2026-07-20", date(2026, 7, 14)), "missing OWNER") + complete = base + "# OWNER: @maintainer\n" + self.assertEqual(quality_policy.justification_error(complete + "# REMOVE-BY: 2026-07-01", date(2026, 7, 14)), "REMOVE-BY is expired") + self.assertEqual(quality_policy.justification_error(complete + "# REMOVE-BY: 2026-09-01", date(2026, 7, 14)), "REMOVE-BY exceeds 30 days") def test_policy_junit_records_violation(self): with tempfile.TemporaryDirectory() as directory: target = Path(directory) / "policy.xml" - violation = quality_policy.Violation("tests/a.py", 3, "skip") + violation = quality_policy.Violation("tests/a.py", 3, "skip", "missing OWNER") quality_policy.write_junit(target, [violation]) self.assertIn("unjustified ignored tests", target.read_text(encoding="utf-8")) From eabeebab84d2748b2134aa5b5550a786dcfdcdf1 Mon Sep 17 00:00:00 2001 From: "Simplicio, Wesley (ext)" Date: Tue, 14 Jul 2026 07:58:06 -0300 Subject: [PATCH 3/6] fix(release): block immutable provenance drift Refs #10 --- .github/workflows/quality.yml | 2 +- .github/workflows/release.yml | 44 ++++++-- CHANGELOG.md | 3 + QUALITY.md | 15 ++- scripts/verify_distribution_consistency.py | 73 ++++++++----- scripts/verify_release_provenance.py | 105 +++++++++++++++++++ tests/test_distribution_consistency.py | 29 ++++++ tests/test_release_provenance.py | 113 +++++++++++++++++++++ 8 files changed, 345 insertions(+), 39 deletions(-) create mode 100644 scripts/verify_release_provenance.py create mode 100644 tests/test_release_provenance.py diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 14ffe6f..e56a35e 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -72,7 +72,7 @@ jobs: run: | coverage run --rcfile=.coveragerc scripts/run_python_tests.py --junit artifacts/unit/python-junit.xml coverage report --fail-under=85 - coverage report --include=scripts/verify_distribution_consistency.py --fail-under=90 + coverage report --include=scripts/verify_distribution_consistency.py,scripts/verify_release_provenance.py --fail-under=90 coverage xml coverage json - name: Node installer coverage diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 81afd5b..bb1f72f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,12 +1,7 @@ name: publish-release -# Republishes only manifest-declared, checksum-verified artifacts. Checked-in -# wrapper binaries are never renamed or treated as release provenance. +# Manual-only publisher for a freshly prepared tag. It refuses to mutate a +# release unless the tag-bound and working manifests have identical provenance. on: - push: - branches: [master] - paths: - - simplicio-update-manifest.json - - .github/workflows/release.yml workflow_dispatch: {} permissions: @@ -19,18 +14,48 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" - name: Read version from manifest id: ver shell: pwsh run: echo "version=$((Get-Content simplicio-update-manifest.json | ConvertFrom-Json).version)" >> $env:GITHUB_OUTPUT - - name: Require an existing immutable release tag + - name: Read immutable tag provenance and remote release state shell: pwsh + env: + GITHUB_TOKEN: ${{ github.token }} run: | $tag = "v${{ steps.ver.outputs.version }}" git show-ref --verify --quiet "refs/tags/$tag" if ($LASTEXITCODE -ne 0) { throw "Refusing to create or move missing tag $tag; publish the signed provenance first." } + New-Item -ItemType Directory -Force -Path .release | Out-Null + git show "${tag}:simplicio-update-manifest.json" | Set-Content -Encoding utf8 .release/tag-manifest.json + if ($LASTEXITCODE -ne 0) { + throw "Tag $tag does not contain simplicio-update-manifest.json." + } + $headers = @{ + Authorization = "Bearer $env:GITHUB_TOKEN" + Accept = "application/vnd.github+json" + "X-GitHub-Api-Version" = "2022-11-28" + } + $uri = "https://api.github.com/repos/$env:GITHUB_REPOSITORY/releases/tags/$tag" + $response = Invoke-WebRequest -Uri $uri -Headers $headers -SkipHttpErrorCheck + if ($response.StatusCode -eq 200) { + $response.Content | Set-Content -Encoding utf8 .release/remote-release.json + } elseif ($response.StatusCode -eq 404) { + '{"assets":[]}' | Set-Content -Encoding utf8 .release/remote-release.json + } else { + throw "Could not verify immutable remote release state: HTTP $($response.StatusCode)." + } + - name: Verify tag provenance and immutable remote assets + run: >- + python scripts/verify_release_provenance.py + --working-manifest simplicio-update-manifest.json + --tag-manifest .release/tag-manifest.json + --remote-release .release/remote-release.json - name: Download and verify manifest artifacts shell: pwsh run: | @@ -62,7 +87,7 @@ jobs: Get-ChildItem dist -File | Sort-Object Name | ForEach-Object { "$(($_ | Get-FileHash -Algorithm SHA256).Hash.ToLower()) *$($_.Name)" } | Set-Content -Encoding ascii dist/SHA256SUMS - - name: Create or update release + - name: Create release or upload only new assets uses: softprops/action-gh-release@v2 with: tag_name: v${{ steps.ver.outputs.version }} @@ -77,4 +102,5 @@ jobs: prerelease: false make_latest: "true" fail_on_unmatched_files: true + overwrite_files: false files: dist/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 63154d1..89291d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). staging set. - Expanded ignored-test enforcement to Python `skipTest`/`SkipTest` and Node skip options, with issue, owner, and 30-day removal metadata required. +- Made publication manual-only and immutable: tag-bound manifest provenance must + match exactly, existing release assets cannot be replaced, and the known stale + `v3.5.2` tag now fails closed pending separate fresh-release preparation. ## [1.6.1] - 2026-07-01 diff --git a/QUALITY.md b/QUALITY.md index 9e6d495..bf00002 100644 --- a/QUALITY.md +++ b/QUALITY.md @@ -48,11 +48,16 @@ updated baseline; the environment override is not configured on protected branch ## Release provenance -`publish-release` never renames a checked-in wrapper binary into a platform artifact. It requires -the version tag to exist, downloads every artifact from the exact version-bound URL declared in -`simplicio-update-manifest.json`, verifies its SHA256 and required signature metadata, then uploads -only that verified staging directory. A missing tag, drifting URL, missing signature, or hash -mismatch stops before the release action; the workflow never creates or moves a tag. +`publish-release` is manual-only; merging a CI or manifest change cannot trigger publication. Before +any download or release mutation, it reads `simplicio-update-manifest.json` from the immutable version +tag and requires exact version plus artifact name/URL/SHA256/signature equality with the working tree. +It also reads the remote release and refuses any declared or generated asset name that already exists; +the release action has `overwrite_files: false` as a second fail-closed guard. + +The existing `v3.5.2` tag contains a `3.5.1` manifest, so the manual workflow is intentionally blocked +for that tag and must not move it or replace its assets. Preparing a fresh version, immutable tag, and +signed artifact set is separate release work. Once those inputs agree, the workflow downloads each +version-bound artifact, verifies SHA256, and uploads only new assets from the verified staging directory. ## Local parity diff --git a/scripts/verify_distribution_consistency.py b/scripts/verify_distribution_consistency.py index e9efbe6..d2e67bb 100644 --- a/scripts/verify_distribution_consistency.py +++ b/scripts/verify_distribution_consistency.py @@ -80,6 +80,51 @@ def iter_install_reference_files(root: Path) -> Iterable[Path]: yield from sorted((root / "READMEs").glob("README*.md")) +def release_workflow_errors(workflow: str) -> list[str]: + lower = workflow.lower() + errors: list[str] = [] + if not re.search(r"(?m)^on:\s*\n\s{2}workflow_dispatch:\s*(?:\{\})?\s*$", workflow): + errors.append("release workflow is not manual-only") + automatic_triggers = re.findall(r"(?m)^\s{2}(push|pull_request|schedule):", workflow) + if automatic_triggers: + errors.append("release workflow has automatic trigger: " + ", ".join(sorted(set(automatic_triggers)))) + required_tokens = ( + "git show-ref --verify --quiet", + 'git show "${tag}:simplicio-update-manifest.json"', + "python scripts/verify_release_provenance.py", + "--working-manifest simplicio-update-manifest.json", + "--tag-manifest .release/tag-manifest.json", + "--remote-release .release/remote-release.json", + "foreach ($artifact in $manifest.artifacts)", + "invoke-webrequest -uri $artifact.url", + "get-filehash $destination -algorithm sha256", + "if ($actualhash -ne $artifact.sha256.tolower())", + "fail_on_unmatched_files: true", + "overwrite_files: false", + "files: dist/*", + "uses: softprops/action-gh-release@v2", + ) + errors.extend(f"missing {token}" for token in required_tokens if token not in lower) + if "overwrite_files: true" in lower: + errors.append("overwrite_files must never be true") + unsafe_lines = ( + "\n - simplicio\n", + "\n - simplicio.exe\n", + "copy-item simplicio ", + "copy-item simplicio.exe ", + "cp simplicio ", + ) + errors.extend(f"unsafe {token.strip()}" for token in unsafe_lines if token in lower) + verifier_index = lower.find("python scripts/verify_release_provenance.py") + download_index = lower.find("invoke-webrequest -uri $artifact.url") + mutation_index = lower.find("uses: softprops/action-gh-release@v2") + if min(verifier_index, download_index, mutation_index) >= 0 and not ( + verifier_index < download_index < mutation_index + ): + errors.append("tag provenance verification must run before download and release mutation") + return errors + + def run_audit(root: Path = ROOT, *, today: date | None = None) -> list[Finding]: today = today or date.today() findings: list[Finding] = [] @@ -176,31 +221,11 @@ def run_audit(root: Path = ROOT, *, today: date | None = None) -> list[Finding]: else: findings.append(Finding("OK", "Formula URL/SHA256/install matches the signed macos-arm64 manifest artifact.")) - release_workflow = read_text(root / ".github/workflows/release.yml") - release_lower = release_workflow.lower() - required_release_tokens = ( - "git show-ref --verify --quiet", - "foreach ($artifact in $manifest.artifacts)", - "invoke-webrequest -uri $artifact.url", - "get-filehash $destination -algorithm sha256", - "if ($actualhash -ne $artifact.sha256.tolower())", - "fail_on_unmatched_files: true", - "files: dist/*", - ) - unsafe_release_lines = ( - "\n - simplicio\n", - "\n - simplicio.exe\n", - "copy-item simplicio ", - "copy-item simplicio.exe ", - "cp simplicio ", - ) - missing_tokens = [token for token in required_release_tokens if token not in release_lower] - unsafe_tokens = [token.strip() for token in unsafe_release_lines if token in release_lower] - if missing_tokens or unsafe_tokens: - details = ", ".join([*(f"missing {token}" for token in missing_tokens), *(f"unsafe {token}" for token in unsafe_tokens)]) - findings.append(Finding("ERROR", f"release workflow provenance is not fail-closed: {details}")) + release_errors = release_workflow_errors(read_text(root / ".github/workflows/release.yml")) + if release_errors: + findings.append(Finding("ERROR", "release workflow provenance is not fail-closed: " + ", ".join(release_errors))) else: - findings.append(Finding("OK", "release workflow downloads only manifest artifacts and verifies SHA256 before upload.")) + findings.append(Finding("OK", "manual release verifies tag provenance and immutable assets before upload.")) ecosystem = read_text(root / "SIMPLICIO_ECOSYSTEM.md") ecosystem_match = ECOSYSTEM_VERSION_RE.search(ecosystem) diff --git a/scripts/verify_release_provenance.py b/scripts/verify_release_provenance.py new file mode 100644 index 0000000..45c0a16 --- /dev/null +++ b/scripts/verify_release_provenance.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Fail closed when a release would mutate tag-bound artifact provenance.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Sequence + +PROVENANCE_FIELDS = ("artifact", "url", "sha256", "signature") +GENERATED_RELEASE_ASSETS = {"simplicio-update-manifest.json", "SHA256SUMS"} + + +def load_json(path: Path) -> dict: + value = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(value, dict): + raise ValueError(f"expected a JSON object in {path}") + return value + + +def provenance_snapshot(manifest: dict) -> tuple[str, tuple[tuple[str, str, str, str], ...]]: + version = str(manifest.get("version") or "").strip() + if not version: + raise ValueError("manifest version is missing") + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, list) or not artifacts: + raise ValueError("manifest artifacts must be a non-empty list") + + records: list[tuple[str, str, str, str]] = [] + names: set[str] = set() + for index, artifact in enumerate(artifacts): + if not isinstance(artifact, dict): + raise ValueError(f"manifest artifact {index} is not an object") + values = tuple(str(artifact.get(field) or "").strip() for field in PROVENANCE_FIELDS) + missing = [field for field, value in zip(PROVENANCE_FIELDS, values) if not value] + if missing: + raise ValueError(f"manifest artifact {index} is missing: {', '.join(missing)}") + if values[0] in names: + raise ValueError(f"manifest contains duplicate artifact name: {values[0]}") + names.add(values[0]) + records.append(values) + return version, tuple(sorted(records)) + + +def compare_tag_provenance(working: dict, tagged: dict) -> list[str]: + working_version, working_artifacts = provenance_snapshot(working) + tagged_version, tagged_artifacts = provenance_snapshot(tagged) + errors: list[str] = [] + if tagged_version != working_version: + errors.append( + f"tag manifest version {tagged_version} does not equal working manifest version {working_version}" + ) + if tagged_artifacts != working_artifacts: + errors.append("tag manifest artifact name/url/sha256/signature provenance differs from working manifest") + return errors + + +def existing_asset_conflicts(working: dict, remote_release: dict) -> list[str]: + _, artifacts = provenance_snapshot(working) + declared_names = {record[0] for record in artifacts} | GENERATED_RELEASE_ASSETS + assets = remote_release.get("assets", []) + if not isinstance(assets, list): + raise ValueError("remote release assets must be a list") + existing_names = { + str(asset.get("name") or "") + for asset in assets + if isinstance(asset, dict) and asset.get("name") + } + return sorted(declared_names & existing_names) + + +def verify(working: dict, tagged: dict, remote_release: dict | None = None) -> list[str]: + errors = compare_tag_provenance(working, tagged) + if remote_release is not None: + conflicts = existing_asset_conflicts(working, remote_release) + if conflicts: + errors.append("immutable release already contains assets: " + ", ".join(conflicts)) + return errors + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--working-manifest", type=Path, required=True) + parser.add_argument("--tag-manifest", type=Path, required=True) + parser.add_argument("--remote-release", type=Path) + args = parser.parse_args(argv) + try: + working = load_json(args.working_manifest) + tagged = load_json(args.tag_manifest) + remote = load_json(args.remote_release) if args.remote_release else None + errors = verify(working, tagged, remote) + except (OSError, ValueError, json.JSONDecodeError) as exc: + errors = [f"invalid release provenance input: {exc}"] + if errors: + for error in errors: + print(f"[ERROR] {error}") + return 1 + print("release-provenance: PASS (tag and remote assets are immutable)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_distribution_consistency.py b/tests/test_distribution_consistency.py index 368fadd..a39303b 100644 --- a/tests/test_distribution_consistency.py +++ b/tests/test_distribution_consistency.py @@ -35,13 +35,22 @@ def write(self) -> None: self.put(relative, "https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.sh\n") self.put( ".github/workflows/release.yml", + "on:\n" + " workflow_dispatch: {}\n" "git show-ref --verify --quiet refs/tags/v3.5.2\n" + 'git show "${tag}:simplicio-update-manifest.json"\n' + "python scripts/verify_release_provenance.py \n" + " --working-manifest simplicio-update-manifest.json \n" + " --tag-manifest .release/tag-manifest.json \n" + " --remote-release .release/remote-release.json\n" "foreach ($artifact in $manifest.artifacts) {\n" " Invoke-WebRequest -Uri $artifact.url -OutFile $destination\n" " $actualHash = (Get-FileHash $destination -Algorithm SHA256).Hash.ToLower()\n" " if ($actualHash -ne $artifact.sha256.ToLower()) { throw 'mismatch' }\n" "}\n" "fail_on_unmatched_files: true\n" + "overwrite_files: false\n" + "uses: softprops/action-gh-release@v2\n" "files: dist/*\n", ) self.put("version.txt", self.version + "\n") @@ -167,6 +176,26 @@ def test_regression_release_must_not_stage_repo_wrapper_binary(self): errors = [item.message for item in audit.run_audit(self.root) if item.level == "ERROR"] self.assertTrue(any("release workflow provenance" in message and "unsafe" in message for message in errors)) + def test_regression_release_must_be_manual_only(self): + workflow = self.root / ".github/workflows/release.yml" + workflow.write_text(workflow.read_text(encoding="utf-8").replace(" workflow_dispatch: {}", " push:\n workflow_dispatch: {}"), encoding="utf-8") + errors = audit.release_workflow_errors(workflow.read_text(encoding="utf-8")) + self.assertTrue(any("automatic trigger: push" in error for error in errors)) + + def test_regression_release_requires_tag_bound_verifier(self): + workflow = self.root / ".github/workflows/release.yml" + workflow.write_text(workflow.read_text(encoding="utf-8").replace("python scripts/verify_release_provenance.py", "python scripts/untrusted.py"), encoding="utf-8") + errors = audit.release_workflow_errors(workflow.read_text(encoding="utf-8")) + self.assertTrue(any("missing python scripts/verify_release_provenance.py" in error for error in errors)) + + def test_regression_release_requires_explicit_no_overwrite(self): + workflow = self.root / ".github/workflows/release.yml" + clean = workflow.read_text(encoding="utf-8") + for unsafe in (clean.replace("overwrite_files: false\n", ""), clean.replace("overwrite_files: false", "overwrite_files: true")): + with self.subTest(unsafe="absent" if "overwrite_files:" not in unsafe else "true"): + errors = audit.release_workflow_errors(unsafe) + self.assertTrue(any("overwrite_files" in error for error in errors)) + def test_regression_manifest_url_must_be_version_bound(self): manifest_path = self.root / "simplicio-update-manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) diff --git a/tests/test_release_provenance.py b/tests/test_release_provenance.py new file mode 100644 index 0000000..144d246 --- /dev/null +++ b/tests/test_release_provenance.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +from scripts import verify_release_provenance as provenance + + +def manifest(version: str = "3.5.2") -> dict: + name = "simplicio-macos-arm64" + return { + "version": version, + "artifacts": [ + { + "artifact": name, + "url": f"https://github.com/wesleysimplicio/simplicio/releases/download/v{version}/{name}", + "sha256": "9" * 64, + "signature": "ed25519:fixture", + } + ], + } + + +class ReleaseProvenanceTests(unittest.TestCase): + def test_equal_tag_provenance_and_new_assets_pass(self): + working = manifest() + self.assertEqual(provenance.verify(working, manifest(), {"assets": []}), []) + + def test_known_tag_version_and_artifact_drift_fail_closed(self): + errors = provenance.verify(manifest("3.5.2"), manifest("3.5.1"), {"assets": []}) + self.assertEqual(len(errors), 2) + self.assertIn("tag manifest version 3.5.1 does not equal working manifest version 3.5.2", errors) + self.assertTrue(any("artifact name/url/sha256/signature" in error for error in errors)) + + def test_any_existing_declared_or_generated_asset_blocks_upload(self): + working = manifest() + for name in ("simplicio-macos-arm64", "simplicio-update-manifest.json", "SHA256SUMS"): + with self.subTest(name=name): + errors = provenance.verify(working, manifest(), {"assets": [{"name": name}]}) + self.assertEqual(errors, [f"immutable release already contains assets: {name}"]) + + def test_invalid_or_duplicate_provenance_is_rejected(self): + broken = manifest() + del broken["artifacts"][0]["signature"] + with self.assertRaisesRegex(ValueError, "missing: signature"): + provenance.provenance_snapshot(broken) + duplicate = manifest() + duplicate["artifacts"].append(dict(duplicate["artifacts"][0])) + with self.assertRaisesRegex(ValueError, "duplicate artifact name"): + provenance.provenance_snapshot(duplicate) + + def test_malformed_manifest_and_remote_shapes_are_rejected(self): + for broken, message in ( + ({"artifacts": []}, "version is missing"), + ({"version": "3.5.2", "artifacts": []}, "non-empty list"), + ({"version": "3.5.2", "artifacts": ["not-an-object"]}, "not an object"), + ): + with self.subTest(message=message), self.assertRaisesRegex(ValueError, message): + provenance.provenance_snapshot(broken) + with self.assertRaisesRegex(ValueError, "remote release assets must be a list"): + provenance.existing_asset_conflicts(manifest(), {"assets": "unknown"}) + + def test_cli_reports_tag_mismatch_and_returns_nonzero(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + working = root / "working.json" + tagged = root / "tagged.json" + remote = root / "remote.json" + working.write_text(json.dumps(manifest("3.5.2")), encoding="utf-8") + tagged.write_text(json.dumps(manifest("3.5.1")), encoding="utf-8") + remote.write_text(json.dumps({"assets": []}), encoding="utf-8") + output = io.StringIO() + with contextlib.redirect_stdout(output): + result = provenance.main( + [ + "--working-manifest", + str(working), + "--tag-manifest", + str(tagged), + "--remote-release", + str(remote), + ] + ) + self.assertEqual(result, 1) + self.assertIn("3.5.1", output.getvalue()) + + def test_cli_passes_equal_inputs_and_fails_non_object_json(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + working = root / "working.json" + tagged = root / "tagged.json" + remote = root / "remote.json" + working.write_text(json.dumps(manifest()), encoding="utf-8") + tagged.write_text(json.dumps(manifest()), encoding="utf-8") + remote.write_text(json.dumps({"assets": []}), encoding="utf-8") + output = io.StringIO() + args = ["--working-manifest", str(working), "--tag-manifest", str(tagged)] + with contextlib.redirect_stdout(output): + self.assertEqual(provenance.main(args), 0) + self.assertIn("release-provenance: PASS", output.getvalue()) + working.write_text("[]", encoding="utf-8") + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.assertEqual(provenance.main(args), 1) + self.assertIn("expected a JSON object", output.getvalue()) + + +if __name__ == "__main__": + unittest.main() From d3dfa6b9143786089045dc6e75c527d71c8e22ae Mon Sep 17 00:00:00 2001 From: "Simplicio, Wesley (ext)" Date: Tue, 14 Jul 2026 08:17:36 -0300 Subject: [PATCH 4/6] fix(release): validate structured publish plan Refs #10 --- .github/workflows/quality.yml | 14 +- .github/workflows/release.yml | 111 +++++---- CHANGELOG.md | 3 + QUALITY.md | 22 +- requirements-quality.txt | 2 + scripts/verify_distribution_consistency.py | 145 ++++++++--- scripts/verify_release_provenance.py | 196 ++++++++++++--- tests/test_distribution_consistency.py | 176 +++++++++++--- tests/test_release_provenance.py | 269 ++++++++++++++++----- 9 files changed, 725 insertions(+), 213 deletions(-) create mode 100644 requirements-quality.txt diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index e56a35e..3166014 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -26,6 +26,8 @@ jobs: - uses: actions/setup-node@v4 with: node-version: "22" + - name: Install pinned quality dependencies + run: python -m pip install -r requirements-quality.txt - name: Python syntax run: python -m compileall -q scripts pypi/simplicio/simplicio tests - name: Node syntax @@ -66,8 +68,8 @@ jobs: - uses: actions/setup-node@v4 with: node-version: "22" - - name: Install coverage tool - run: python -m pip install coverage==7.6.12 + - name: Install pinned quality dependencies + run: python -m pip install -r requirements-quality.txt - name: Python tests with JUnit and coverage run: | coverage run --rcfile=.coveragerc scripts/run_python_tests.py --junit artifacts/unit/python-junit.xml @@ -112,6 +114,8 @@ jobs: - uses: actions/setup-node@v4 with: node-version: "22" + - name: Install pinned quality dependencies + run: python -m pip install -r requirements-quality.txt - name: Strict distribution regression audit run: >- python scripts/verify_distribution_consistency.py --strict @@ -174,6 +178,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.13" + - name: Install pinned quality dependencies + run: python -m pip install -r requirements-quality.txt - name: Deterministic secret and dangerous-code scan run: python scripts/security_scan.py --junit artifacts/security/secret-scan-junit.xml - name: Upload security evidence @@ -193,6 +199,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.13" + - name: Install pinned quality dependencies + run: python -m pip install -r requirements-quality.txt - name: Enforce versioned regression budget run: >- python scripts/benchmark_distribution.py @@ -218,6 +226,8 @@ jobs: - uses: actions/setup-node@v4 with: node-version: "22" + - name: Install pinned quality dependencies + run: python -m pip install -r requirements-quality.txt - name: Repeat Python unit suite run: >- python scripts/flaky_check.py --attempts 3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bb1f72f..ec0c107 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,8 +1,13 @@ name: publish-release -# Manual-only publisher for a freshly prepared tag. It refuses to mutate a -# release unless the tag-bound and working manifests have identical provenance. -on: - workflow_dispatch: {} +# Manual-only immutable publisher. New artifacts come from a distinct, +# version-bound staging origin; coherent existing releases are no-op success. +"on": + workflow_dispatch: + inputs: + artifact_base_url: + description: Immutable HTTPS staging base containing the versioned artifacts + required: true + type: string permissions: contents: write @@ -17,24 +22,28 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.13" + - name: Install pinned release verifier dependencies + run: python -m pip install -r requirements-quality.txt - name: Read version from manifest - id: ver + id: version shell: pwsh run: echo "version=$((Get-Content simplicio-update-manifest.json | ConvertFrom-Json).version)" >> $env:GITHUB_OUTPUT - - name: Read immutable tag provenance and remote release state + - name: Read target tag and release state without mutation + id: state shell: pwsh env: GITHUB_TOKEN: ${{ github.token }} run: | - $tag = "v${{ steps.ver.outputs.version }}" - git show-ref --verify --quiet "refs/tags/$tag" - if ($LASTEXITCODE -ne 0) { - throw "Refusing to create or move missing tag $tag; publish the signed provenance first." - } + $tag = "v${{ steps.version.outputs.version }}" New-Item -ItemType Directory -Force -Path .release | Out-Null - git show "${tag}:simplicio-update-manifest.json" | Set-Content -Encoding utf8 .release/tag-manifest.json - if ($LASTEXITCODE -ne 0) { - throw "Tag $tag does not contain simplicio-update-manifest.json." + $matchingTag = git tag --list $tag + if ($matchingTag -eq $tag) { + "tag_exists=true" >> $env:GITHUB_OUTPUT + git show "${tag}:simplicio-update-manifest.json" | Set-Content -Encoding utf8 .release/tag-manifest.json + if ($LASTEXITCODE -ne 0) { throw "Tag $tag has no update manifest." } + } else { + "tag_exists=false" >> $env:GITHUB_OUTPUT + '{}' | Set-Content -Encoding utf8 .release/tag-manifest.json } $headers = @{ Authorization = "Bearer $env:GITHUB_TOKEN" @@ -46,52 +55,64 @@ jobs: if ($response.StatusCode -eq 200) { $response.Content | Set-Content -Encoding utf8 .release/remote-release.json } elseif ($response.StatusCode -eq 404) { - '{"assets":[]}' | Set-Content -Encoding utf8 .release/remote-release.json + '{"exists":false,"assets":[]}' | Set-Content -Encoding utf8 .release/remote-release.json } else { - throw "Could not verify immutable remote release state: HTTP $($response.StatusCode)." + throw "Could not verify remote release state: HTTP $($response.StatusCode)." } - - name: Verify tag provenance and immutable remote assets - run: >- - python scripts/verify_release_provenance.py - --working-manifest simplicio-update-manifest.json - --tag-manifest .release/tag-manifest.json - --remote-release .release/remote-release.json - - name: Download and verify manifest artifacts + - name: Plan immutable release + id: provenance shell: pwsh + env: + ARTIFACT_BASE_URL: ${{ inputs.artifact_base_url }} + run: | + $tagArgument = @() + if ("${{ steps.state.outputs.tag_exists }}" -eq "true") { $tagArgument = @("--tag-exists") } + python scripts/verify_release_provenance.py plan ` + --working-manifest simplicio-update-manifest.json ` + --tag-manifest .release/tag-manifest.json ` + --remote-release .release/remote-release.json ` + --artifact-base-url "$env:ARTIFACT_BASE_URL" ` + --repository "$env:GITHUB_REPOSITORY" ` + --github-output "$env:GITHUB_OUTPUT" @tagArgument + - name: Download new artifacts from immutable staging + id: download + if: steps.provenance.outputs.mode == 'publish' + shell: pwsh + env: + ARTIFACT_BASE_URL: ${{ inputs.artifact_base_url }} run: | $manifest = Get-Content simplicio-update-manifest.json -Raw | ConvertFrom-Json - if (-not $manifest.artifacts -or $manifest.artifacts.Count -eq 0) { - throw "Manifest contains no release artifacts." - } + $base = $env:ARTIFACT_BASE_URL.TrimEnd('/') New-Item -ItemType Directory -Force -Path dist | Out-Null - $releasePrefix = "https://github.com/$env:GITHUB_REPOSITORY/releases/download/v$($manifest.version)/" foreach ($artifact in $manifest.artifacts) { - if (-not $artifact.artifact -or -not $artifact.url -or -not $artifact.sha256) { - throw "Manifest artifact is missing artifact/url/sha256 provenance." - } - $expectedUrl = "$releasePrefix$($artifact.artifact)" - if ($artifact.url -ne $expectedUrl) { - throw "Artifact URL must be version-bound: expected $expectedUrl, got $($artifact.url)." - } - if ($manifest.security.signature_required -and -not $artifact.signature) { - throw "Artifact $($artifact.artifact) is missing its required signature." - } + $source = "$base/$($artifact.artifact)" $destination = Join-Path dist $artifact.artifact - Invoke-WebRequest -Uri $artifact.url -OutFile $destination -UseBasicParsing - $actualHash = (Get-FileHash $destination -Algorithm SHA256).Hash.ToLower() - if ($actualHash -ne $artifact.sha256.ToLower()) { - throw "SHA256 mismatch for $($artifact.artifact): expected $($artifact.sha256), got $actualHash." - } + Invoke-WebRequest -Uri $source -OutFile $destination -UseBasicParsing } + - name: Verify staged bytes and signature metadata + id: verify_staged + if: steps.provenance.outputs.mode == 'publish' + run: >- + python scripts/verify_release_provenance.py verify-staged + --working-manifest simplicio-update-manifest.json + --staging-dir dist + - name: Generate verified release metadata + id: metadata + if: steps.provenance.outputs.mode == 'publish' + shell: pwsh + run: | Copy-Item simplicio-update-manifest.json dist/ Get-ChildItem dist -File | Sort-Object Name | ForEach-Object { "$(($_ | Get-FileHash -Algorithm SHA256).Hash.ToLower()) *$($_.Name)" } | Set-Content -Encoding ascii dist/SHA256SUMS - - name: Create release or upload only new assets + - name: Create release from verified new-tag staging + id: publish + if: steps.provenance.outputs.mode == 'publish' uses: softprops/action-gh-release@v2 with: - tag_name: v${{ steps.ver.outputs.version }} - name: "v${{ steps.ver.outputs.version }} — Public Beta" + tag_name: v${{ steps.version.outputs.version }} + target_commitish: ${{ github.sha }} + name: "v${{ steps.version.outputs.version }} — Public Beta" body: | Free public beta. All features remain unlocked during the public-beta phase. diff --git a/CHANGELOG.md b/CHANGELOG.md index 89291d3..ad39c0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Made publication manual-only and immutable: tag-bound manifest provenance must match exactly, existing release assets cannot be replaced, and the known stale `v3.5.2` tag now fails closed pending separate fresh-release preparation. +- Restored a safe new-release path through distinct versioned HTTPS staging, with + idempotent no-op success for coherent existing releases and structured PyYAML + validation of real workflow steps, conditions, ordering, and action inputs. ## [1.6.1] - 2026-07-01 diff --git a/QUALITY.md b/QUALITY.md index bf00002..fb974a0 100644 --- a/QUALITY.md +++ b/QUALITY.md @@ -48,16 +48,24 @@ updated baseline; the environment override is not configured on protected branch ## Release provenance -`publish-release` is manual-only; merging a CI or manifest change cannot trigger publication. Before -any download or release mutation, it reads `simplicio-update-manifest.json` from the immutable version -tag and requires exact version plus artifact name/URL/SHA256/signature equality with the working tree. -It also reads the remote release and refuses any declared or generated asset name that already exists; -the release action has `overwrite_files: false` as a second fail-closed guard. +`publish-release` is manual-only; merging a CI or manifest change cannot trigger publication. Its +required `artifact_base_url` input must be an HTTPS, version-segmented staging origin distinct from +the target GitHub release URL. The workflow reads tag and release state before any mutation, then a +reusable Python planner chooses exactly one mode: + +- `idempotent`: an existing tag manifest matches version plus artifact name/URL/SHA256/signature and + every remote artifact has the declared SHA256 digest. Download, metadata, and publish steps are skipped. +- `publish`: neither tag nor release exists and the staging URL is safe. Artifacts are downloaded from + staging, never from the not-yet-existing target release, and their bytes plus signature metadata are + verified before release creation with `overwrite_files: false`. The existing `v3.5.2` tag contains a `3.5.1` manifest, so the manual workflow is intentionally blocked for that tag and must not move it or replace its assets. Preparing a fresh version, immutable tag, and -signed artifact set is separate release work. Once those inputs agree, the workflow downloads each -version-bound artifact, verifies SHA256, and uploads only new assets from the verified staging directory. +signed staging set is separate release work. + +The strict distribution audit parses workflow YAML using pinned PyYAML and validates the actual trigger, +input, `jobs.release.steps`, executable commands, action inputs, conditions, and ordering. Comments and +environment variables cannot satisfy executable-step requirements. ## Local parity diff --git a/requirements-quality.txt b/requirements-quality.txt new file mode 100644 index 0000000..4072e32 --- /dev/null +++ b/requirements-quality.txt @@ -0,0 +1,2 @@ +coverage==7.6.12 +PyYAML==6.0.3 diff --git a/scripts/verify_distribution_consistency.py b/scripts/verify_distribution_consistency.py index d2e67bb..5117621 100644 --- a/scripts/verify_distribution_consistency.py +++ b/scripts/verify_distribution_consistency.py @@ -13,6 +13,8 @@ from pathlib import Path from typing import Iterable, Sequence +import yaml + ROOT = Path(__file__).resolve().parents[1] CANONICAL_BRANCH = "master" MAIN_INSTALL_RE = re.compile( @@ -80,48 +82,111 @@ def iter_install_reference_files(root: Path) -> Iterable[Path]: yield from sorted((root / "READMEs").glob("README*.md")) +def executable_run(step: dict) -> str: + run = step.get("run") + if not isinstance(run, str): + return "" + return "\n".join( + line.strip().lower() + for line in run.splitlines() + if line.strip() and not line.lstrip().startswith("#") + ) + + def release_workflow_errors(workflow: str) -> list[str]: - lower = workflow.lower() + try: + document = yaml.safe_load(workflow) + except yaml.YAMLError as exc: + return [f"release workflow YAML is invalid: {exc}"] + if not isinstance(document, dict): + return ["release workflow must be a YAML mapping"] errors: list[str] = [] - if not re.search(r"(?m)^on:\s*\n\s{2}workflow_dispatch:\s*(?:\{\})?\s*$", workflow): - errors.append("release workflow is not manual-only") - automatic_triggers = re.findall(r"(?m)^\s{2}(push|pull_request|schedule):", workflow) - if automatic_triggers: - errors.append("release workflow has automatic trigger: " + ", ".join(sorted(set(automatic_triggers)))) - required_tokens = ( - "git show-ref --verify --quiet", - 'git show "${tag}:simplicio-update-manifest.json"', - "python scripts/verify_release_provenance.py", - "--working-manifest simplicio-update-manifest.json", - "--tag-manifest .release/tag-manifest.json", - "--remote-release .release/remote-release.json", - "foreach ($artifact in $manifest.artifacts)", - "invoke-webrequest -uri $artifact.url", - "get-filehash $destination -algorithm sha256", - "if ($actualhash -ne $artifact.sha256.tolower())", - "fail_on_unmatched_files: true", - "overwrite_files: false", - "files: dist/*", - "uses: softprops/action-gh-release@v2", - ) - errors.extend(f"missing {token}" for token in required_tokens if token not in lower) - if "overwrite_files: true" in lower: - errors.append("overwrite_files must never be true") - unsafe_lines = ( - "\n - simplicio\n", - "\n - simplicio.exe\n", - "copy-item simplicio ", - "copy-item simplicio.exe ", - "cp simplicio ", - ) - errors.extend(f"unsafe {token.strip()}" for token in unsafe_lines if token in lower) - verifier_index = lower.find("python scripts/verify_release_provenance.py") - download_index = lower.find("invoke-webrequest -uri $artifact.url") - mutation_index = lower.find("uses: softprops/action-gh-release@v2") - if min(verifier_index, download_index, mutation_index) >= 0 and not ( - verifier_index < download_index < mutation_index + triggers = document.get("on") + if not isinstance(triggers, dict) or set(triggers) != {"workflow_dispatch"}: + errors.append("release workflow must have only workflow_dispatch trigger") + dispatch = {} + else: + dispatch = triggers.get("workflow_dispatch") or {} + inputs = dispatch.get("inputs") if isinstance(dispatch, dict) else None + base_input = inputs.get("artifact_base_url") if isinstance(inputs, dict) else None + if not isinstance(base_input, dict) or base_input.get("required") is not True or base_input.get("type") != "string": + errors.append("workflow_dispatch must require string input artifact_base_url") + + jobs = document.get("jobs") + release = jobs.get("release") if isinstance(jobs, dict) else None + steps = release.get("steps") if isinstance(release, dict) else None + if not isinstance(steps, list) or not steps or not all(isinstance(step, dict) for step in steps): + errors.append("jobs.release.steps must be a non-empty list of mappings") + return errors + indexed: dict[str, tuple[int, dict]] = {} + for index, step in enumerate(steps): + step_id = step.get("id") + if isinstance(step_id, str): + if step_id in indexed: + errors.append(f"duplicate release step id: {step_id}") + indexed[step_id] = (index, step) + required_ids = ("state", "provenance", "download", "verify_staged", "metadata", "publish") + missing_ids = [step_id for step_id in required_ids if step_id not in indexed] + if missing_ids: + errors.append("missing release step ids: " + ", ".join(missing_ids)) + return errors + positions = [indexed[step_id][0] for step_id in required_ids] + if positions != sorted(positions) or len(set(positions)) != len(positions): + errors.append("release steps must order state, provenance, download, verify_staged, metadata, publish") + + install_runs = [executable_run(step) for step in steps] + if not any("pip install -r requirements-quality.txt" in run for run in install_runs): + errors.append("release job must install pinned requirements-quality.txt") + + state_run = executable_run(indexed["state"][1]) + for command in ("git tag --list", "git show \"${tag}:simplicio-update-manifest.json\"", "remote-release.json"): + if command not in state_run: + errors.append(f"state step lacks executable command: {command}") + provenance_run = executable_run(indexed["provenance"][1]) + for argument in ( + "scripts/verify_release_provenance.py plan", + "--working-manifest", + "--tag-manifest", + "--remote-release", + "--artifact-base-url", + "--github-output", + "--tag-exists", ): - errors.append("tag provenance verification must run before download and release mutation") + if argument not in provenance_run: + errors.append(f"provenance step lacks executable argument: {argument}") + + publish_condition = "steps.provenance.outputs.mode == 'publish'" + guarded_ids = ("download", "verify_staged", "metadata", "publish") + for step_id in guarded_ids: + if indexed[step_id][1].get("if") != publish_condition: + errors.append(f"{step_id} step must be guarded by publish mode") + download_run = executable_run(indexed["download"][1]) + if "$env:artifact_base_url" not in download_run or "invoke-webrequest -uri $source" not in download_run: + errors.append("download step must source artifacts from artifact_base_url") + if "$artifact.url" in download_run or "releases/download" in download_run: + errors.append("download step must not source a new binary from its target release URL") + if "copy-item simplicio " in download_run or "copy-item simplicio.exe " in download_run: + errors.append("download step must not stage checked-in wrappers") + + staged_run = executable_run(indexed["verify_staged"][1]) + if "scripts/verify_release_provenance.py verify-staged" not in staged_run or "--staging-dir" not in staged_run: + errors.append("verify_staged step must execute the reusable digest verifier") + publish = indexed["publish"][1] + if publish.get("uses") != "softprops/action-gh-release@v2": + errors.append("publish step must use softprops/action-gh-release@v2") + publish_with = publish.get("with") + if not isinstance(publish_with, dict): + errors.append("publish step must define structured with inputs") + else: + if publish_with.get("overwrite_files") is not False: + errors.append("publish step must set overwrite_files: false") + if publish_with.get("fail_on_unmatched_files") is not True: + errors.append("publish step must set fail_on_unmatched_files: true") + if publish_with.get("files") != "dist/*": + errors.append("publish step must upload only dist/*") + release_actions = [step for step in steps if step.get("uses") == "softprops/action-gh-release@v2"] + if len(release_actions) != 1: + errors.append("release job must contain exactly one publish action") return errors @@ -225,7 +290,7 @@ def run_audit(root: Path = ROOT, *, today: date | None = None) -> list[Finding]: if release_errors: findings.append(Finding("ERROR", "release workflow provenance is not fail-closed: " + ", ".join(release_errors))) else: - findings.append(Finding("OK", "manual release verifies tag provenance and immutable assets before upload.")) + findings.append(Finding("OK", "structured manual release supports idempotent state or verified staging publish.")) ecosystem = read_text(root / "SIMPLICIO_ECOSYSTEM.md") ecosystem_match = ECOSYSTEM_VERSION_RE.search(ecosystem) diff --git a/scripts/verify_release_provenance.py b/scripts/verify_release_provenance.py index 45c0a16..059cc80 100644 --- a/scripts/verify_release_provenance.py +++ b/scripts/verify_release_provenance.py @@ -1,16 +1,24 @@ #!/usr/bin/env python3 -"""Fail closed when a release would mutate tag-bound artifact provenance.""" +"""Plan immutable releases and verify bytes from a distinct staging origin.""" from __future__ import annotations import argparse +import hashlib import json import sys +from dataclasses import dataclass from pathlib import Path from typing import Sequence +from urllib.parse import urlparse PROVENANCE_FIELDS = ("artifact", "url", "sha256", "signature") -GENERATED_RELEASE_ASSETS = {"simplicio-update-manifest.json", "SHA256SUMS"} + + +@dataclass(frozen=True) +class ReleasePlan: + mode: str + errors: tuple[str, ...] = () def load_json(path: Path) -> dict: @@ -39,11 +47,44 @@ def provenance_snapshot(manifest: dict) -> tuple[str, tuple[tuple[str, str, str, raise ValueError(f"manifest artifact {index} is missing: {', '.join(missing)}") if values[0] in names: raise ValueError(f"manifest contains duplicate artifact name: {values[0]}") + if Path(values[0]).name != values[0] or "/" in values[0] or "\\" in values[0]: + raise ValueError(f"manifest artifact name is not a safe filename: {values[0]}") + if len(values[2]) != 64 or any(character not in "0123456789abcdefABCDEF" for character in values[2]): + raise ValueError(f"manifest artifact {values[0]} has invalid SHA256") + if not values[3].startswith("ed25519:"): + raise ValueError(f"manifest artifact {values[0]} lacks Ed25519 signature metadata") names.add(values[0]) - records.append(values) + records.append((values[0], values[1], values[2].lower(), values[3])) return version, tuple(sorted(records)) +def validate_staging_base_url(base_url: str, version: str, repository: str) -> list[str]: + parsed = urlparse(base_url) + errors: list[str] = [] + if parsed.scheme != "https" or not parsed.netloc: + errors.append("artifact_base_url must be absolute HTTPS") + if parsed.query or parsed.fragment or parsed.username or parsed.password: + errors.append("artifact_base_url must not contain credentials, query, or fragment") + segments = [segment for segment in parsed.path.split("/") if segment] + if f"v{version}" not in segments: + errors.append(f"artifact_base_url must contain immutable version segment v{version}") + target = f"https://github.com/{repository}/releases/download/v{version}".rstrip("/").lower() + normalized = base_url.rstrip("/").lower() + if normalized == target or normalized.startswith(target + "/"): + errors.append("artifact_base_url must be distinct from the target release URL") + return errors + + +def target_url_errors(working: dict, repository: str) -> list[str]: + version, artifacts = provenance_snapshot(working) + errors: list[str] = [] + for name, url, _, _ in artifacts: + expected = f"https://github.com/{repository}/releases/download/v{version}/{name}" + if url != expected: + errors.append(f"target release URL mismatch for {name}") + return errors + + def compare_tag_provenance(working: dict, tagged: dict) -> list[str]: working_version, working_artifacts = provenance_snapshot(working) tagged_version, tagged_artifacts = provenance_snapshot(tagged) @@ -57,49 +98,140 @@ def compare_tag_provenance(working: dict, tagged: dict) -> list[str]: return errors -def existing_asset_conflicts(working: dict, remote_release: dict) -> list[str]: +def remote_exists(remote_release: dict) -> bool: + explicit = remote_release.get("exists") + if isinstance(explicit, bool): + return explicit + return bool(remote_release.get("id") or remote_release.get("tag_name")) + + +def remote_digest_errors(working: dict, remote_release: dict) -> list[str]: _, artifacts = provenance_snapshot(working) - declared_names = {record[0] for record in artifacts} | GENERATED_RELEASE_ASSETS - assets = remote_release.get("assets", []) + assets = remote_release.get("assets") if not isinstance(assets, list): - raise ValueError("remote release assets must be a list") - existing_names = { - str(asset.get("name") or "") - for asset in assets - if isinstance(asset, dict) and asset.get("name") - } - return sorted(declared_names & existing_names) - - -def verify(working: dict, tagged: dict, remote_release: dict | None = None) -> list[str]: - errors = compare_tag_provenance(working, tagged) - if remote_release is not None: - conflicts = existing_asset_conflicts(working, remote_release) - if conflicts: - errors.append("immutable release already contains assets: " + ", ".join(conflicts)) + return ["remote release assets must be a list"] + by_name: dict[str, list[dict]] = {} + for asset in assets: + if isinstance(asset, dict) and asset.get("name"): + by_name.setdefault(str(asset["name"]), []).append(asset) + errors: list[str] = [] + for name, _, sha256, _ in artifacts: + matches = by_name.get(name, []) + if len(matches) != 1: + errors.append(f"remote release must contain exactly one {name} asset") + continue + digest = str(matches[0].get("digest") or "").lower() + if digest != f"sha256:{sha256}": + errors.append(f"remote asset digest mismatch for {name}") return errors -def main(argv: Sequence[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--working-manifest", type=Path, required=True) - parser.add_argument("--tag-manifest", type=Path, required=True) - parser.add_argument("--remote-release", type=Path) - args = parser.parse_args(argv) +def plan_release( + working: dict, + *, + tag_exists: bool, + tagged: dict | None, + remote_release: dict, + artifact_base_url: str, + repository: str, +) -> ReleasePlan: + version, _ = provenance_snapshot(working) + errors = validate_staging_base_url(artifact_base_url, version, repository) + errors.extend(target_url_errors(working, repository)) + release_exists = remote_exists(remote_release) + if tag_exists: + if tagged is None: + errors.append("existing tag is missing its manifest") + else: + errors.extend(compare_tag_provenance(working, tagged)) + if not release_exists: + errors.append("existing tag has no corresponding release; refusing mutation") + else: + errors.extend(remote_digest_errors(working, remote_release)) + return ReleasePlan("blocked" if errors else "idempotent", tuple(errors)) + if tagged is not None: + errors.append("tag manifest was supplied for a tag reported as absent") + if release_exists: + errors.append("remote release exists while target tag is absent") + return ReleasePlan("blocked" if errors else "publish", tuple(errors)) + + +def verify_staged_files(working: dict, staging_dir: Path) -> list[str]: + _, artifacts = provenance_snapshot(working) + errors: list[str] = [] + for name, _, expected_sha256, _ in artifacts: + path = staging_dir / name + if not path.is_file(): + errors.append(f"staged artifact is missing: {name}") + continue + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected_sha256: + errors.append(f"staged artifact digest mismatch for {name}") + return errors + + +def write_output(path: Path | None, mode: str) -> None: + if path: + with path.open("a", encoding="utf-8") as stream: + stream.write(f"mode={mode}\n") + + +def plan_command(args: argparse.Namespace) -> int: try: working = load_json(args.working_manifest) - tagged = load_json(args.tag_manifest) - remote = load_json(args.remote_release) if args.remote_release else None - errors = verify(working, tagged, remote) + tagged = load_json(args.tag_manifest) if args.tag_exists else None + remote = load_json(args.remote_release) + plan = plan_release( + working, + tag_exists=args.tag_exists, + tagged=tagged, + remote_release=remote, + artifact_base_url=args.artifact_base_url, + repository=args.repository, + ) + except (OSError, ValueError, json.JSONDecodeError) as exc: + plan = ReleasePlan("blocked", (f"invalid release provenance input: {exc}",)) + write_output(args.github_output, plan.mode) + if plan.errors: + for error in plan.errors: + print(f"[ERROR] {error}") + return 1 + print(f"release-provenance: PASS mode={plan.mode}") + return 0 + + +def staged_command(args: argparse.Namespace) -> int: + try: + errors = verify_staged_files(load_json(args.working_manifest), args.staging_dir) except (OSError, ValueError, json.JSONDecodeError) as exc: - errors = [f"invalid release provenance input: {exc}"] + errors = [f"invalid staged provenance input: {exc}"] if errors: for error in errors: print(f"[ERROR] {error}") return 1 - print("release-provenance: PASS (tag and remote assets are immutable)") + print("release-provenance: staged artifact digests PASS") return 0 +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + plan = subparsers.add_parser("plan") + plan.add_argument("--working-manifest", type=Path, required=True) + plan.add_argument("--tag-exists", action="store_true") + plan.add_argument("--tag-manifest", type=Path, required=True) + plan.add_argument("--remote-release", type=Path, required=True) + plan.add_argument("--artifact-base-url", required=True) + plan.add_argument("--repository", required=True) + plan.add_argument("--github-output", type=Path) + plan.set_defaults(handler=plan_command) + staged = subparsers.add_parser("verify-staged") + staged.add_argument("--working-manifest", type=Path, required=True) + staged.add_argument("--staging-dir", type=Path, required=True) + staged.set_defaults(handler=staged_command) + args = parser.parse_args(argv) + return args.handler(args) + + if __name__ == "__main__": sys.exit(main()) diff --git a/tests/test_distribution_consistency.py b/tests/test_distribution_consistency.py index a39303b..384ae48 100644 --- a/tests/test_distribution_consistency.py +++ b/tests/test_distribution_consistency.py @@ -8,6 +8,8 @@ from datetime import date from pathlib import Path +import yaml + from scripts import verify_distribution_consistency as audit @@ -35,23 +37,45 @@ def write(self) -> None: self.put(relative, "https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.sh\n") self.put( ".github/workflows/release.yml", - "on:\n" - " workflow_dispatch: {}\n" - "git show-ref --verify --quiet refs/tags/v3.5.2\n" - 'git show "${tag}:simplicio-update-manifest.json"\n' - "python scripts/verify_release_provenance.py \n" - " --working-manifest simplicio-update-manifest.json \n" - " --tag-manifest .release/tag-manifest.json \n" - " --remote-release .release/remote-release.json\n" - "foreach ($artifact in $manifest.artifacts) {\n" - " Invoke-WebRequest -Uri $artifact.url -OutFile $destination\n" - " $actualHash = (Get-FileHash $destination -Algorithm SHA256).Hash.ToLower()\n" - " if ($actualHash -ne $artifact.sha256.ToLower()) { throw 'mismatch' }\n" - "}\n" - "fail_on_unmatched_files: true\n" - "overwrite_files: false\n" - "uses: softprops/action-gh-release@v2\n" - "files: dist/*\n", + '''"on": + workflow_dispatch: + inputs: + artifact_base_url: + required: true + type: string +jobs: + release: + steps: + - run: python -m pip install -r requirements-quality.txt + - id: state + run: | + git tag --list v3.5.2 + git show "${tag}:simplicio-update-manifest.json" + echo remote-release.json + - id: provenance + run: | + python scripts/verify_release_provenance.py plan --working-manifest manifest.json + --tag-manifest tag.json --remote-release remote.json --artifact-base-url "$base" + --github-output "$output" --tag-exists + - id: download + if: steps.provenance.outputs.mode == 'publish' + run: | + $source = "$env:ARTIFACT_BASE_URL/$name" + Invoke-WebRequest -Uri $source -OutFile $destination + - id: verify_staged + if: steps.provenance.outputs.mode == 'publish' + run: python scripts/verify_release_provenance.py verify-staged --staging-dir dist + - id: metadata + if: steps.provenance.outputs.mode == 'publish' + run: echo metadata + - id: publish + if: steps.provenance.outputs.mode == 'publish' + uses: softprops/action-gh-release@v2 + with: + fail_on_unmatched_files: true + overwrite_files: false + files: dist/* +''', ) self.put("version.txt", self.version + "\n") artifact_url = ( @@ -172,30 +196,124 @@ def test_regression_formula_must_install_versioned_asset(self): def test_regression_release_must_not_stage_repo_wrapper_binary(self): workflow = self.root / ".github/workflows/release.yml" - workflow.write_text(workflow.read_text(encoding="utf-8") + "Copy-Item simplicio dist/simplicio-macos-arm64\n", encoding="utf-8") + document = yaml.safe_load(workflow.read_text(encoding="utf-8")) + download = next(step for step in document["jobs"]["release"]["steps"] if step.get("id") == "download") + download["run"] += "\nCopy-Item simplicio dist/simplicio-macos-arm64" + workflow.write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8") errors = [item.message for item in audit.run_audit(self.root) if item.level == "ERROR"] - self.assertTrue(any("release workflow provenance" in message and "unsafe" in message for message in errors)) + self.assertTrue(any("release workflow provenance" in message and "checked-in wrappers" in message for message in errors)) def test_regression_release_must_be_manual_only(self): workflow = self.root / ".github/workflows/release.yml" - workflow.write_text(workflow.read_text(encoding="utf-8").replace(" workflow_dispatch: {}", " push:\n workflow_dispatch: {}"), encoding="utf-8") - errors = audit.release_workflow_errors(workflow.read_text(encoding="utf-8")) - self.assertTrue(any("automatic trigger: push" in error for error in errors)) + document = yaml.safe_load(workflow.read_text(encoding="utf-8")) + document["on"]["push"] = {"branches": ["master"]} + errors = audit.release_workflow_errors(yaml.safe_dump(document, sort_keys=False)) + self.assertTrue(any("only workflow_dispatch" in error for error in errors)) def test_regression_release_requires_tag_bound_verifier(self): workflow = self.root / ".github/workflows/release.yml" - workflow.write_text(workflow.read_text(encoding="utf-8").replace("python scripts/verify_release_provenance.py", "python scripts/untrusted.py"), encoding="utf-8") - errors = audit.release_workflow_errors(workflow.read_text(encoding="utf-8")) - self.assertTrue(any("missing python scripts/verify_release_provenance.py" in error for error in errors)) + document = yaml.safe_load(workflow.read_text(encoding="utf-8")) + provenance_step = next(step for step in document["jobs"]["release"]["steps"] if step.get("id") == "provenance") + provenance_step["run"] = provenance_step["run"].replace("scripts/verify_release_provenance.py", "scripts/untrusted.py") + errors = audit.release_workflow_errors(yaml.safe_dump(document, sort_keys=False)) + self.assertTrue(any("provenance step lacks executable" in error for error in errors)) def test_regression_release_requires_explicit_no_overwrite(self): workflow = self.root / ".github/workflows/release.yml" - clean = workflow.read_text(encoding="utf-8") - for unsafe in (clean.replace("overwrite_files: false\n", ""), clean.replace("overwrite_files: false", "overwrite_files: true")): - with self.subTest(unsafe="absent" if "overwrite_files:" not in unsafe else "true"): - errors = audit.release_workflow_errors(unsafe) + clean = yaml.safe_load(workflow.read_text(encoding="utf-8")) + for value in (None, True): + document = json.loads(json.dumps(clean)) + publish = next(step for step in document["jobs"]["release"]["steps"] if step.get("id") == "publish") + if value is None: + del publish["with"]["overwrite_files"] + else: + publish["with"]["overwrite_files"] = value + with self.subTest(value=value): + errors = audit.release_workflow_errors(yaml.safe_dump(document, sort_keys=False)) self.assertTrue(any("overwrite_files" in error for error in errors)) + def test_adversarial_comments_and_env_cannot_spoof_executable_workflow(self): + for workflow in ( + '''"on": + workflow_dispatch: + inputs: + artifact_base_url: {required: true, type: string} +# jobs: release: steps: verify_release_provenance overwrite_files: false +''', + '''"on": + workflow_dispatch: + inputs: + artifact_base_url: {required: true, type: string} +env: + SPOOF: "jobs release steps verify_release_provenance overwrite_files false" +''', + ): + with self.subTest(workflow=workflow): + errors = audit.release_workflow_errors(workflow) + self.assertTrue(any("jobs.release.steps" in error for error in errors)) + workflow = self.root / ".github/workflows/release.yml" + document = yaml.safe_load(workflow.read_text(encoding="utf-8")) + provenance_step = next(step for step in document["jobs"]["release"]["steps"] if step.get("id") == "provenance") + provenance_step["env"] = {"SPOOF": "scripts/verify_release_provenance.py plan --tag-exists"} + provenance_step["run"] = "# python scripts/verify_release_provenance.py plan --working-manifest --tag-manifest --remote-release --artifact-base-url --github-output --tag-exists" + errors = audit.release_workflow_errors(yaml.safe_dump(document, sort_keys=False)) + self.assertTrue(any("provenance step lacks executable" in error for error in errors)) + + def test_regression_release_step_order_is_semantic(self): + workflow = self.root / ".github/workflows/release.yml" + document = yaml.safe_load(workflow.read_text(encoding="utf-8")) + steps = document["jobs"]["release"]["steps"] + provenance_index = next(index for index, step in enumerate(steps) if step.get("id") == "provenance") + download_index = next(index for index, step in enumerate(steps) if step.get("id") == "download") + steps[provenance_index], steps[download_index] = steps[download_index], steps[provenance_index] + errors = audit.release_workflow_errors(yaml.safe_dump(document, sort_keys=False)) + self.assertTrue(any("must order" in error for error in errors)) + + def test_structured_workflow_rejects_malformed_step_shapes(self): + workflow = self.root / ".github/workflows/release.yml" + clean = yaml.safe_load(workflow.read_text(encoding="utf-8")) + + self.assertTrue(any("YAML is invalid" in error for error in audit.release_workflow_errors("jobs: ["))) + self.assertTrue(any("YAML mapping" in error for error in audit.release_workflow_errors("[]"))) + + variants = [] + duplicate = json.loads(json.dumps(clean)) + duplicate["jobs"]["release"]["steps"].append({"id": "state", "run": "echo duplicate"}) + variants.append(duplicate) + missing = json.loads(json.dumps(clean)) + missing["jobs"]["release"]["steps"] = [step for step in missing["jobs"]["release"]["steps"] if step.get("id") != "publish"] + variants.append(missing) + no_install = json.loads(json.dumps(clean)) + no_install["jobs"]["release"]["steps"][0]["run"] = "echo no dependencies" + variants.append(no_install) + bad_state = json.loads(json.dumps(clean)) + next(step for step in bad_state["jobs"]["release"]["steps"] if step.get("id") == "state")["run"] = "echo remote-release.json" + variants.append(bad_state) + bad_guard = json.loads(json.dumps(clean)) + next(step for step in bad_guard["jobs"]["release"]["steps"] if step.get("id") == "download")["if"] = "always()" + variants.append(bad_guard) + bad_download = json.loads(json.dumps(clean)) + next(step for step in bad_download["jobs"]["release"]["steps"] if step.get("id") == "download")["run"] = "Invoke-WebRequest -Uri $artifact.url # releases/download" + variants.append(bad_download) + bad_staged = json.loads(json.dumps(clean)) + next(step for step in bad_staged["jobs"]["release"]["steps"] if step.get("id") == "verify_staged")["run"] = "echo unchecked" + variants.append(bad_staged) + bad_action = json.loads(json.dumps(clean)) + next(step for step in bad_action["jobs"]["release"]["steps"] if step.get("id") == "publish")["uses"] = "example/unsafe@v1" + variants.append(bad_action) + bad_with = json.loads(json.dumps(clean)) + next(step for step in bad_with["jobs"]["release"]["steps"] if step.get("id") == "publish")["with"] = "spoof" + variants.append(bad_with) + bad_inputs = json.loads(json.dumps(clean)) + publish = next(step for step in bad_inputs["jobs"]["release"]["steps"] if step.get("id") == "publish") + publish["with"]["fail_on_unmatched_files"] = False + publish["with"]["files"] = "**/*" + bad_inputs["jobs"]["release"]["steps"].append({"uses": "softprops/action-gh-release@v2"}) + variants.append(bad_inputs) + for document in variants: + with self.subTest(document=document): + self.assertTrue(audit.release_workflow_errors(yaml.safe_dump(document, sort_keys=False))) + def test_regression_manifest_url_must_be_version_bound(self): manifest_path = self.root / "simplicio-update-manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) diff --git a/tests/test_release_provenance.py b/tests/test_release_provenance.py index 144d246..88a6c6e 100644 --- a/tests/test_release_provenance.py +++ b/tests/test_release_provenance.py @@ -1,6 +1,7 @@ from __future__ import annotations import contextlib +import hashlib import io import json import tempfile @@ -9,104 +10,256 @@ from scripts import verify_release_provenance as provenance +REPOSITORY = "wesleysimplicio/simplicio" +STAGING = "https://artifacts.example/simplicio/v3.5.2" -def manifest(version: str = "3.5.2") -> dict: + +def manifest(version: str = "3.5.2", *, payload: bytes | None = None) -> dict: name = "simplicio-macos-arm64" + sha256 = hashlib.sha256(payload).hexdigest() if payload is not None else "9" * 64 return { "version": version, "artifacts": [ { "artifact": name, - "url": f"https://github.com/wesleysimplicio/simplicio/releases/download/v{version}/{name}", - "sha256": "9" * 64, + "url": f"https://github.com/{REPOSITORY}/releases/download/v{version}/{name}", + "sha256": sha256, "signature": "ed25519:fixture", } ], } +def remote_release(value: dict, *, digest: str | None = None) -> dict: + artifact = value["artifacts"][0] + return { + "id": 123, + "tag_name": f"v{value['version']}", + "assets": [ + { + "name": artifact["artifact"], + "digest": digest or f"sha256:{artifact['sha256']}", + } + ], + } + + class ReleaseProvenanceTests(unittest.TestCase): - def test_equal_tag_provenance_and_new_assets_pass(self): + def test_existing_coherent_tag_is_idempotent_no_publish(self): working = manifest() - self.assertEqual(provenance.verify(working, manifest(), {"assets": []}), []) + plan = provenance.plan_release( + working, + tag_exists=True, + tagged=manifest(), + remote_release=remote_release(working), + artifact_base_url=STAGING, + repository=REPOSITORY, + ) + self.assertEqual(plan, provenance.ReleasePlan("idempotent")) - def test_known_tag_version_and_artifact_drift_fail_closed(self): - errors = provenance.verify(manifest("3.5.2"), manifest("3.5.1"), {"assets": []}) - self.assertEqual(len(errors), 2) - self.assertIn("tag manifest version 3.5.1 does not equal working manifest version 3.5.2", errors) - self.assertTrue(any("artifact name/url/sha256/signature" in error for error in errors)) + def test_existing_mismatched_tag_blocks_without_mutation(self): + working = manifest("3.5.2") + plan = provenance.plan_release( + working, + tag_exists=True, + tagged=manifest("3.5.1"), + remote_release=remote_release(working), + artifact_base_url=STAGING, + repository=REPOSITORY, + ) + self.assertEqual(plan.mode, "blocked") + self.assertTrue(any("3.5.1" in error and "3.5.2" in error for error in plan.errors)) + self.assertTrue(any("artifact name/url/sha256/signature" in error for error in plan.errors)) - def test_any_existing_declared_or_generated_asset_blocks_upload(self): + def test_new_tag_with_distinct_versioned_staging_is_publish_ready(self): + plan = provenance.plan_release( + manifest(), + tag_exists=False, + tagged=None, + remote_release={"exists": False, "assets": []}, + artifact_base_url=STAGING, + repository=REPOSITORY, + ) + self.assertEqual(plan, provenance.ReleasePlan("publish")) + + def test_missing_or_unsafe_staging_blocks(self): + for base, message in ( + ("", "absolute HTTPS"), + ("http://artifacts.example/v3.5.2", "absolute HTTPS"), + ("https://artifacts.example/latest", "immutable version segment"), + ( + "https://github.com/wesleysimplicio/simplicio/releases/download/v3.5.2", + "distinct from the target release", + ), + ( + "https://github.com/wesleysimplicio/simplicio/releases/download/v3.5.2/staging", + "distinct from the target release", + ), + ("https://user:secret@artifacts.example/v3.5.2?token=x", "credentials, query, or fragment"), + ): + with self.subTest(base=base): + plan = provenance.plan_release( + manifest(), + tag_exists=False, + tagged=None, + remote_release={"exists": False, "assets": []}, + artifact_base_url=base, + repository=REPOSITORY, + ) + self.assertEqual(plan.mode, "blocked") + self.assertTrue(any(message in error for error in plan.errors)) + + def test_existing_release_requires_exact_remote_digest(self): working = manifest() - for name in ("simplicio-macos-arm64", "simplicio-update-manifest.json", "SHA256SUMS"): - with self.subTest(name=name): - errors = provenance.verify(working, manifest(), {"assets": [{"name": name}]}) - self.assertEqual(errors, [f"immutable release already contains assets: {name}"]) + for remote, message in ( + ({"exists": False, "assets": []}, "no corresponding release"), + ({"id": 123, "assets": []}, "exactly one"), + (remote_release(working, digest="sha256:" + "8" * 64), "digest mismatch"), + ): + with self.subTest(message=message): + plan = provenance.plan_release( + working, + tag_exists=True, + tagged=manifest(), + remote_release=remote, + artifact_base_url=STAGING, + repository=REPOSITORY, + ) + self.assertEqual(plan.mode, "blocked") + self.assertTrue(any(message in error for error in plan.errors)) - def test_invalid_or_duplicate_provenance_is_rejected(self): - broken = manifest() - del broken["artifacts"][0]["signature"] - with self.assertRaisesRegex(ValueError, "missing: signature"): - provenance.provenance_snapshot(broken) - duplicate = manifest() - duplicate["artifacts"].append(dict(duplicate["artifacts"][0])) - with self.assertRaisesRegex(ValueError, "duplicate artifact name"): - provenance.provenance_snapshot(duplicate) + def test_new_tag_rejects_existing_release_or_supplied_tag_manifest(self): + working = manifest() + plan = provenance.plan_release( + working, + tag_exists=False, + tagged=manifest(), + remote_release=remote_release(working), + artifact_base_url=STAGING, + repository=REPOSITORY, + ) + self.assertEqual(plan.mode, "blocked") + self.assertEqual(len(plan.errors), 2) - def test_malformed_manifest_and_remote_shapes_are_rejected(self): + def test_staged_artifact_must_exist_and_match_digest(self): + payload = b"signed release bytes" + working = manifest(payload=payload) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.assertEqual(provenance.verify_staged_files(working, root), ["staged artifact is missing: simplicio-macos-arm64"]) + artifact = root / "simplicio-macos-arm64" + artifact.write_bytes(b"wrong") + self.assertEqual(provenance.verify_staged_files(working, root), ["staged artifact digest mismatch for simplicio-macos-arm64"]) + artifact.write_bytes(payload) + self.assertEqual(provenance.verify_staged_files(working, root), []) + + def test_invalid_manifest_shapes_and_signature_are_rejected(self): for broken, message in ( ({"artifacts": []}, "version is missing"), ({"version": "3.5.2", "artifacts": []}, "non-empty list"), - ({"version": "3.5.2", "artifacts": ["not-an-object"]}, "not an object"), + ({"version": "3.5.2", "artifacts": ["bad"]}, "not an object"), ): with self.subTest(message=message), self.assertRaisesRegex(ValueError, message): provenance.provenance_snapshot(broken) - with self.assertRaisesRegex(ValueError, "remote release assets must be a list"): - provenance.existing_asset_conflicts(manifest(), {"assets": "unknown"}) + broken = manifest() + broken["artifacts"][0]["signature"] = "unsigned" + with self.assertRaisesRegex(ValueError, "Ed25519"): + provenance.provenance_snapshot(broken) + broken = manifest() + broken["artifacts"][0]["sha256"] = "z" * 64 + with self.assertRaisesRegex(ValueError, "invalid SHA256"): + provenance.provenance_snapshot(broken) + broken = manifest() + del broken["artifacts"][0]["url"] + with self.assertRaisesRegex(ValueError, "missing: url"): + provenance.provenance_snapshot(broken) + duplicate = manifest() + duplicate["artifacts"].append(dict(duplicate["artifacts"][0])) + with self.assertRaisesRegex(ValueError, "duplicate artifact"): + provenance.provenance_snapshot(duplicate) + unsafe = manifest() + unsafe["artifacts"][0]["artifact"] = "../escape" + with self.assertRaisesRegex(ValueError, "safe filename"): + provenance.provenance_snapshot(unsafe) - def test_cli_reports_tag_mismatch_and_returns_nonzero(self): + def test_plan_rejects_target_url_and_malformed_existing_state(self): + wrong_target = manifest() + wrong_target["artifacts"][0]["url"] = "https://example.test/wrong" + plan = provenance.plan_release( + wrong_target, + tag_exists=False, + tagged=None, + remote_release={"exists": False, "assets": []}, + artifact_base_url=STAGING, + repository=REPOSITORY, + ) + self.assertTrue(any("target release URL mismatch" in error for error in plan.errors)) + no_manifest = provenance.plan_release( + manifest(), + tag_exists=True, + tagged=None, + remote_release={"id": 1, "assets": "invalid"}, + artifact_base_url=STAGING, + repository=REPOSITORY, + ) + self.assertTrue(any("missing its manifest" in error for error in no_manifest.errors)) + self.assertTrue(any("assets must be a list" in error for error in no_manifest.errors)) + + def test_plan_cli_writes_idempotent_mode_and_staged_cli_passes(self): + payload = b"release" + working_value = manifest(payload=payload) with tempfile.TemporaryDirectory() as directory: root = Path(directory) working = root / "working.json" tagged = root / "tagged.json" remote = root / "remote.json" - working.write_text(json.dumps(manifest("3.5.2")), encoding="utf-8") - tagged.write_text(json.dumps(manifest("3.5.1")), encoding="utf-8") - remote.write_text(json.dumps({"assets": []}), encoding="utf-8") - output = io.StringIO() - with contextlib.redirect_stdout(output): - result = provenance.main( - [ - "--working-manifest", - str(working), - "--tag-manifest", - str(tagged), - "--remote-release", - str(remote), - ] + output = root / "github-output.txt" + working.write_text(json.dumps(working_value), encoding="utf-8") + tagged.write_text(json.dumps(working_value), encoding="utf-8") + remote.write_text(json.dumps(remote_release(working_value)), encoding="utf-8") + args = [ + "plan", + "--working-manifest", str(working), + "--tag-exists", + "--tag-manifest", str(tagged), + "--remote-release", str(remote), + "--artifact-base-url", STAGING, + "--repository", REPOSITORY, + "--github-output", str(output), + ] + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(provenance.main(args), 0) + self.assertEqual(output.read_text(encoding="utf-8"), "mode=idempotent\n") + artifact = root / "simplicio-macos-arm64" + artifact.write_bytes(payload) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual( + provenance.main(["verify-staged", "--working-manifest", str(working), "--staging-dir", str(root)]), + 0, ) - self.assertEqual(result, 1) - self.assertIn("3.5.1", output.getvalue()) - def test_cli_passes_equal_inputs_and_fails_non_object_json(self): + def test_cli_failures_are_fail_closed(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) working = root / "working.json" tagged = root / "tagged.json" remote = root / "remote.json" - working.write_text(json.dumps(manifest()), encoding="utf-8") - tagged.write_text(json.dumps(manifest()), encoding="utf-8") - remote.write_text(json.dumps({"assets": []}), encoding="utf-8") - output = io.StringIO() - args = ["--working-manifest", str(working), "--tag-manifest", str(tagged)] - with contextlib.redirect_stdout(output): - self.assertEqual(provenance.main(args), 0) - self.assertIn("release-provenance: PASS", output.getvalue()) working.write_text("[]", encoding="utf-8") - output = io.StringIO() - with contextlib.redirect_stdout(output): - self.assertEqual(provenance.main(args), 1) - self.assertIn("expected a JSON object", output.getvalue()) + tagged.write_text(json.dumps(manifest()), encoding="utf-8") + remote.write_text(json.dumps({"exists": False, "assets": []}), encoding="utf-8") + plan_args = [ + "plan", "--working-manifest", str(working), "--tag-manifest", str(tagged), + "--remote-release", str(remote), "--artifact-base-url", STAGING, + "--repository", REPOSITORY, + ] + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(provenance.main(plan_args), 1) + working.write_text(json.dumps(manifest(payload=b"expected")), encoding="utf-8") + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual( + provenance.main(["verify-staged", "--working-manifest", str(working), "--staging-dir", str(root)]), + 1, + ) if __name__ == "__main__": From fba3b4d820596cc158c30472f79da7d3f11e4e08 Mon Sep 17 00:00:00 2001 From: "Simplicio, Wesley (ext)" Date: Tue, 14 Jul 2026 08:43:32 -0300 Subject: [PATCH 5/6] fix(release): enforce closed-world workflow Refs #10 --- .github/workflows/release.yml | 106 +++--------- CHANGELOG.md | 3 + QUALITY.md | 6 +- scripts/verify_distribution_consistency.py | 185 +++++++++++++-------- scripts/verify_release_provenance.py | 180 ++++++++++++++++++-- tests/test_distribution_consistency.py | 84 +++++++--- tests/test_release_provenance.py | 89 ++++++++++ 7 files changed, 468 insertions(+), 185 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ec0c107..77df905 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,6 @@ name: publish-release -# Manual-only immutable publisher. New artifacts come from a distinct, -# version-bound staging origin; coherent existing releases are no-op success. +# Closed-world manual publisher. Every executable step is a canonical repository +# command; existing coherent releases are no-op success. "on": workflow_dispatch: inputs: @@ -10,109 +10,51 @@ name: publish-release type: string permissions: - contents: write + contents: read jobs: release: + permissions: + contents: write runs-on: windows-latest steps: - - uses: actions/checkout@v4 + - id: checkout + uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v5 + - id: setup_python + uses: actions/setup-python@v5 with: python-version: "3.13" - - name: Install pinned release verifier dependencies + - id: install run: python -m pip install -r requirements-quality.txt - - name: Read version from manifest - id: version - shell: pwsh - run: echo "version=$((Get-Content simplicio-update-manifest.json | ConvertFrom-Json).version)" >> $env:GITHUB_OUTPUT - - name: Read target tag and release state without mutation - id: state - shell: pwsh + - id: state env: GITHUB_TOKEN: ${{ github.token }} - run: | - $tag = "v${{ steps.version.outputs.version }}" - New-Item -ItemType Directory -Force -Path .release | Out-Null - $matchingTag = git tag --list $tag - if ($matchingTag -eq $tag) { - "tag_exists=true" >> $env:GITHUB_OUTPUT - git show "${tag}:simplicio-update-manifest.json" | Set-Content -Encoding utf8 .release/tag-manifest.json - if ($LASTEXITCODE -ne 0) { throw "Tag $tag has no update manifest." } - } else { - "tag_exists=false" >> $env:GITHUB_OUTPUT - '{}' | Set-Content -Encoding utf8 .release/tag-manifest.json - } - $headers = @{ - Authorization = "Bearer $env:GITHUB_TOKEN" - Accept = "application/vnd.github+json" - "X-GitHub-Api-Version" = "2022-11-28" - } - $uri = "https://api.github.com/repos/$env:GITHUB_REPOSITORY/releases/tags/$tag" - $response = Invoke-WebRequest -Uri $uri -Headers $headers -SkipHttpErrorCheck - if ($response.StatusCode -eq 200) { - $response.Content | Set-Content -Encoding utf8 .release/remote-release.json - } elseif ($response.StatusCode -eq 404) { - '{"exists":false,"assets":[]}' | Set-Content -Encoding utf8 .release/remote-release.json - } else { - throw "Could not verify remote release state: HTTP $($response.StatusCode)." - } - - name: Plan immutable release - id: provenance - shell: pwsh + run: python scripts/verify_release_provenance.py state + - id: provenance env: ARTIFACT_BASE_URL: ${{ inputs.artifact_base_url }} - run: | - $tagArgument = @() - if ("${{ steps.state.outputs.tag_exists }}" -eq "true") { $tagArgument = @("--tag-exists") } - python scripts/verify_release_provenance.py plan ` - --working-manifest simplicio-update-manifest.json ` - --tag-manifest .release/tag-manifest.json ` - --remote-release .release/remote-release.json ` - --artifact-base-url "$env:ARTIFACT_BASE_URL" ` - --repository "$env:GITHUB_REPOSITORY" ` - --github-output "$env:GITHUB_OUTPUT" @tagArgument - - name: Download new artifacts from immutable staging - id: download + TAG_EXISTS: ${{ steps.state.outputs.tag_exists }} + run: python scripts/verify_release_provenance.py plan + - id: download if: steps.provenance.outputs.mode == 'publish' - shell: pwsh env: ARTIFACT_BASE_URL: ${{ inputs.artifact_base_url }} - run: | - $manifest = Get-Content simplicio-update-manifest.json -Raw | ConvertFrom-Json - $base = $env:ARTIFACT_BASE_URL.TrimEnd('/') - New-Item -ItemType Directory -Force -Path dist | Out-Null - foreach ($artifact in $manifest.artifacts) { - $source = "$base/$($artifact.artifact)" - $destination = Join-Path dist $artifact.artifact - Invoke-WebRequest -Uri $source -OutFile $destination -UseBasicParsing - } - - name: Verify staged bytes and signature metadata - id: verify_staged + run: python scripts/verify_release_provenance.py download + - id: verify_staged if: steps.provenance.outputs.mode == 'publish' - run: >- - python scripts/verify_release_provenance.py verify-staged - --working-manifest simplicio-update-manifest.json - --staging-dir dist - - name: Generate verified release metadata - id: metadata + run: python scripts/verify_release_provenance.py verify-staged + - id: metadata if: steps.provenance.outputs.mode == 'publish' - shell: pwsh - run: | - Copy-Item simplicio-update-manifest.json dist/ - Get-ChildItem dist -File | Sort-Object Name | ForEach-Object { - "$(($_ | Get-FileHash -Algorithm SHA256).Hash.ToLower()) *$($_.Name)" - } | Set-Content -Encoding ascii dist/SHA256SUMS - - name: Create release from verified new-tag staging - id: publish + run: python scripts/verify_release_provenance.py metadata + - id: publish if: steps.provenance.outputs.mode == 'publish' uses: softprops/action-gh-release@v2 with: - tag_name: v${{ steps.version.outputs.version }} + tag_name: v${{ steps.state.outputs.version }} target_commitish: ${{ github.sha }} - name: "v${{ steps.version.outputs.version }} — Public Beta" + name: "v${{ steps.state.outputs.version }} — Public Beta" body: | Free public beta. All features remain unlocked during the public-beta phase. diff --git a/CHANGELOG.md b/CHANGELOG.md index ad39c0c..c6b4fbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Restored a safe new-release path through distinct versioned HTTPS staging, with idempotent no-op success for coherent existing releases and structured PyYAML validation of real workflow steps, conditions, ordering, and action inputs. +- Closed the release topology to an exact job/step/action/command/environment + allowlist, moving state, download, and metadata logic into fixed verifier + subcommands so extra write steps and appended force/clobber commands fail. ## [1.6.1] - 2026-07-01 diff --git a/QUALITY.md b/QUALITY.md index fb974a0..17ebb49 100644 --- a/QUALITY.md +++ b/QUALITY.md @@ -65,7 +65,11 @@ signed staging set is separate release work. The strict distribution audit parses workflow YAML using pinned PyYAML and validates the actual trigger, input, `jobs.release.steps`, executable commands, action inputs, conditions, and ordering. Comments and -environment variables cannot satisfy executable-step requirements. +environment variables cannot satisfy executable-step requirements. The topology is closed-world: one +release job, nine unique ordered step IDs, three approved actions, exact environment maps, and six +canonical single-command script invocations. Any extra job/step/key/action/command or appended shell +separator fails the audit. Workflow permissions default to `contents: read`; only that exact release job +elevates to `contents: write`. ## Local parity diff --git a/scripts/verify_distribution_consistency.py b/scripts/verify_distribution_consistency.py index 5117621..04b4087 100644 --- a/scripts/verify_distribution_consistency.py +++ b/scripts/verify_distribution_consistency.py @@ -28,6 +28,23 @@ CURRENT_VERSION_RE = re.compile(r"## Current Version:\s*v([^\s]+)") +class UniqueKeyLoader(yaml.SafeLoader): + """Safe YAML loader that rejects ambiguous duplicate mapping keys.""" + + +def construct_unique_mapping(loader: UniqueKeyLoader, node: yaml.MappingNode, deep: bool = False) -> dict: + mapping = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if key in mapping: + raise yaml.constructor.ConstructorError(None, None, f"duplicate YAML key: {key}", key_node.start_mark) + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +UniqueKeyLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_unique_mapping) + + @dataclass(frozen=True) class Finding: level: str @@ -82,25 +99,16 @@ def iter_install_reference_files(root: Path) -> Iterable[Path]: yield from sorted((root / "READMEs").glob("README*.md")) -def executable_run(step: dict) -> str: - run = step.get("run") - if not isinstance(run, str): - return "" - return "\n".join( - line.strip().lower() - for line in run.splitlines() - if line.strip() and not line.lstrip().startswith("#") - ) - - def release_workflow_errors(workflow: str) -> list[str]: try: - document = yaml.safe_load(workflow) + document = yaml.load(workflow, Loader=UniqueKeyLoader) except yaml.YAMLError as exc: return [f"release workflow YAML is invalid: {exc}"] if not isinstance(document, dict): return ["release workflow must be a YAML mapping"] errors: list[str] = [] + if set(document) != {"name", "on", "permissions", "jobs"}: + errors.append("release workflow has unexpected or missing top-level keys") triggers = document.get("on") if not isinstance(triggers, dict) or set(triggers) != {"workflow_dispatch"}: errors.append("release workflow must have only workflow_dispatch trigger") @@ -111,82 +119,119 @@ def release_workflow_errors(workflow: str) -> list[str]: base_input = inputs.get("artifact_base_url") if isinstance(inputs, dict) else None if not isinstance(base_input, dict) or base_input.get("required") is not True or base_input.get("type") != "string": errors.append("workflow_dispatch must require string input artifact_base_url") - + if document.get("permissions") != {"contents": "read"}: + errors.append("workflow permissions must default to contents: read") jobs = document.get("jobs") - release = jobs.get("release") if isinstance(jobs, dict) else None + if not isinstance(jobs, dict) or set(jobs) != {"release"}: + errors.append("release workflow must contain exactly one release job") + return errors + release = jobs.get("release") + if not isinstance(release, dict) or set(release) != {"permissions", "runs-on", "steps"}: + errors.append("release job has unexpected or missing keys") + return errors + if release.get("permissions") != {"contents": "write"}: + errors.append("only the release job may elevate contents: write") + if release.get("runs-on") != "windows-latest": + errors.append("release job must run on windows-latest") steps = release.get("steps") if isinstance(release, dict) else None - if not isinstance(steps, list) or not steps or not all(isinstance(step, dict) for step in steps): - errors.append("jobs.release.steps must be a non-empty list of mappings") + expected_ids = ( + "checkout", + "setup_python", + "install", + "state", + "provenance", + "download", + "verify_staged", + "metadata", + "publish", + ) + if not isinstance(steps, list) or not all(isinstance(step, dict) for step in steps): + errors.append("jobs.release.steps must be a list of mappings") return errors - indexed: dict[str, tuple[int, dict]] = {} - for index, step in enumerate(steps): - step_id = step.get("id") - if isinstance(step_id, str): - if step_id in indexed: - errors.append(f"duplicate release step id: {step_id}") - indexed[step_id] = (index, step) - required_ids = ("state", "provenance", "download", "verify_staged", "metadata", "publish") - missing_ids = [step_id for step_id in required_ids if step_id not in indexed] - if missing_ids: - errors.append("missing release step ids: " + ", ".join(missing_ids)) + actual_ids = tuple(step.get("id") for step in steps) + if actual_ids != expected_ids or len(set(actual_ids)) != len(actual_ids): + errors.append("release step IDs must match the exact ordered closed-world allowlist") return errors - positions = [indexed[step_id][0] for step_id in required_ids] - if positions != sorted(positions) or len(set(positions)) != len(positions): - errors.append("release steps must order state, provenance, download, verify_staged, metadata, publish") - - install_runs = [executable_run(step) for step in steps] - if not any("pip install -r requirements-quality.txt" in run for run in install_runs): - errors.append("release job must install pinned requirements-quality.txt") - - state_run = executable_run(indexed["state"][1]) - for command in ("git tag --list", "git show \"${tag}:simplicio-update-manifest.json\"", "remote-release.json"): - if command not in state_run: - errors.append(f"state step lacks executable command: {command}") - provenance_run = executable_run(indexed["provenance"][1]) - for argument in ( - "scripts/verify_release_provenance.py plan", - "--working-manifest", - "--tag-manifest", - "--remote-release", - "--artifact-base-url", - "--github-output", - "--tag-exists", - ): - if argument not in provenance_run: - errors.append(f"provenance step lacks executable argument: {argument}") - + indexed = {step["id"]: step for step in steps} + expected_keys = { + "checkout": {"id", "uses", "with"}, + "setup_python": {"id", "uses", "with"}, + "install": {"id", "run"}, + "state": {"id", "env", "run"}, + "provenance": {"id", "env", "run"}, + "download": {"id", "if", "env", "run"}, + "verify_staged": {"id", "if", "run"}, + "metadata": {"id", "if", "run"}, + "publish": {"id", "if", "uses", "with"}, + } + expected_uses = { + "checkout": "actions/checkout@v4", + "setup_python": "actions/setup-python@v5", + "publish": "softprops/action-gh-release@v2", + } + expected_runs = { + "install": "python -m pip install -r requirements-quality.txt", + "state": "python scripts/verify_release_provenance.py state", + "provenance": "python scripts/verify_release_provenance.py plan", + "download": "python scripts/verify_release_provenance.py download", + "verify_staged": "python scripts/verify_release_provenance.py verify-staged", + "metadata": "python scripts/verify_release_provenance.py metadata", + } + expected_env = { + "state": {"GITHUB_TOKEN": "${{ github.token }}"}, + "provenance": { + "ARTIFACT_BASE_URL": "${{ inputs.artifact_base_url }}", + "TAG_EXISTS": "${{ steps.state.outputs.tag_exists }}", + }, + "download": {"ARTIFACT_BASE_URL": "${{ inputs.artifact_base_url }}"}, + } + for step_id in expected_ids: + step = indexed[step_id] + if set(step) != expected_keys[step_id]: + errors.append(f"{step_id} step has unexpected or missing keys") + if step_id in expected_uses and step.get("uses") != expected_uses[step_id]: + errors.append(f"{step_id} step uses an unapproved action") + if step_id in expected_runs and step.get("run") != expected_runs[step_id]: + errors.append(f"{step_id} step must equal its canonical single command") + if step_id in expected_env and step.get("env") != expected_env[step_id]: + errors.append(f"{step_id} step env must match the exact allowlist") publish_condition = "steps.provenance.outputs.mode == 'publish'" guarded_ids = ("download", "verify_staged", "metadata", "publish") for step_id in guarded_ids: - if indexed[step_id][1].get("if") != publish_condition: + if indexed[step_id].get("if") != publish_condition: errors.append(f"{step_id} step must be guarded by publish mode") - download_run = executable_run(indexed["download"][1]) - if "$env:artifact_base_url" not in download_run or "invoke-webrequest -uri $source" not in download_run: - errors.append("download step must source artifacts from artifact_base_url") - if "$artifact.url" in download_run or "releases/download" in download_run: - errors.append("download step must not source a new binary from its target release URL") - if "copy-item simplicio " in download_run or "copy-item simplicio.exe " in download_run: - errors.append("download step must not stage checked-in wrappers") - - staged_run = executable_run(indexed["verify_staged"][1]) - if "scripts/verify_release_provenance.py verify-staged" not in staged_run or "--staging-dir" not in staged_run: - errors.append("verify_staged step must execute the reusable digest verifier") - publish = indexed["publish"][1] - if publish.get("uses") != "softprops/action-gh-release@v2": - errors.append("publish step must use softprops/action-gh-release@v2") + if indexed["checkout"].get("with") != {"fetch-depth": 0}: + errors.append("checkout step inputs must match the exact allowlist") + if indexed["setup_python"].get("with") != {"python-version": "3.13"}: + errors.append("setup_python step inputs must match the exact allowlist") + publish = indexed["publish"] publish_with = publish.get("with") if not isinstance(publish_with, dict): errors.append("publish step must define structured with inputs") else: + expected_publish_keys = { + "tag_name", + "target_commitish", + "name", + "body", + "prerelease", + "make_latest", + "fail_on_unmatched_files", + "overwrite_files", + "files", + } + if set(publish_with) != expected_publish_keys: + errors.append("publish inputs must match the exact closed-world allowlist") + if publish_with.get("tag_name") != "v${{ steps.state.outputs.version }}": + errors.append("publish tag_name must come from verified state") + if publish_with.get("target_commitish") != "${{ github.sha }}": + errors.append("publish target_commitish must be the dispatched commit") if publish_with.get("overwrite_files") is not False: errors.append("publish step must set overwrite_files: false") if publish_with.get("fail_on_unmatched_files") is not True: errors.append("publish step must set fail_on_unmatched_files: true") if publish_with.get("files") != "dist/*": errors.append("publish step must upload only dist/*") - release_actions = [step for step in steps if step.get("uses") == "softprops/action-gh-release@v2"] - if len(release_actions) != 1: - errors.append("release job must contain exactly one publish action") return errors diff --git a/scripts/verify_release_provenance.py b/scripts/verify_release_provenance.py index 059cc80..df5edcc 100644 --- a/scripts/verify_release_provenance.py +++ b/scripts/verify_release_provenance.py @@ -6,11 +6,16 @@ import argparse import hashlib import json +import os +import shutil +import subprocess import sys from dataclasses import dataclass from pathlib import Path from typing import Sequence +from urllib.error import HTTPError from urllib.parse import urlparse +from urllib.request import Request, urlopen PROVENANCE_FIELDS = ("artifact", "url", "sha256", "signature") @@ -28,6 +33,63 @@ def load_json(path: Path) -> dict: return value +def append_outputs(path: Path | None, values: dict[str, str]) -> None: + if path: + with path.open("a", encoding="utf-8") as stream: + for key, value in values.items(): + stream.write(f"{key}={value}\n") + + +def collect_release_state( + working_manifest: Path, + repository: str, + token: str, + state_dir: Path, + github_output: Path | None, + *, + runner=subprocess.run, + opener=urlopen, +) -> bool: + if not repository or not token: + raise ValueError("GITHUB_REPOSITORY and GITHUB_TOKEN are required") + working = load_json(working_manifest) + version, _ = provenance_snapshot(working) + tag = f"v{version}" + state_dir.mkdir(parents=True, exist_ok=True) + tag_result = runner(["git", "tag", "--list", tag], check=True, capture_output=True, text=True) + tag_exists = tag_result.stdout.strip() == tag + tag_manifest_path = state_dir / "tag-manifest.json" + if tag_exists: + tagged = runner( + ["git", "show", f"{tag}:simplicio-update-manifest.json"], + check=True, + capture_output=True, + text=True, + ).stdout + tag_manifest_path.write_text(tagged, encoding="utf-8") + else: + tag_manifest_path.write_text("{}\n", encoding="utf-8") + request = Request( + f"https://api.github.com/repos/{repository}/releases/tags/{tag}", + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with opener(request) as response: + remote = json.loads(response.read().decode("utf-8")) + except HTTPError as exc: + if exc.code != 404: + raise + exc.close() + remote = {"exists": False, "assets": []} + (state_dir / "remote-release.json").write_text(json.dumps(remote, indent=2) + "\n", encoding="utf-8") + append_outputs(github_output, {"version": version, "tag_exists": str(tag_exists).lower()}) + return tag_exists + + def provenance_snapshot(manifest: dict) -> tuple[str, tuple[tuple[str, str, str, str], ...]]: version = str(manifest.get("version") or "").strip() if not version: @@ -170,10 +232,57 @@ def verify_staged_files(working: dict, staging_dir: Path) -> list[str]: return errors +def download_staged_files( + working: dict, + artifact_base_url: str, + repository: str, + staging_dir: Path, + *, + opener=urlopen, +) -> None: + version, artifacts = provenance_snapshot(working) + errors = validate_staging_base_url(artifact_base_url, version, repository) + errors.extend(target_url_errors(working, repository)) + if errors: + raise ValueError("; ".join(errors)) + staging_dir.mkdir(parents=True, exist_ok=True) + base = artifact_base_url.rstrip("/") + for name, _, _, _ in artifacts: + with opener(Request(f"{base}/{name}")) as response: + (staging_dir / name).write_bytes(response.read()) + + +def generate_release_metadata(working_manifest: Path, staging_dir: Path) -> None: + working = load_json(working_manifest) + errors = verify_staged_files(working, staging_dir) + if errors: + raise ValueError("; ".join(errors)) + shutil.copy2(working_manifest, staging_dir / "simplicio-update-manifest.json") + lines = [] + for path in sorted(staging_dir.iterdir(), key=lambda item: item.name): + if path.is_file() and path.name != "SHA256SUMS": + lines.append(f"{hashlib.sha256(path.read_bytes()).hexdigest()} *{path.name}") + (staging_dir / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii") + + def write_output(path: Path | None, mode: str) -> None: - if path: - with path.open("a", encoding="utf-8") as stream: - stream.write(f"mode={mode}\n") + append_outputs(path, {"mode": mode}) + + +def state_command(args: argparse.Namespace) -> int: + try: + tag_exists = collect_release_state( + args.working_manifest, + args.repository, + args.github_token, + args.state_dir, + args.github_output, + ) + except (OSError, ValueError, json.JSONDecodeError, subprocess.CalledProcessError, HTTPError) as exc: + print(f"[ERROR] release state collection failed: {exc}") + return 1 + print(f"release-provenance: state PASS tag_exists={str(tag_exists).lower()}") + return 0 def plan_command(args: argparse.Namespace) -> int: @@ -213,22 +322,69 @@ def staged_command(args: argparse.Namespace) -> int: return 0 +def download_command(args: argparse.Namespace) -> int: + try: + download_staged_files( + load_json(args.working_manifest), + args.artifact_base_url, + args.repository, + args.staging_dir, + ) + except (OSError, ValueError, json.JSONDecodeError, HTTPError) as exc: + print(f"[ERROR] staged artifact download failed: {exc}") + return 1 + print("release-provenance: staging download PASS") + return 0 + + +def metadata_command(args: argparse.Namespace) -> int: + try: + generate_release_metadata(args.working_manifest, args.staging_dir) + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"[ERROR] release metadata generation failed: {exc}") + return 1 + print("release-provenance: metadata PASS") + return 0 + + +def environment_path(name: str) -> Path | None: + value = os.environ.get(name) + return Path(value) if value else None + + def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest="command", required=True) + state = subparsers.add_parser("state") + state.add_argument("--working-manifest", type=Path, default=Path("simplicio-update-manifest.json")) + state.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", "")) + state.add_argument("--github-token", default=os.environ.get("GITHUB_TOKEN", "")) + state.add_argument("--state-dir", type=Path, default=Path(".release")) + state.add_argument("--github-output", type=Path, default=environment_path("GITHUB_OUTPUT")) + state.set_defaults(handler=state_command) plan = subparsers.add_parser("plan") - plan.add_argument("--working-manifest", type=Path, required=True) - plan.add_argument("--tag-exists", action="store_true") - plan.add_argument("--tag-manifest", type=Path, required=True) - plan.add_argument("--remote-release", type=Path, required=True) - plan.add_argument("--artifact-base-url", required=True) - plan.add_argument("--repository", required=True) - plan.add_argument("--github-output", type=Path) + plan.add_argument("--working-manifest", type=Path, default=Path("simplicio-update-manifest.json")) + plan.add_argument("--tag-exists", action="store_true", default=os.environ.get("TAG_EXISTS", "").lower() == "true") + plan.add_argument("--tag-manifest", type=Path, default=Path(".release/tag-manifest.json")) + plan.add_argument("--remote-release", type=Path, default=Path(".release/remote-release.json")) + plan.add_argument("--artifact-base-url", default=os.environ.get("ARTIFACT_BASE_URL", "")) + plan.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", "")) + plan.add_argument("--github-output", type=Path, default=environment_path("GITHUB_OUTPUT")) plan.set_defaults(handler=plan_command) + download = subparsers.add_parser("download") + download.add_argument("--working-manifest", type=Path, default=Path("simplicio-update-manifest.json")) + download.add_argument("--artifact-base-url", default=os.environ.get("ARTIFACT_BASE_URL", "")) + download.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", "")) + download.add_argument("--staging-dir", type=Path, default=Path("dist")) + download.set_defaults(handler=download_command) staged = subparsers.add_parser("verify-staged") - staged.add_argument("--working-manifest", type=Path, required=True) - staged.add_argument("--staging-dir", type=Path, required=True) + staged.add_argument("--working-manifest", type=Path, default=Path("simplicio-update-manifest.json")) + staged.add_argument("--staging-dir", type=Path, default=Path("dist")) staged.set_defaults(handler=staged_command) + metadata = subparsers.add_parser("metadata") + metadata.add_argument("--working-manifest", type=Path, default=Path("simplicio-update-manifest.json")) + metadata.add_argument("--staging-dir", type=Path, default=Path("dist")) + metadata.set_defaults(handler=metadata_command) args = parser.parse_args(argv) return args.handler(args) diff --git a/tests/test_distribution_consistency.py b/tests/test_distribution_consistency.py index 384ae48..83c7ff7 100644 --- a/tests/test_distribution_consistency.py +++ b/tests/test_distribution_consistency.py @@ -37,41 +37,61 @@ def write(self) -> None: self.put(relative, "https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.sh\n") self.put( ".github/workflows/release.yml", - '''"on": + '''name: publish-release +"on": workflow_dispatch: inputs: artifact_base_url: required: true type: string +permissions: + contents: read jobs: release: + permissions: + contents: write + runs-on: windows-latest steps: - - run: python -m pip install -r requirements-quality.txt + - id: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: setup_python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + - id: install + run: python -m pip install -r requirements-quality.txt - id: state - run: | - git tag --list v3.5.2 - git show "${tag}:simplicio-update-manifest.json" - echo remote-release.json + env: + GITHUB_TOKEN: ${{ github.token }} + run: python scripts/verify_release_provenance.py state - id: provenance - run: | - python scripts/verify_release_provenance.py plan --working-manifest manifest.json - --tag-manifest tag.json --remote-release remote.json --artifact-base-url "$base" - --github-output "$output" --tag-exists + env: + ARTIFACT_BASE_URL: ${{ inputs.artifact_base_url }} + TAG_EXISTS: ${{ steps.state.outputs.tag_exists }} + run: python scripts/verify_release_provenance.py plan - id: download if: steps.provenance.outputs.mode == 'publish' - run: | - $source = "$env:ARTIFACT_BASE_URL/$name" - Invoke-WebRequest -Uri $source -OutFile $destination + env: + ARTIFACT_BASE_URL: ${{ inputs.artifact_base_url }} + run: python scripts/verify_release_provenance.py download - id: verify_staged if: steps.provenance.outputs.mode == 'publish' - run: python scripts/verify_release_provenance.py verify-staged --staging-dir dist + run: python scripts/verify_release_provenance.py verify-staged - id: metadata if: steps.provenance.outputs.mode == 'publish' - run: echo metadata + run: python scripts/verify_release_provenance.py metadata - id: publish if: steps.provenance.outputs.mode == 'publish' uses: softprops/action-gh-release@v2 with: + tag_name: v${{ steps.state.outputs.version }} + target_commitish: ${{ github.sha }} + name: fixture + body: fixture + prerelease: false + make_latest: "true" fail_on_unmatched_files: true overwrite_files: false files: dist/* @@ -201,7 +221,7 @@ def test_regression_release_must_not_stage_repo_wrapper_binary(self): download["run"] += "\nCopy-Item simplicio dist/simplicio-macos-arm64" workflow.write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8") errors = [item.message for item in audit.run_audit(self.root) if item.level == "ERROR"] - self.assertTrue(any("release workflow provenance" in message and "checked-in wrappers" in message for message in errors)) + self.assertTrue(any("release workflow provenance" in message and "canonical single command" in message for message in errors)) def test_regression_release_must_be_manual_only(self): workflow = self.root / ".github/workflows/release.yml" @@ -216,7 +236,7 @@ def test_regression_release_requires_tag_bound_verifier(self): provenance_step = next(step for step in document["jobs"]["release"]["steps"] if step.get("id") == "provenance") provenance_step["run"] = provenance_step["run"].replace("scripts/verify_release_provenance.py", "scripts/untrusted.py") errors = audit.release_workflow_errors(yaml.safe_dump(document, sort_keys=False)) - self.assertTrue(any("provenance step lacks executable" in error for error in errors)) + self.assertTrue(any("canonical single command" in error for error in errors)) def test_regression_release_requires_explicit_no_overwrite(self): workflow = self.root / ".github/workflows/release.yml" @@ -250,14 +270,15 @@ def test_adversarial_comments_and_env_cannot_spoof_executable_workflow(self): ): with self.subTest(workflow=workflow): errors = audit.release_workflow_errors(workflow) - self.assertTrue(any("jobs.release.steps" in error for error in errors)) + self.assertTrue(any("release job" in error for error in errors)) workflow = self.root / ".github/workflows/release.yml" document = yaml.safe_load(workflow.read_text(encoding="utf-8")) provenance_step = next(step for step in document["jobs"]["release"]["steps"] if step.get("id") == "provenance") provenance_step["env"] = {"SPOOF": "scripts/verify_release_provenance.py plan --tag-exists"} provenance_step["run"] = "# python scripts/verify_release_provenance.py plan --working-manifest --tag-manifest --remote-release --artifact-base-url --github-output --tag-exists" errors = audit.release_workflow_errors(yaml.safe_dump(document, sort_keys=False)) - self.assertTrue(any("provenance step lacks executable" in error for error in errors)) + self.assertTrue(any("canonical single command" in error for error in errors)) + self.assertTrue(any("env must match" in error for error in errors)) def test_regression_release_step_order_is_semantic(self): workflow = self.root / ".github/workflows/release.yml" @@ -267,13 +288,36 @@ def test_regression_release_step_order_is_semantic(self): download_index = next(index for index, step in enumerate(steps) if step.get("id") == "download") steps[provenance_index], steps[download_index] = steps[download_index], steps[provenance_index] errors = audit.release_workflow_errors(yaml.safe_dump(document, sort_keys=False)) - self.assertTrue(any("must order" in error for error in errors)) + self.assertTrue(any("closed-world allowlist" in error for error in errors)) + + def test_malicious_extra_step_command_and_env_are_rejected(self): + workflow = self.root / ".github/workflows/release.yml" + clean = yaml.safe_load(workflow.read_text(encoding="utf-8")) + attack = "gh release upload --clobber; git tag -f; git push --force" + + extra = json.loads(json.dumps(clean)) + extra["jobs"]["release"]["steps"].insert(-1, {"id": "attacker", "run": attack}) + errors = audit.release_workflow_errors(yaml.safe_dump(extra, sort_keys=False)) + self.assertTrue(any("closed-world allowlist" in error for error in errors)) + + appended = json.loads(json.dumps(clean)) + state = next(step for step in appended["jobs"]["release"]["steps"] if step.get("id") == "state") + state["run"] += "; " + attack + errors = audit.release_workflow_errors(yaml.safe_dump(appended, sort_keys=False)) + self.assertTrue(any("canonical single command" in error for error in errors)) + + poisoned_env = json.loads(json.dumps(clean)) + provenance_step = next(step for step in poisoned_env["jobs"]["release"]["steps"] if step.get("id") == "provenance") + provenance_step["env"]["POST_PLAN"] = attack + errors = audit.release_workflow_errors(yaml.safe_dump(poisoned_env, sort_keys=False)) + self.assertTrue(any("env must match" in error for error in errors)) def test_structured_workflow_rejects_malformed_step_shapes(self): workflow = self.root / ".github/workflows/release.yml" clean = yaml.safe_load(workflow.read_text(encoding="utf-8")) self.assertTrue(any("YAML is invalid" in error for error in audit.release_workflow_errors("jobs: ["))) + self.assertTrue(any("duplicate YAML key" in error for error in audit.release_workflow_errors("jobs: {}\njobs: {}"))) self.assertTrue(any("YAML mapping" in error for error in audit.release_workflow_errors("[]"))) variants = [] diff --git a/tests/test_release_provenance.py b/tests/test_release_provenance.py index 88a6c6e..7e9e38e 100644 --- a/tests/test_release_provenance.py +++ b/tests/test_release_provenance.py @@ -7,6 +7,7 @@ import tempfile import unittest from pathlib import Path +from urllib.error import HTTPError from scripts import verify_release_provenance as provenance @@ -14,6 +15,20 @@ STAGING = "https://artifacts.example/simplicio/v3.5.2" +class FakeResponse: + def __init__(self, value: bytes): + self.value = value + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self) -> bytes: + return self.value + + def manifest(version: str = "3.5.2", *, payload: bytes | None = None) -> dict: name = "simplicio-macos-arm64" sha256 = hashlib.sha256(payload).hexdigest() if payload is not None else "9" * 64 @@ -45,6 +60,58 @@ def remote_release(value: dict, *, digest: str | None = None) -> dict: class ReleaseProvenanceTests(unittest.TestCase): + def test_state_collection_writes_existing_and_absent_tag_receipts(self): + working_value = manifest() + remote_value = remote_release(working_value) + + class Result: + def __init__(self, stdout: str): + self.stdout = stdout + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + working = root / "working.json" + working.write_text(json.dumps(working_value), encoding="utf-8") + output = root / "output.txt" + + def existing_runner(command, **_kwargs): + return Result("v3.5.2\n" if command[1] == "tag" else json.dumps(working_value)) + + exists = provenance.collect_release_state( + working, + REPOSITORY, + "token", + root / "existing", + output, + runner=existing_runner, + opener=lambda _request: FakeResponse(json.dumps(remote_value).encode()), + ) + self.assertTrue(exists) + self.assertEqual(json.loads((root / "existing/tag-manifest.json").read_text()), working_value) + self.assertIn("tag_exists=true", output.read_text(encoding="utf-8")) + + def absent_runner(_command, **_kwargs): + return Result("") + + def missing_release(request): + raise HTTPError(request.full_url, 404, "missing", {}, None) + + exists = provenance.collect_release_state( + working, + REPOSITORY, + "token", + root / "absent", + None, + runner=absent_runner, + opener=missing_release, + ) + self.assertFalse(exists) + self.assertEqual(json.loads((root / "absent/remote-release.json").read_text()), {"exists": False, "assets": []}) + + def test_state_collection_requires_repository_and_token(self): + with self.assertRaisesRegex(ValueError, "required"): + provenance.collect_release_state(Path("missing"), "", "", Path("state"), None) + def test_existing_coherent_tag_is_idempotent_no_publish(self): working = manifest() plan = provenance.plan_release( @@ -153,6 +220,28 @@ def test_staged_artifact_must_exist_and_match_digest(self): artifact.write_bytes(payload) self.assertEqual(provenance.verify_staged_files(working, root), []) + def test_download_and_metadata_use_verified_staging(self): + payload = b"immutable staged bytes" + working_value = manifest(payload=payload) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + working = root / "manifest.json" + working.write_text(json.dumps(working_value), encoding="utf-8") + dist = root / "dist" + provenance.download_staged_files( + working_value, + STAGING, + REPOSITORY, + dist, + opener=lambda _request: FakeResponse(payload), + ) + self.assertEqual((dist / "simplicio-macos-arm64").read_bytes(), payload) + provenance.generate_release_metadata(working, dist) + self.assertTrue((dist / "simplicio-update-manifest.json").is_file()) + sums = (dist / "SHA256SUMS").read_text(encoding="ascii") + self.assertIn("simplicio-macos-arm64", sums) + self.assertIn("simplicio-update-manifest.json", sums) + def test_invalid_manifest_shapes_and_signature_are_rejected(self): for broken, message in ( ({"artifacts": []}, "version is missing"), From a188cb6ac8efdbfad8a88fabb4aafede3a967484 Mon Sep 17 00:00:00 2001 From: "Simplicio, Wesley (ext)" Date: Tue, 14 Jul 2026 08:57:21 -0300 Subject: [PATCH 6/6] fix(release): enforce exact publish set Refs #10 --- CHANGELOG.md | 3 + QUALITY.md | 8 ++- scripts/verify_distribution_consistency.py | 43 ++++++------- scripts/verify_release_provenance.py | 58 +++++++++++++++--- tests/test_distribution_consistency.py | 31 +++++++++- tests/test_release_provenance.py | 70 +++++++++++++++++++++- 6 files changed, 172 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6b4fbd..87bb42c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Closed the release topology to an exact job/step/action/command/environment allowlist, moving state, download, and metadata logic into fixed verifier subcommands so extra write steps and appended force/clobber commands fail. +- Restricted staging and publication to exact manifest-derived file sets, with + stale/extra/directory/symlink rejection, allowlisted checksum generation, and + full canonical comparison of every publish-action input. ## [1.6.1] - 2026-07-01 diff --git a/QUALITY.md b/QUALITY.md index 17ebb49..5d73a06 100644 --- a/QUALITY.md +++ b/QUALITY.md @@ -57,7 +57,10 @@ reusable Python planner chooses exactly one mode: every remote artifact has the declared SHA256 digest. Download, metadata, and publish steps are skipped. - `publish`: neither tag nor release exists and the staging URL is safe. Artifacts are downloaded from staging, never from the not-yet-existing target release, and their bytes plus signature metadata are - verified before release creation with `overwrite_files: false`. + verified before release creation with `overwrite_files: false`. The destination must start empty; + before metadata its entries must equal the manifest artifact names exactly, and after metadata the + only additions allowed are `simplicio-update-manifest.json` and `SHA256SUMS`. Directories, symlinks, + stale files, and unmanifested files fail closed. Checksums iterate that allowlist rather than a glob. The existing `v3.5.2` tag contains a `3.5.1` manifest, so the manual workflow is intentionally blocked for that tag and must not move it or replace its assets. Preparing a fresh version, immutable tag, and @@ -69,7 +72,8 @@ environment variables cannot satisfy executable-step requirements. The topology release job, nine unique ordered step IDs, three approved actions, exact environment maps, and six canonical single-command script invocations. Any extra job/step/key/action/command or appended shell separator fails the audit. Workflow permissions default to `contents: read`; only that exact release job -elevates to `contents: write`. +elevates to `contents: write`. The publish action's complete `with` mapping is canonical; changing its +tag/name/body/prerelease/latest/overwrite/file inputs or adding another input fails the audit. ## Local parity diff --git a/scripts/verify_distribution_consistency.py b/scripts/verify_distribution_consistency.py index 04b4087..46692d7 100644 --- a/scripts/verify_distribution_consistency.py +++ b/scripts/verify_distribution_consistency.py @@ -26,6 +26,23 @@ BETA_NO_END_RE = re.compile(r"public beta with no end date", re.IGNORECASE) ECOSYSTEM_VERSION_RE = re.compile(r"## Versão atual\s+([^\n]+)", re.MULTILINE) CURRENT_VERSION_RE = re.compile(r"## Current Version:\s*v([^\s]+)") +PUBLISH_BODY = ( + "Free public beta. All features remain unlocked during the public-beta phase.\n\n" + "Windows: `irm https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.ps1 | iex`\n" + "macOS/Linux: `curl -fsSL https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.sh | sh`\n\n" + "Signed update manifest included (`simplicio update check`).\n" +) +CANONICAL_PUBLISH_WITH = { + "tag_name": "v${{ steps.state.outputs.version }}", + "target_commitish": "${{ github.sha }}", + "name": "v${{ steps.state.outputs.version }} — Public Beta", + "body": PUBLISH_BODY, + "prerelease": False, + "make_latest": "true", + "fail_on_unmatched_files": True, + "overwrite_files": False, + "files": "dist/*", +} class UniqueKeyLoader(yaml.SafeLoader): @@ -208,30 +225,8 @@ def release_workflow_errors(workflow: str) -> list[str]: publish_with = publish.get("with") if not isinstance(publish_with, dict): errors.append("publish step must define structured with inputs") - else: - expected_publish_keys = { - "tag_name", - "target_commitish", - "name", - "body", - "prerelease", - "make_latest", - "fail_on_unmatched_files", - "overwrite_files", - "files", - } - if set(publish_with) != expected_publish_keys: - errors.append("publish inputs must match the exact closed-world allowlist") - if publish_with.get("tag_name") != "v${{ steps.state.outputs.version }}": - errors.append("publish tag_name must come from verified state") - if publish_with.get("target_commitish") != "${{ github.sha }}": - errors.append("publish target_commitish must be the dispatched commit") - if publish_with.get("overwrite_files") is not False: - errors.append("publish step must set overwrite_files: false") - if publish_with.get("fail_on_unmatched_files") is not True: - errors.append("publish step must set fail_on_unmatched_files: true") - if publish_with.get("files") != "dist/*": - errors.append("publish step must upload only dist/*") + elif publish_with != CANONICAL_PUBLISH_WITH: + errors.append("publish with mapping must equal the complete canonical mapping") return errors diff --git a/scripts/verify_release_provenance.py b/scripts/verify_release_provenance.py index df5edcc..1f2d8cc 100644 --- a/scripts/verify_release_provenance.py +++ b/scripts/verify_release_provenance.py @@ -218,20 +218,47 @@ def plan_release( return ReleasePlan("blocked" if errors else "publish", tuple(errors)) +def exact_directory_errors(directory: Path, expected_names: set[str], label: str) -> list[str]: + errors: list[str] = [] + if directory.is_symlink() or not directory.is_dir(): + return [f"{label} directory is missing or not a real directory"] + entries = list(directory.iterdir()) + actual_names = {entry.name for entry in entries} + missing = sorted(expected_names - actual_names) + extra = sorted(actual_names - expected_names) + if missing: + errors.append(f"{label} set is missing: {', '.join(missing)}") + if extra: + errors.append(f"{label} set has unmanifested entries: {', '.join(extra)}") + for entry in entries: + if entry.name in expected_names and (entry.is_symlink() or not entry.is_file()): + errors.append(f"{label} entry is not a regular file: {entry.name}") + return errors + + def verify_staged_files(working: dict, staging_dir: Path) -> list[str]: _, artifacts = provenance_snapshot(working) - errors: list[str] = [] + expected_names = {record[0] for record in artifacts} + errors = exact_directory_errors(staging_dir, expected_names, "staging") + if errors: + return errors for name, _, expected_sha256, _ in artifacts: path = staging_dir / name - if not path.is_file(): - errors.append(f"staged artifact is missing: {name}") - continue actual = hashlib.sha256(path.read_bytes()).hexdigest() if actual != expected_sha256: errors.append(f"staged artifact digest mismatch for {name}") return errors +def verify_publish_files(working: dict, staging_dir: Path) -> list[str]: + _, artifacts = provenance_snapshot(working) + expected_names = {record[0] for record in artifacts} | { + "simplicio-update-manifest.json", + "SHA256SUMS", + } + return exact_directory_errors(staging_dir, expected_names, "publish") + + def download_staged_files( working: dict, artifact_base_url: str, @@ -245,7 +272,14 @@ def download_staged_files( errors.extend(target_url_errors(working, repository)) if errors: raise ValueError("; ".join(errors)) - staging_dir.mkdir(parents=True, exist_ok=True) + if staging_dir.exists(): + if staging_dir.is_symlink() or not staging_dir.is_dir(): + raise ValueError("staging destination must be a real directory") + stale = sorted(entry.name for entry in staging_dir.iterdir()) + if stale: + raise ValueError("staging destination is not empty: " + ", ".join(stale)) + else: + staging_dir.mkdir(parents=True) base = artifact_base_url.rstrip("/") for name, _, _, _ in artifacts: with opener(Request(f"{base}/{name}")) as response: @@ -254,15 +288,21 @@ def download_staged_files( def generate_release_metadata(working_manifest: Path, staging_dir: Path) -> None: working = load_json(working_manifest) + _, artifacts = provenance_snapshot(working) errors = verify_staged_files(working, staging_dir) if errors: raise ValueError("; ".join(errors)) + lines = [ + f"{hashlib.sha256((staging_dir / name).read_bytes()).hexdigest()} *{name}" + for name, _, _, _ in artifacts + ] shutil.copy2(working_manifest, staging_dir / "simplicio-update-manifest.json") - lines = [] - for path in sorted(staging_dir.iterdir(), key=lambda item: item.name): - if path.is_file() and path.name != "SHA256SUMS": - lines.append(f"{hashlib.sha256(path.read_bytes()).hexdigest()} *{path.name}") + manifest_path = staging_dir / "simplicio-update-manifest.json" + lines.append(f"{hashlib.sha256(manifest_path.read_bytes()).hexdigest()} *{manifest_path.name}") (staging_dir / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii") + final_errors = verify_publish_files(working, staging_dir) + if final_errors: + raise ValueError("; ".join(final_errors)) def write_output(path: Path | None, mode: str) -> None: diff --git a/tests/test_distribution_consistency.py b/tests/test_distribution_consistency.py index 83c7ff7..c98a1d7 100644 --- a/tests/test_distribution_consistency.py +++ b/tests/test_distribution_consistency.py @@ -88,8 +88,14 @@ def write(self) -> None: with: tag_name: v${{ steps.state.outputs.version }} target_commitish: ${{ github.sha }} - name: fixture - body: fixture + name: "v${{ steps.state.outputs.version }} — Public Beta" + body: | + Free public beta. All features remain unlocked during the public-beta phase. + + Windows: `irm https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.ps1 | iex` + macOS/Linux: `curl -fsSL https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.sh | sh` + + Signed update manifest included (`simplicio update check`). prerelease: false make_latest: "true" fail_on_unmatched_files: true @@ -250,7 +256,26 @@ def test_regression_release_requires_explicit_no_overwrite(self): publish["with"]["overwrite_files"] = value with self.subTest(value=value): errors = audit.release_workflow_errors(yaml.safe_dump(document, sort_keys=False)) - self.assertTrue(any("overwrite_files" in error for error in errors)) + self.assertTrue(any("complete canonical mapping" in error for error in errors)) + + def test_publish_mapping_rejects_prerelease_name_body_and_latest_mutations(self): + workflow = self.root / ".github/workflows/release.yml" + clean = yaml.safe_load(workflow.read_text(encoding="utf-8")) + for key, value in ( + ("prerelease", True), + ("name", "attacker-controlled release"), + ("body", "mutated body"), + ("make_latest", "false"), + ): + document = json.loads(json.dumps(clean)) + publish = next(step for step in document["jobs"]["release"]["steps"] if step.get("id") == "publish") + publish["with"][key] = value + with self.subTest(key=key): + errors = audit.release_workflow_errors(yaml.safe_dump(document, sort_keys=False)) + self.assertEqual( + [error for error in errors if "publish with mapping" in error], + ["publish with mapping must equal the complete canonical mapping"], + ) def test_adversarial_comments_and_env_cannot_spoof_executable_workflow(self): for workflow in ( diff --git a/tests/test_release_provenance.py b/tests/test_release_provenance.py index 7e9e38e..cf4a0b3 100644 --- a/tests/test_release_provenance.py +++ b/tests/test_release_provenance.py @@ -7,6 +7,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock from urllib.error import HTTPError from scripts import verify_release_provenance as provenance @@ -213,7 +214,7 @@ def test_staged_artifact_must_exist_and_match_digest(self): working = manifest(payload=payload) with tempfile.TemporaryDirectory() as directory: root = Path(directory) - self.assertEqual(provenance.verify_staged_files(working, root), ["staged artifact is missing: simplicio-macos-arm64"]) + self.assertEqual(provenance.verify_staged_files(working, root), ["staging set is missing: simplicio-macos-arm64"]) artifact = root / "simplicio-macos-arm64" artifact.write_bytes(b"wrong") self.assertEqual(provenance.verify_staged_files(working, root), ["staged artifact digest mismatch for simplicio-macos-arm64"]) @@ -238,10 +239,71 @@ def test_download_and_metadata_use_verified_staging(self): self.assertEqual((dist / "simplicio-macos-arm64").read_bytes(), payload) provenance.generate_release_metadata(working, dist) self.assertTrue((dist / "simplicio-update-manifest.json").is_file()) + self.assertEqual( + {path.name for path in dist.iterdir()}, + {"simplicio-macos-arm64", "simplicio-update-manifest.json", "SHA256SUMS"}, + ) + self.assertEqual(provenance.verify_publish_files(working_value, dist), []) sums = (dist / "SHA256SUMS").read_text(encoding="ascii") self.assertIn("simplicio-macos-arm64", sums) self.assertIn("simplicio-update-manifest.json", sums) + def test_unmanifested_stale_directory_and_symlink_entries_are_rejected(self): + payload = b"release" + working = manifest(payload=payload) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + dist = root / "dist" + dist.mkdir() + (dist / "simplicio-macos-arm64").write_bytes(payload) + (dist / "unmanifested.exe").write_bytes(b"attack") + self.assertTrue(any("unmanifested.exe" in error for error in provenance.verify_staged_files(working, dist))) + with self.assertRaisesRegex(ValueError, "not empty"): + provenance.download_staged_files( + working, + STAGING, + REPOSITORY, + dist, + opener=lambda _request: FakeResponse(payload), + ) + + (dist / "unmanifested.exe").unlink() + artifact = dist / "simplicio-macos-arm64" + original_is_symlink = Path.is_symlink + + def simulated_symlink(path): + return path == artifact or original_is_symlink(path) + + with mock.patch.object(Path, "is_symlink", simulated_symlink): + self.assertEqual( + provenance.verify_staged_files(working, dist), + ["staging entry is not a regular file: simplicio-macos-arm64"], + ) + + artifact.unlink() + artifact.mkdir() + self.assertEqual( + provenance.verify_staged_files(working, dist), + ["staging entry is not a regular file: simplicio-macos-arm64"], + ) + + def test_final_publish_set_rejects_any_extra_file(self): + payload = b"release" + working_value = manifest(payload=payload) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest_path = root / "manifest.json" + manifest_path.write_text(json.dumps(working_value), encoding="utf-8") + dist = root / "dist" + dist.mkdir() + (dist / "simplicio-macos-arm64").write_bytes(payload) + provenance.generate_release_metadata(manifest_path, dist) + (dist / "stale.bin").write_bytes(b"stale") + self.assertEqual( + provenance.verify_publish_files(working_value, dist), + ["publish set has unmanifested entries: stale.bin"], + ) + def test_invalid_manifest_shapes_and_signature_are_rejected(self): for broken, message in ( ({"artifacts": []}, "version is missing"), @@ -319,11 +381,13 @@ def test_plan_cli_writes_idempotent_mode_and_staged_cli_passes(self): with contextlib.redirect_stdout(io.StringIO()): self.assertEqual(provenance.main(args), 0) self.assertEqual(output.read_text(encoding="utf-8"), "mode=idempotent\n") - artifact = root / "simplicio-macos-arm64" + dist = root / "dist" + dist.mkdir() + artifact = dist / "simplicio-macos-arm64" artifact.write_bytes(payload) with contextlib.redirect_stdout(io.StringIO()): self.assertEqual( - provenance.main(["verify-staged", "--working-manifest", str(working), "--staging-dir", str(root)]), + provenance.main(["verify-staged", "--working-manifest", str(working), "--staging-dir", str(dist)]), 0, )