From 37484ba502b2461be3e24cab9b4981874a51ddcd Mon Sep 17 00:00:00 2001 From: Ajay Surya Senthilrajan Date: Mon, 20 Jul 2026 17:23:51 +0530 Subject: [PATCH] release: prepare v0.1.0 launch --- .github/ISSUE_TEMPLATE/bug.yml | 75 +++++++++ .github/ISSUE_TEMPLATE/config.yml | 11 ++ .github/ISSUE_TEMPLATE/feature.yml | 59 +++++++ .github/dependabot.yml | 23 +++ .github/pull_request_template.md | 29 ++++ .github/workflows/ci.yml | 195 ++++++++++++++++++++++ .github/workflows/codeql.yml | 37 +++++ .github/workflows/dependency-review.yml | 20 +++ .github/workflows/release.yml | 207 ++++++++++++++++++++++++ .gitleaks.toml | 11 ++ README.md | 47 +++--- pyproject.toml | 57 ++++++- scripts/generate_spdx_sbom.py | 130 +++++++++++++++ scripts/smoke_test.sh | 2 +- scripts/verify_artifacts.py | 181 +++++++++++++++++++++ scripts/verify_installed_release.py | 81 ++++++++++ scripts/verify_pypi_release.py | 71 ++++++++ scripts/verify_release.py | 66 ++++++++ scripts/verify_test_count.py | 33 ++++ scripts/verify_workflow_pins.py | 32 ++++ 20 files changed, 1344 insertions(+), 23 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/dependency-review.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitleaks.toml create mode 100644 scripts/generate_spdx_sbom.py create mode 100644 scripts/verify_artifacts.py create mode 100644 scripts/verify_installed_release.py create mode 100644 scripts/verify_pypi_release.py create mode 100644 scripts/verify_release.py create mode 100644 scripts/verify_test_count.py create mode 100644 scripts/verify_workflow_pins.py diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 0000000..52446cd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,75 @@ +name: Bug report +description: Report reproducible incorrect behavior in the kernel or a repository integration. +title: "[Bug]: " +labels: + - bug +body: + - type: markdown + attributes: + value: | + Thanks for helping improve evalopt. Do not include credentials, private data, or undisclosed security issues. Report vulnerabilities through the private security advisory link below. + - type: input + id: version + attributes: + label: evalopt version or commit + description: For example, `0.1.0` or a full commit SHA. + placeholder: 0.1.0 + validations: + required: true + - type: dropdown + id: python + attributes: + label: Python version + options: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + - Not applicable + validations: + required: true + - type: input + id: platform + attributes: + label: Operating system and architecture + placeholder: Ubuntu 24.04, x86_64 + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Minimal reproduction + description: Include the smallest policy, input, and command or code that reproduces the problem. + render: python + validations: + required: true + - type: textarea + id: observed + attributes: + label: Observed behavior + description: Include the decision status, reason codes, traceback, or sanitized output. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected behavior + validations: + required: true + - type: textarea + id: context + attributes: + label: Additional context + description: Explain whether this affects the stable kernel, a deprecated compatibility surface, or a host integration. + - type: checkboxes + id: checks + attributes: + label: Checklist + options: + - label: I searched existing issues and discussions. + required: true + - label: This report contains no secrets or private data. + required: true + - label: This is not an undisclosed security vulnerability. + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..ddd1d2e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: Questions and integration help + url: https://github.com/ajaysurya1221/evalopt-graph/discussions + about: Ask usage questions and share integration ideas in Discussions. + - name: Security vulnerability + url: https://github.com/ajaysurya1221/evalopt-graph/security/advisories/new + about: Report vulnerabilities privately. Do not disclose them in a public issue. + - name: Documentation + url: https://github.com/ajaysurya1221/evalopt-graph#readme + about: Read the quickstart, boundaries, evidence status, and project documentation. diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 0000000..25303aa --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,59 @@ +name: Design proposal +description: Propose a focused kernel, adapter, documentation, or conformance improvement. +title: "[Proposal]: " +labels: + - enhancement +body: + - type: markdown + attributes: + value: | + evalopt deliberately keeps orchestration outside its stable kernel. Explain the trust-boundary impact of the proposal and the smallest public surface that solves the problem. + - type: textarea + id: problem + attributes: + label: Problem + description: What concrete user or integration problem needs to be solved? + validations: + required: true + - type: dropdown + id: boundary + attributes: + label: Primary area + options: + - Stable governance kernel + - Host or verifier adapter + - Deprecated compatibility surface + - Documentation or examples + - Conformance or evaluation + validations: + required: true + - type: textarea + id: behavior + attributes: + label: Proposed behavior + description: Describe inputs, outputs, invariants, and failure behavior without assuming a specific agent runtime. + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Could the host own this behavior without changing the kernel? + validations: + required: true + - type: textarea + id: compatibility + attributes: + label: Compatibility and evidence impact + description: Note effects on the ten-symbol API, serialized records, replay, reason codes, or evidence claims. + validations: + required: true + - type: checkboxes + id: checks + attributes: + label: Checklist + options: + - label: I searched existing issues and discussions. + required: true + - label: I have not presented authored or generated cases as benchmark or SOTA evidence. + required: true diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..be72df7 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,23 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "04:00" + timezone: "Etc/UTC" + open-pull-requests-limit: 5 + commit-message: + prefix: "deps" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "04:30" + timezone: "Etc/UTC" + open-pull-requests-limit: 5 + commit-message: + prefix: "ci" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..4b209a8 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,29 @@ +## Summary + + + +## Boundary and compatibility + + + +- Stable ten-symbol API impact: +- Serialized record / replay impact: +- Trust-boundary impact: + +## Evidence and testing + + + +```text +commands and results +``` + +## Checklist + +- [ ] The change is focused and includes regression tests where behavior changed. +- [ ] Kernel and host responsibilities remain explicit; no second acceptance-authority path was added. +- [ ] Stable API, reason-code, serialization, and migration effects are documented. +- [ ] Evidence claims match the demonstrated evidence level and do not imply benchmark or SOTA proof. +- [ ] No tests or security checks were weakened to make the change pass. +- [ ] No secrets, private data, generated run artifacts, caches, or local environment files are included. +- [ ] User-facing changes are reflected in the README, changelog, or release notes as appropriate. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b5b55cb --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,195 @@ +name: CI + +on: + push: + branches: + - main + - "release/**" + pull_request: + branches: + - main + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + GITLEAKS_LINUX_X64_SHA256: "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb" + GITLEAKS_VERSION: "8.30.1" + UV_VERSION: "0.11.29" + +jobs: + linux-tests: + name: Ubuntu / Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + steps: + - name: Check out source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install uv + run: python -m pip install --disable-pip-version-check "uv==${UV_VERSION}" + - name: Sync locked development environment + run: uv sync --frozen --extra dev + - name: Run full test suite + run: uv run --frozen --extra dev python -m pytest -p no:cacheprovider + + macos-tests: + name: macOS / Python 3.14 + runs-on: macos-latest + timeout-minutes: 25 + steps: + - name: Check out source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + - name: Install uv + run: python -m pip install --disable-pip-version-check "uv==${UV_VERSION}" + - name: Sync locked development environment + run: uv sync --frozen --extra dev + - name: Run full test suite + run: uv run --frozen --extra dev python -m pytest -p no:cacheprovider + + quality-build: + name: Quality, demos, smoke, and artifacts + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + - name: Install uv + run: python -m pip install --disable-pip-version-check "uv==${UV_VERSION}" + - name: Verify lock and sync environment + run: | + uv lock --check + uv sync --frozen --extra dev + - name: Verify action pins and test floor + run: | + uv run python scripts/verify_workflow_pins.py + uv run --frozen --extra dev python scripts/verify_test_count.py --minimum 408 + - name: Compile and lint + run: | + uv run --frozen --extra dev python -m compileall -q src tests scripts + uv run --frozen --extra dev ruff check src tests scripts + uv run --frozen --extra dev ruff format --check src tests scripts + - name: Run deterministic demos + run: | + uv run --frozen --extra dev python scripts/edit_demo.py + uv run --frozen --extra dev python scripts/epistemic_demo.py + uv run --frozen --extra dev python scripts/unattended_demo.py + uv run --frozen --extra dev python scripts/provenance_benchmark.py --out "$RUNNER_TEMP/conformance.json" + - name: Run offline smoke test + run: bash scripts/smoke_test.sh + - name: Build distributions once + run: uv build --out-dir dist + - name: Validate distributions + run: | + uvx --from twine==6.2.0 twine check dist/* + python scripts/verify_artifacts.py dist --version 0.1.0 + - name: Upload CI distributions + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-distributions + path: dist/ + if-no-files-found: error + include-hidden-files: false + retention-days: 7 + + windows-wheel: + name: Windows / clean wheel / Python 3.14 + needs: quality-build + runs-on: windows-latest + timeout-minutes: 15 + steps: + - name: Check out verification script + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + - name: Download wheel + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ci-distributions + path: dist + - name: Install wheel without dependencies + shell: pwsh + run: | + $wheel = (Get-ChildItem -Path dist -Filter *.whl -File -Recurse -ErrorAction Stop) + if ($wheel.Count -ne 1) { throw "Expected exactly one wheel, found $($wheel.Count)" } + python -m pip install --disable-pip-version-check --no-deps $wheel[0].FullName + - name: Verify exact API, serialization, and replay + run: python scripts/verify_installed_release.py --version 0.1.0 + + gitleaks: + name: Gitleaks / full history + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out full history + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + - name: Install checksum-pinned Gitleaks + run: | + archive="$RUNNER_TEMP/gitleaks.tar.gz" + bin_dir="$RUNNER_TEMP/gitleaks-bin" + curl --fail --silent --show-error --location \ + "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" \ + --output "$archive" + echo "${GITLEAKS_LINUX_X64_SHA256} ${archive}" | sha256sum --check --strict - + mkdir -p "$bin_dir" + tar -xzf "$archive" -C "$bin_dir" gitleaks + echo "$bin_dir" >> "$GITHUB_PATH" + - name: Scan every reachable commit for secrets + run: gitleaks git --redact --config .gitleaks.toml --log-opts="--all" . + + required: + name: CI / required + if: always() + needs: + - linux-tests + - macos-tests + - quality-build + - windows-wheel + - gitleaks + runs-on: ubuntu-latest + steps: + - name: Require every CI lane + env: + LINUX: ${{ needs.linux-tests.result }} + MACOS: ${{ needs.macos-tests.result }} + QUALITY: ${{ needs.quality-build.result }} + WINDOWS: ${{ needs.windows-wheel.result }} + GITLEAKS: ${{ needs.gitleaks.result }} + run: | + for result in "$LINUX" "$MACOS" "$QUALITY" "$WINDOWS" "$GITLEAKS"; do + test "$result" = "success" + done diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..e3f2c51 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,37 @@ +name: CodeQL + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "17 3 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + analyze: + name: CodeQL / Python + if: github.event.repository.visibility == 'public' + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + actions: read + contents: read + security-events: write + steps: + - name: Check out source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Initialize CodeQL + uses: github/codeql-action/init@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1 + with: + languages: python + - name: Analyze + uses: github/codeql-action/analyze@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1 + with: + category: "/language:python" diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 0000000..2cc68c9 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,20 @@ +name: Dependency review + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + review: + name: Dependency review + if: github.event.repository.visibility == 'public' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Review dependency changes + uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + fail-on-severity: moderate diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..744c2cc --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,207 @@ +name: Release + +on: + push: + tags: + - "v*.*.*" + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +env: + RELEASE_VERSION: "0.1.0" + UV_VERSION: "0.11.29" + +jobs: + required-checks: + name: Required checks + if: github.repository == 'ajaysurya1221/evalopt-graph' + uses: ./.github/workflows/ci.yml + permissions: + contents: read + + build: + name: Build and attest release + if: github.repository == 'ajaysurya1221/evalopt-graph' + needs: required-checks + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + attestations: write + contents: read + id-token: write + steps: + - name: Check out tagged source and history + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + - name: Install uv + run: python -m pip install --disable-pip-version-check "uv==${UV_VERSION}" + - name: Fetch and verify release identity + run: | + git fetch --force --no-tags origin main:refs/remotes/origin/main + uv sync --frozen --extra dev + uv run --frozen --extra dev python scripts/verify_release.py \ + --expected-version "$RELEASE_VERSION" \ + --tag "$GITHUB_REF_NAME" \ + --main-ref refs/remotes/origin/main \ + --require-annotated-tag + - name: Set reproducible build timestamp + run: echo "SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)" >> "$GITHUB_ENV" + - name: Build distributions once + run: uv build --out-dir dist + - name: Validate release distributions + run: | + uvx --from twine==6.2.0 twine check dist/* + python scripts/verify_artifacts.py dist --version "$RELEASE_VERSION" + - name: Generate checksums and SPDX SBOM + run: | + mkdir -p release-assets + (cd dist && sha256sum -- * > ../release-assets/SHA256SUMS.txt) + python scripts/generate_spdx_sbom.py \ + --dist-dir dist \ + --output "release-assets/evalopt-graph-${RELEASE_VERSION}.spdx.json" \ + --version "$RELEASE_VERSION" + - name: Attest build provenance + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-path: "dist/*" + - name: Attest SPDX SBOM + uses: actions/attest-sbom@c604332985a26aa8cf1bdc465b92731239ec6b9e # v4.1.0 + with: + subject-path: "dist/*" + sbom-path: "release-assets/evalopt-graph-${{ env.RELEASE_VERSION }}.spdx.json" + - name: Upload immutable release bundle + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-bundle + path: | + dist/ + release-assets/ + if-no-files-found: error + include-hidden-files: false + retention-days: 7 + + draft-github-release: + name: Draft GitHub release + needs: build + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - name: Check out release notes + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Download release bundle + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-bundle + path: bundle + - name: Create or refresh draft release + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + run: | + tag="$GITHUB_REF_NAME" + title="evalopt v${RELEASE_VERSION} — Evidence over vibes" + gh api "repos/${GH_REPO}/git/ref/tags/${tag}" >/dev/null + mapfile -t assets < <(find bundle/dist bundle/release-assets -maxdepth 1 -type f -print | sort) + test "${#assets[@]}" -eq 4 + state=$(gh release view "$tag" --json isDraft --jq .isDraft 2>/dev/null || true) + if [[ "$state" == "false" ]]; then + echo "Refusing to modify an already-published release" >&2 + exit 1 + elif [[ "$state" == "true" ]]; then + gh release edit "$tag" --title "$title" --notes-file docs/releases/v0.1.0.md + existing_dir="$RUNNER_TEMP/existing-release-assets" + mkdir -p "$existing_dir" + gh release download "$tag" --dir "$existing_dir" + mapfile -t existing_assets < <(find "$existing_dir" -maxdepth 1 -type f -print | sort) + if [[ "${#existing_assets[@]}" -ne "${#assets[@]}" ]]; then + echo "Existing draft has a partial or unexpected asset set; refusing to mutate it" >&2 + exit 1 + fi + for expected in "${assets[@]}"; do + actual="$existing_dir/$(basename "$expected")" + if [[ ! -f "$actual" ]] || ! cmp --silent "$expected" "$actual"; then + echo "Existing draft asset differs from the rebuilt bundle: $(basename "$expected")" >&2 + exit 1 + fi + done + echo "Existing draft assets are byte-identical; leaving them untouched" + else + gh release create "$tag" "${assets[@]}" \ + --draft \ + --verify-tag \ + --title "$title" \ + --notes-file docs/releases/v0.1.0.md + fi + + publish-pypi: + name: Publish to PyPI + needs: draft-github-release + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: + name: pypi + url: https://pypi.org/p/evalopt-graph + permissions: + id-token: write + steps: + - name: Download release bundle + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-bundle + path: bundle + - name: Publish distributions with Trusted Publishing + uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1 + with: + packages-dir: bundle/dist/ + print-hash: true + + verify-pypi: + name: Verify clean PyPI install + needs: publish-pypi + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Check out verifier + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + - name: Install from PyPI and verify replay + run: python scripts/verify_pypi_release.py --version "$RELEASE_VERSION" --attempts 6 --delay-seconds 10 + + publish-github-release: + name: Publish GitHub release + needs: verify-pypi + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: write + steps: + - name: Publish draft as Latest + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + run: | + gh release edit "$GITHUB_REF_NAME" --draft=false --latest + test "$(gh release view "$GITHUB_REF_NAME" --json isDraft --jq .isDraft)" = "false" + test "$(gh release view --json tagName --jq .tagName)" = "$GITHUB_REF_NAME" diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..0468db7 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,11 @@ +title = "evalopt-graph secret scanning" + +[extend] +useDefault = true + +[[allowlists]] +description = "Exact synthetic OpenAI-shaped token used by redaction and credential-boundary tests" +condition = "AND" +paths = ['''^tests/(test_context_hygiene|test_unattended)\.py$'''] +regexTarget = "match" +regexes = ['''sk-abcdefghij1234567890'''] diff --git a/README.md b/README.md index c080370..abf34b1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@
- evalopt: Agents propose. Policy decides. A noisy proposal signal passes through Gates, Evidence, and Bounds, then forks to Accepted or Blocked. + evalopt: Agents propose. Policy decides. A noisy proposal signal passes through Gates, Evidence, and Bounds, then forks to Accepted or Blocked.
@@ -8,7 +8,7 @@ [![CI](https://github.com/ajaysurya1221/evalopt-graph/actions/workflows/ci.yml/badge.svg)](https://github.com/ajaysurya1221/evalopt-graph/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/evalopt-graph)](https://pypi.org/project/evalopt-graph/) [![Python](https://img.shields.io/pypi/pyversions/evalopt-graph)](https://pypi.org/project/evalopt-graph/) -[![License: MIT](https://img.shields.io/badge/license-MIT-49C6B8.svg)](LICENSE) +[![License: MIT](https://img.shields.io/badge/license-MIT-49C6B8.svg)](https://github.com/ajaysurya1221/evalopt-graph/blob/main/LICENSE) **Agents propose. Policy decides.** @@ -84,8 +84,10 @@ from evalopt_graph import ( - `evaluate_acceptance` evaluates a complete immutable observation once and returns a replayable `AcceptanceDecision`. -See [`kernel.py`](src/evalopt_graph/kernel.py) for the compact public contract and -[`test_kernel.py`](tests/test_kernel.py) for complete evidence-authority examples. +See [`kernel.py`](https://github.com/ajaysurya1221/evalopt-graph/blob/main/src/evalopt_graph/kernel.py) +for the compact public contract and +[`test_kernel.py`](https://github.com/ajaysurya1221/evalopt-graph/blob/main/tests/test_kernel.py) for +complete evidence-authority examples. ## Kernel owns / host owns @@ -125,9 +127,11 @@ prove semantic truth, deployed behavior, source correctness, or model capability boundary mechanism. - **Not established:** benchmark superiority, external comparison, independent reproduction, or SOTA. -Read the [evidence report](docs/BENCHMARK_RESULTS.md) and -[prospective external protocol](docs/BENCHMARK_PROTOCOL.md) for the claims, controls, and gaps. There is -no live-model governance campaign or official Docker SWE-bench result in this release. +Read the +[evidence report](https://github.com/ajaysurya1221/evalopt-graph/blob/main/docs/BENCHMARK_RESULTS.md) and +[prospective external protocol](https://github.com/ajaysurya1221/evalopt-graph/blob/main/docs/BENCHMARK_PROTOCOL.md) +for the claims, controls, and gaps. There is no live-model governance campaign or official Docker +SWE-bench result in this release. ## Integrate it anywhere @@ -135,9 +139,10 @@ The kernel is host-independent: adapt observations from Codex, Claude Code, Open or your own runtime into `AcceptanceInput`, then store the returned decision beside the policy and input used to create it. -An optional [Harbor 0.18 conformance task](bench/harbor/README.md) demonstrates verifier-side mapping -without importing Harbor into the kernel. It validates integration wiring only; it is not an external -benchmark. +An optional +[Harbor 0.18 conformance task](https://github.com/ajaysurya1221/evalopt-graph/blob/main/bench/harbor/README.md) +demonstrates verifier-side mapping without importing Harbor into the kernel. It validates integration +wiring only; it is not an external benchmark. The historical graph, standalone CLI, provider clients, Codex bridge, research loop, and filesystem adapters remain as **deprecated compatibility surfaces for one migration cycle**. They are not stable @@ -145,21 +150,24 @@ API or policy authorities. New integrations should depend only on the ten root e ## Documentation -- [Architecture and trust boundaries](docs/architecture.md) -- [Evidence results and limitations](docs/BENCHMARK_RESULTS.md) -- [Prospective benchmark protocol](docs/BENCHMARK_PROTOCOL.md) -- [Harbor integration](bench/harbor/README.md) -- [v0.1.0 release notes](docs/releases/v0.1.0.md) -- [Changelog](CHANGELOG.md) +- [Architecture and trust boundaries](https://github.com/ajaysurya1221/evalopt-graph/blob/main/docs/architecture.md) +- [Evidence results and limitations](https://github.com/ajaysurya1221/evalopt-graph/blob/main/docs/BENCHMARK_RESULTS.md) +- [Prospective benchmark protocol](https://github.com/ajaysurya1221/evalopt-graph/blob/main/docs/BENCHMARK_PROTOCOL.md) +- [Harbor integration](https://github.com/ajaysurya1221/evalopt-graph/blob/main/bench/harbor/README.md) +- [v0.1.0 release notes](https://github.com/ajaysurya1221/evalopt-graph/blob/main/docs/releases/v0.1.0.md) +- [Changelog](https://github.com/ajaysurya1221/evalopt-graph/blob/main/CHANGELOG.md) ## Community and security Bug reports, focused proposals, documentation improvements, and integrations are welcome. Start with -the [contribution guide](CONTRIBUTING.md) and follow the [Code of Conduct](CODE_OF_CONDUCT.md). +the [contribution guide](https://github.com/ajaysurya1221/evalopt-graph/blob/main/CONTRIBUTING.md) and +follow the [Code of Conduct](https://github.com/ajaysurya1221/evalopt-graph/blob/main/CODE_OF_CONDUCT.md). Please report vulnerabilities privately using [GitHub Security Advisories](https://github.com/ajaysurya1221/evalopt-graph/security/advisories/new), not a -public issue. See the [security policy](SECURITY.md) for supported versions and response expectations. +public issue. See the +[security policy](https://github.com/ajaysurya1221/evalopt-graph/blob/main/SECURITY.md) for supported +versions and response expectations. ## Development @@ -180,4 +188,5 @@ uv build ## License -Released under the [MIT License](LICENSE). Copyright © 2026 Ajay Surya Senthilrajan. +Released under the [MIT License](https://github.com/ajaysurya1221/evalopt-graph/blob/main/LICENSE). +Copyright © 2026 Ajay Surya Senthilrajan. diff --git a/pyproject.toml b/pyproject.toml index 82d4307..f2d83ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,16 +1,31 @@ [build-system] -requires = ["hatchling"] +requires = ["hatchling==1.31.0"] build-backend = "hatchling.build" [project] name = "evalopt-graph" version = "0.1.0" -description = "Host-independent deterministic governance and evidence-integrity kernel." +description = "A zero-dependency governance and evidence-integrity kernel for any agent runtime." readme = "README.md" requires-python = ">=3.10" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] authors = [{ name = "Ajay Surya Senthilrajan" }] +maintainers = [{ name = "Ajay Surya Senthilrajan" }] keywords = ["agents", "governance", "evaluation", "evidence", "reproducibility"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Software Development :: Quality Assurance", +] # The stable kernel has zero required dependencies and imports no host runtime or provider SDK. dependencies = [] @@ -24,9 +39,45 @@ all = ["evalopt-graph[llm,yaml,dev]"] # self-referential to avoid constr [project.scripts] evalopt = "evalopt_graph.cli:main" +[project.urls] +Homepage = "https://github.com/ajaysurya1221/evalopt-graph" +Documentation = "https://github.com/ajaysurya1221/evalopt-graph#readme" +Repository = "https://github.com/ajaysurya1221/evalopt-graph" +Issues = "https://github.com/ajaysurya1221/evalopt-graph/issues" +Changelog = "https://github.com/ajaysurya1221/evalopt-graph/blob/main/CHANGELOG.md" + [tool.hatch.build.targets.wheel] packages = ["src/evalopt_graph"] +[tool.hatch.build.targets.sdist] +include = [ + "/.gitignore", + "/src", + "/tests", + "/bench/harbor", + "/claude_assets", + "/docs", + "/scripts", + "/CHANGELOG.md", + "/CODE_OF_CONDUCT.md", + "/CONTRIBUTING.md", + "/LICENSE", + "/README.md", + "/SECURITY.md", + "/pyproject.toml", +] +exclude = [ + "/.backups", + "/.context", + "/.evalopt", + "/.github", + "/.pytest_cache", + "/.ruff_cache", + "/.venv", + "/build", + "/dist", +] + [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-q" diff --git a/scripts/generate_spdx_sbom.py b/scripts/generate_spdx_sbom.py new file mode 100644 index 0000000..3c5369e --- /dev/null +++ b/scripts/generate_spdx_sbom.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Generate a compact SPDX 2.3 JSON SBOM for the two release archives.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from datetime import datetime, timezone +from pathlib import Path + + +def digest(path: Path, algorithm: str) -> str: + hasher = hashlib.new(algorithm) + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def creation_timestamp() -> str: + """Return a reproducible SPDX timestamp when SOURCE_DATE_EPOCH is set.""" + raw_epoch = os.environ.get("SOURCE_DATE_EPOCH") + moment = ( + datetime.fromtimestamp(int(raw_epoch), timezone.utc) + if raw_epoch is not None + else datetime.now(timezone.utc) + ) + return moment.replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dist-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--version", required=True) + args = parser.parse_args() + + artifacts = sorted([*args.dist_dir.glob("*.whl"), *args.dist_dir.glob("*.tar.gz")]) + if len(artifacts) != 2: + raise SystemExit(f"expected one wheel and one sdist, found {artifacts!r}") + identity = hashlib.sha256("".join(digest(path, "sha256") for path in artifacts).encode()).hexdigest() + files = [] + packages = [] + relationships = [] + for index, path in enumerate(artifacts, 1): + file_id = f"SPDXRef-File-{index}" + package_id = f"SPDXRef-Package-{index}" + sha1 = digest(path, "sha1") + sha256 = digest(path, "sha256") + verification_code = hashlib.sha1(sha1.encode("ascii")).hexdigest() + kind = "wheel" if path.suffix == ".whl" else "sdist" + files.append( + { + "SPDXID": file_id, + "fileName": f"./{path.name}", + "checksums": [ + {"algorithm": "SHA1", "checksumValue": sha1}, + {"algorithm": "SHA256", "checksumValue": sha256}, + ], + "fileTypes": ["ARCHIVE"], + "licenseConcluded": "MIT", + "copyrightText": "Copyright (c) 2026 Ajay Surya Senthilrajan", + } + ) + packages.append( + { + "SPDXID": package_id, + "name": f"evalopt-graph-{kind}", + "versionInfo": args.version, + "packageFileName": path.name, + "downloadLocation": f"https://pypi.org/project/evalopt-graph/{args.version}/#files", + "filesAnalyzed": True, + "packageVerificationCode": {"packageVerificationCodeValue": verification_code}, + "checksums": [{"algorithm": "SHA256", "checksumValue": sha256}], + "licenseConcluded": "MIT", + "licenseDeclared": "MIT", + "copyrightText": "Copyright (c) 2026 Ajay Surya Senthilrajan", + "supplier": "Person: Ajay Surya Senthilrajan", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": f"pkg:pypi/evalopt-graph@{args.version}", + } + ], + } + ) + relationships.extend( + [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": package_id, + }, + { + "spdxElementId": package_id, + "relationshipType": "CONTAINS", + "relatedSpdxElement": file_id, + }, + ] + ) + + document = { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": f"evalopt-graph-{args.version}-release", + "documentNamespace": f"https://github.com/ajaysurya1221/evalopt-graph/releases/download/v{args.version}/spdx-{identity}", + "creationInfo": { + "created": creation_timestamp(), + "creators": [ + "Tool: evalopt-release-automation/1", + "Person: Ajay Surya Senthilrajan", + ], + }, + "documentDescribes": [package["SPDXID"] for package in packages], + "packages": packages, + "files": files, + "relationships": relationships, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(args.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh index a7018f7..a4eeff3 100755 --- a/scripts/smoke_test.sh +++ b/scripts/smoke_test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Offline smoke test for the Evaluator-Optimizer Graph template. +# Offline smoke test for the evalopt compatibility host. # Requires NO API keys and does NOT install LangGraph. Uses uv if present, else python venv. set -euo pipefail diff --git a/scripts/verify_artifacts.py b/scripts/verify_artifacts.py new file mode 100644 index 0000000..22026f9 --- /dev/null +++ b/scripts/verify_artifacts.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Inspect release archives for package shape, hygiene, and dependency invariants.""" + +from __future__ import annotations + +import argparse +import email +import re +import tarfile +import zipfile +from pathlib import Path, PurePosixPath + +ALLOWED_SDIST_ROOTS = { + ".gitignore", + "CHANGELOG.md", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "PKG-INFO", + "README.md", + "SECURITY.md", + "bench", + "claude_assets", + "docs", + "pyproject.toml", + "scripts", + "src", + "tests", +} +FORBIDDEN_COMPONENTS = { + ".backups", + ".context", + ".evalopt", + ".git", + ".github", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".venv", + "__pycache__", + "build", + "dist", + "runs", +} +PERSONAL_MARKERS = tuple( + "".join(parts).encode() + for parts in ( + ("/Users/", "ajay/"), + ("evalopt-graph-", "template"), + ("conductor/", "workspaces"), + ("~/", "Developer/"), + ) +) + + +def fail(message: str) -> None: + raise AssertionError(message) + + +def check_common(path: PurePosixPath, data: bytes | None = None) -> None: + if any(part in FORBIDDEN_COMPONENTS for part in path.parts): + fail(f"forbidden artifact path: {path}") + name = path.name.casefold() + if name == ".env" or name.startswith(".env.") or name.endswith((".pyc", ".pyo")): + fail(f"environment or cache file in artifact: {path}") + if data is not None and b"\x00" not in data[:8192]: + for marker in PERSONAL_MARKERS: + if marker in data: + fail(f"personal workspace marker {marker!r} in {path}") + + +def verify_wheel(wheel: Path, version: str) -> None: + with zipfile.ZipFile(wheel) as archive: + names = [PurePosixPath(name) for name in archive.namelist() if not name.endswith("/")] + package_files = [path for path in names if path.parts[0] == "evalopt_graph"] + dist_info_roots = {path.parts[0] for path in names if path.parts[0].endswith(".dist-info")} + if not package_files: + fail("wheel does not contain evalopt_graph") + if len(dist_info_roots) != 1: + fail(f"wheel must contain exactly one dist-info directory: {dist_info_roots!r}") + dist_info = next(iter(dist_info_roots)) + for path in names: + check_common(path, archive.read(str(path))) + if path.parts[0] not in {"evalopt_graph", dist_info}: + fail(f"wheel contains non-package payload: {path}") + required = {"METADATA", "RECORD", "WHEEL"} + present = {path.name for path in names if path.parts[0] == dist_info} + if not required <= present: + fail(f"wheel metadata missing {sorted(required - present)}") + if not any( + path.parts[0] == dist_info and "licenses" in path.parts and path.name == "LICENSE" + for path in names + ): + fail("wheel does not carry the MIT LICENSE as license metadata") + + metadata_path = next(path for path in names if path.parts == (dist_info, "METADATA")) + metadata = email.message_from_bytes(archive.read(str(metadata_path))) + if metadata["Name"] != "evalopt-graph" or metadata["Version"] != version: + fail("wheel name/version metadata does not match the release") + if metadata["License-Expression"] != "MIT": + fail("wheel does not declare the MIT SPDX license expression") + if metadata["Description-Content-Type"] != "text/markdown" or not metadata.get_payload().strip(): + fail("wheel metadata does not contain the Markdown README") + if metadata["Requires-Python"] != ">=3.10": + fail("wheel Python requirement does not match the supported range") + runtime = [ + requirement + for requirement in metadata.get_all("Requires-Dist", []) + if not re.search(r"\bextra\s*==", requirement) + ] + if runtime: + fail(f"wheel declares runtime dependencies: {runtime!r}") + + +def verify_sdist(sdist: Path, version: str) -> None: + with tarfile.open(sdist, "r:gz") as archive: + members = [member for member in archive.getmembers() if member.isfile()] + roots = {PurePosixPath(member.name).parts[0] for member in members} + if len(roots) != 1: + fail(f"sdist must have one archive root: {roots!r}") + archive_root = next(iter(roots)) + paths: dict[PurePosixPath, bytes] = {} + for member in members: + full = PurePosixPath(member.name) + relative = PurePosixPath(*full.parts[1:]) + extracted = archive.extractfile(member) + data = extracted.read() if extracted is not None else b"" + check_common(relative, data) + if not relative.parts: + continue + if relative.parts[0] not in ALLOWED_SDIST_ROOTS: + fail(f"sdist path is outside the explicit public allowlist: {relative}") + if relative.parts[0] == "bench" and relative.parts[:2] != ("bench", "harbor"): + fail(f"only Harbor conformance material may ship under bench/: {relative}") + paths[relative] = data + + required = { + PurePosixPath("LICENSE"), + PurePosixPath("README.md"), + PurePosixPath("pyproject.toml"), + PurePosixPath("src/evalopt_graph/__init__.py"), + } + missing = required - paths.keys() + if missing: + fail(f"sdist missing required public files: {sorted(map(str, missing))}") + if not paths[PurePosixPath("README.md")].strip(): + fail("sdist README is empty") + if b"Copyright (c) 2026 Ajay Surya Senthilrajan" not in paths[PurePosixPath("LICENSE")]: + fail("sdist LICENSE is not the expected MIT license") + expected_root = f"evalopt_graph-{version}" + if archive_root != expected_root: + fail(f"sdist root is {archive_root!r}, expected {expected_root!r}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("dist", type=Path) + parser.add_argument("--version", default="0.1.0") + args = parser.parse_args() + wheels = sorted(args.dist.glob("*.whl")) + sdists = sorted(args.dist.glob("*.tar.gz")) + # uv owns this exact output-directory marker. It is not part of either archive and + # upload-artifact is explicitly configured to exclude hidden files. + unexpected = sorted( + path.name + for path in args.dist.iterdir() + if path.is_file() and path not in {*wheels, *sdists} and path.name != ".gitignore" + ) + if len(wheels) != 1 or len(sdists) != 1 or unexpected: + fail( + f"expected one wheel and one sdist, found wheels={wheels}, sdists={sdists}, " + f"unexpected={unexpected}" + ) + verify_wheel(wheels[0], args.version) + verify_sdist(sdists[0], args.version) + print(f"verified wheel and sdist hygiene for evalopt-graph {args.version}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_installed_release.py b/scripts/verify_installed_release.py new file mode 100644 index 0000000..f79a6ba --- /dev/null +++ b/scripts/verify_installed_release.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Verify the public API and deterministic replay from an installed wheel.""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import json +import subprocess +import sys + +STABLE_API = [ + "GovernancePolicy", + "ClaimRecord", + "EvidenceRequest", + "EvidenceMaterial", + "EvidenceAttestation", + "SupportAssessment", + "EvidenceAuthority", + "AcceptanceInput", + "AcceptanceDecision", + "evaluate_acceptance", +] + + +def verify(expected_version: str) -> None: + import evalopt_graph as evalopt + + assert evalopt.__version__ == expected_version, ( + f"package __version__ is {evalopt.__version__!r}, expected {expected_version!r}" + ) + assert importlib.metadata.version("evalopt-graph") == expected_version + assert evalopt.__all__ == STABLE_API, ( + f"stable API changed: expected {STABLE_API!r}, found {evalopt.__all__!r}" + ) + + metadata = importlib.metadata.metadata("evalopt-graph") + runtime_requirements = [ + requirement + for requirement in metadata.get_all("Requires-Dist", []) + if "extra ==" not in requirement and "extra == " not in requirement + ] + assert not runtime_requirements, f"unexpected runtime dependencies: {runtime_requirements!r}" + + policy = evalopt.GovernancePolicy(required_gates=("tests",)) + input_ = evalopt.AcceptanceInput( + observed_at="2026-07-20T00:00:00+00:00", + gate_results=(("tests", "PASS"),), + ) + decision = evalopt.evaluate_acceptance(policy, input_) + assert decision.status == "ACCEPTED" + assert decision.validate() + assert decision.replay(policy, input_) + + encoded = json.dumps(decision.to_dict(), sort_keys=True) + restored = evalopt.AcceptanceDecision.from_dict(json.loads(encoded)) + assert restored == decision + assert restored.validate() + assert restored.replay(policy, input_) + + cli_version = subprocess.run( + [sys.executable, "-m", "evalopt_graph.cli", "--version"], + check=True, + capture_output=True, + text=True, + timeout=30, + ).stdout.strip() + assert cli_version == f"evalopt {expected_version}", cli_version + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True) + args = parser.parse_args() + verify(args.version) + print(f"installed evalopt-graph {args.version} API and replay verified") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_pypi_release.py b/scripts/verify_pypi_release.py new file mode 100644 index 0000000..6686717 --- /dev/null +++ b/scripts/verify_pypi_release.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Perform a bounded, clean PyPI install and run the installed-release verifier.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True) + parser.add_argument("--attempts", type=int, default=6) + parser.add_argument("--delay-seconds", type=int, default=10) + args = parser.parse_args() + if args.attempts < 1 or not 0 <= args.delay_seconds <= 30: + raise SystemExit("attempts must be positive and delay must be between 0 and 30 seconds") + + with tempfile.TemporaryDirectory(prefix="evalopt-pypi-") as directory: + environment = Path(directory) / "venv" + subprocess.run([sys.executable, "-m", "venv", str(environment)], check=True, timeout=60) + python = environment / ("Scripts/python.exe" if sys.platform == "win32" else "bin/python") + command = [ + str(python), + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--no-cache-dir", + "--no-deps", + "--only-binary=:all:", + "--retries=0", + "--timeout=15", + "--index-url=https://pypi.org/simple", + f"evalopt-graph=={args.version}", + ] + last: subprocess.SubprocessError | None = None + for attempt in range(1, args.attempts + 1): + try: + subprocess.run(command, check=True, timeout=30) + last = None + break + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc: + last = exc + if attempt < args.attempts: + print(f"PyPI install attempt {attempt} failed; retrying", file=sys.stderr) + time.sleep(args.delay_seconds) + if last is not None: + raise SystemExit(f"PyPI install failed after {args.attempts} attempts") from last + subprocess.run( + [ + str(python), + str(ROOT / "scripts" / "verify_installed_release.py"), + "--version", + args.version, + ], + check=True, + timeout=60, + ) + print(f"PyPI evalopt-graph {args.version} installation verified") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_release.py b/scripts/verify_release.py new file mode 100644 index 0000000..73243b8 --- /dev/null +++ b/scripts/verify_release.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Verify release version identity and, for tag builds, Git provenance.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +import tomllib +from verify_installed_release import verify + +ROOT = Path(__file__).resolve().parents[1] + + +def git(*args: str) -> str: + return subprocess.run( + ["git", "-C", str(ROOT), *args], + check=True, + capture_output=True, + text=True, + timeout=30, + ).stdout.strip() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--expected-version", required=True) + parser.add_argument("--tag") + parser.add_argument("--main-ref") + parser.add_argument("--require-annotated-tag", action="store_true") + args = parser.parse_args() + + project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] + assert project["name"] == "evalopt-graph" + assert project["version"] == args.expected_version + assert project["dependencies"] == [], "the stable kernel must have zero runtime dependencies" + assert project["requires-python"] == ">=3.10" + verify(args.expected_version) + + if args.tag: + expected_tag = f"v{args.expected_version}" + assert args.tag == expected_tag, f"tag {args.tag!r} does not match {expected_tag!r}" + tag_type = git("cat-file", "-t", args.tag) + if args.require_annotated_tag: + assert tag_type == "tag", f"{args.tag} must be an annotated tag, found {tag_type!r}" + tag_commit = git("rev-parse", f"{args.tag}^{{commit}}") + head_commit = git("rev-parse", "HEAD") + assert tag_commit == head_commit, "checked-out commit is not the tagged commit" + if args.main_ref: + main_commit = git("rev-parse", f"{args.main_ref}^{{commit}}") + assert tag_commit == main_commit, ( + f"tag commit {tag_commit} is not the exact {args.main_ref} commit {main_commit}" + ) + + print(f"release identity verified for evalopt-graph {args.expected_version}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (AssertionError, subprocess.CalledProcessError) as exc: + print(f"release verification failed: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/scripts/verify_test_count.py b/scripts/verify_test_count.py new file mode 100644 index 0000000..5a4fb8a --- /dev/null +++ b/scripts/verify_test_count.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Fail when pytest collects fewer than the release's regression-test floor.""" + +from __future__ import annotations + +import argparse + +import pytest + + +class CollectionCounter: + count = 0 + + def pytest_collection_finish(self, session: pytest.Session) -> None: + self.count = len(session.items) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--minimum", type=int, required=True) + args = parser.parse_args() + counter = CollectionCounter() + result = pytest.main(["--collect-only", "-q", "-p", "no:cacheprovider"], plugins=[counter]) + if result != pytest.ExitCode.OK: + raise SystemExit(int(result)) + if counter.count < args.minimum: + raise SystemExit(f"collected {counter.count} tests; release floor is {args.minimum}") + print(f"collected {counter.count} tests (minimum {args.minimum})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_workflow_pins.py b/scripts/verify_workflow_pins.py new file mode 100644 index 0000000..afed672 --- /dev/null +++ b/scripts/verify_workflow_pins.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Require immutable 40-character commit pins for external GitHub Actions.""" + +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +USE = re.compile(r"^\s*-?\s*uses:\s*([^\s#]+)", re.MULTILINE) +PINNED = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_./-]+)?@[0-9a-f]{40}$") + + +def main() -> int: + failures: list[str] = [] + workflows = sorted((ROOT / ".github" / "workflows").glob("*.y*ml")) + if not workflows: + raise SystemExit("no GitHub Actions workflows found") + for workflow in workflows: + for action in USE.findall(workflow.read_text(encoding="utf-8")): + if action.startswith("./"): + continue + if not PINNED.fullmatch(action): + failures.append(f"{workflow.relative_to(ROOT)}: {action}") + if failures: + raise SystemExit("unpinned GitHub Actions:\n" + "\n".join(failures)) + print(f"verified immutable action pins in {len(workflows)} workflows") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())