From 0d98797d5116c2a8e79675fc039fda3db2cdcc51 Mon Sep 17 00:00:00 2001 From: ci-smoke Date: Fri, 10 Jul 2026 15:21:07 +0800 Subject: [PATCH] chore(security): add public repository boundary --- .agents/skills/hackforger-development | 1 + .claude/agents/forgejo-dev.md | 2 + .claude/skills/hackforger-development | 1 + .github/pull_request_template.md | 14 + .github/workflows/content-publisher.yml | 31 + .../workflows/public-repository-boundary.yml | 261 +++++ .gitignore | 15 +- AGENTS.md | 5 + CODEOWNERS | 25 + SECURITY.md | 34 + deploy/README.md | 34 + deploy/content/README.md | 176 ++++ deploy/content/config.example | 10 + deploy/content/generate-manifest.sh | 72 ++ deploy/content/publish-overlay.sh | 619 +++++++++++ deploy/content/remote-transaction.sh | 854 +++++++++++++++ deploy/content/rename-exchange.py | 154 +++ deploy/content/tests/run.sh | 585 +++++++++++ scripts/check-public-repository-boundary.sh | 20 + scripts/ci/boundary_guard_policy.py | 24 + scripts/ci/check_boundary_guard_integrity.py | 65 ++ .../ci/check_public_repository_boundary.py | 972 ++++++++++++++++++ .../ci/fingerprint_private_content_marker.py | 23 + scripts/ci/private-content-markers.txt | 10 + scripts/ci/test_boundary_guard_integrity.py | 90 ++ scripts/ci/test_hook_installer.py | 338 ++++++ scripts/ci/test_pre_push_boundary.py | 452 ++++++++ scripts/ci/test_public_repository_boundary.py | 593 +++++++++++ scripts/install-public-boundary-hook.sh | 224 ++++ scripts/pre-push-public-boundary.sh | 210 ++++ skills/hackforger-development/SKILL.md | 102 ++ .../hackforger-development/agents/openai.yaml | 4 + 32 files changed, 6016 insertions(+), 4 deletions(-) create mode 120000 .agents/skills/hackforger-development create mode 120000 .claude/skills/hackforger-development create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/content-publisher.yml create mode 100644 .github/workflows/public-repository-boundary.yml create mode 100644 AGENTS.md create mode 100644 SECURITY.md create mode 100644 deploy/README.md create mode 100644 deploy/content/README.md create mode 100644 deploy/content/config.example create mode 100755 deploy/content/generate-manifest.sh create mode 100755 deploy/content/publish-overlay.sh create mode 100755 deploy/content/remote-transaction.sh create mode 100755 deploy/content/rename-exchange.py create mode 100755 deploy/content/tests/run.sh create mode 100755 scripts/check-public-repository-boundary.sh create mode 100644 scripts/ci/boundary_guard_policy.py create mode 100644 scripts/ci/check_boundary_guard_integrity.py create mode 100755 scripts/ci/check_public_repository_boundary.py create mode 100755 scripts/ci/fingerprint_private_content_marker.py create mode 100644 scripts/ci/private-content-markers.txt create mode 100644 scripts/ci/test_boundary_guard_integrity.py create mode 100755 scripts/ci/test_hook_installer.py create mode 100755 scripts/ci/test_pre_push_boundary.py create mode 100755 scripts/ci/test_public_repository_boundary.py create mode 100755 scripts/install-public-boundary-hook.sh create mode 100755 scripts/pre-push-public-boundary.sh create mode 100644 skills/hackforger-development/SKILL.md create mode 100644 skills/hackforger-development/agents/openai.yaml diff --git a/.agents/skills/hackforger-development b/.agents/skills/hackforger-development new file mode 120000 index 0000000000..c829692618 --- /dev/null +++ b/.agents/skills/hackforger-development @@ -0,0 +1 @@ +../../skills/hackforger-development \ No newline at end of file diff --git a/.claude/agents/forgejo-dev.md b/.claude/agents/forgejo-dev.md index e4bc16d48d..6c4daaf30c 100644 --- a/.claude/agents/forgejo-dev.md +++ b/.claude/agents/forgejo-dev.md @@ -9,6 +9,8 @@ model: opus You are an expert Go developer specializing in Forgejo's architecture. When implementing HackForger features: +Before editing, read and follow `skills/hackforger-development/SKILL.md`, including its public-repository boundary and private-content hydration rules. + ## Architecture Rules 1. **Layer discipline**: routers -> services -> models -> modules. Never import upward. 2. **XORM patterns**: Use `xorm:"pk autoincr"` tags. Register tables in `models/hackforger/init.go`. diff --git a/.claude/skills/hackforger-development b/.claude/skills/hackforger-development new file mode 120000 index 0000000000..c829692618 --- /dev/null +++ b/.claude/skills/hackforger-development @@ -0,0 +1 @@ +../../skills/hackforger-development \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000000..295bbea477 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,14 @@ +## Summary + + + +## Verification + +- [ ] I ran the focused tests for this change. +- [ ] I ran `bash scripts/check-public-repository-boundary.sh`. +- [ ] This PR contains no branded/customer content, real deployment facts, + credentials, production evidence, or operator-local files. +- [ ] Any business-specific counterpart was placed in the confirmed private + business repository and is referenced only from its private handoff. +- [ ] User-facing behavior was verified with appropriate runtime evidence; + instance-specific evidence is stored outside this public repository. diff --git a/.github/workflows/content-publisher.yml b/.github/workflows/content-publisher.yml new file mode 100644 index 0000000000..06a3a61de5 --- /dev/null +++ b/.github/workflows/content-publisher.yml @@ -0,0 +1,31 @@ +name: Content publisher + +on: + pull_request: + push: + branches: + - v0.1-dev/hackforger + - prod + workflow_dispatch: + +permissions: + contents: read + +jobs: + publisher: + name: Content publisher + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - name: Check scripts + run: | + bash -n deploy/content/*.sh deploy/content/tests/run.sh + python3 -c 'import ast, pathlib; ast.parse(pathlib.Path("deploy/content/rename-exchange.py").read_text())' + + - name: Run content publisher transaction suite + run: bash deploy/content/tests/run.sh diff --git a/.github/workflows/public-repository-boundary.yml b/.github/workflows/public-repository-boundary.yml new file mode 100644 index 0000000000..e2f6617be5 --- /dev/null +++ b/.github/workflows/public-repository-boundary.yml @@ -0,0 +1,261 @@ +name: Public repository boundary + +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review, edited] + push: + branches: ['**'] + tags: ['**'] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: public-boundary-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + trusted-pr-boundary: + if: github.event_name == 'pull_request_target' + name: Trusted public-boundary policy + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read + statuses: write + steps: + - name: Mark candidate boundary status pending + id: pending_status + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] + gh api --method POST "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" \ + -f state=pending \ + -f context='Public repository boundary' \ + -f description='Trusted default-branch policy is scanning this commit' \ + -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + + # The policy always comes from the protected default branch. The target + # base is checked out separately and is used only as the opaque-file + # baseline. Candidate code is data and is never executed. + - name: Check out trusted default-branch policy + id: policy_checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ github.repository }} + ref: ${{ github.event.repository.default_branch }} + path: policy + persist-credentials: false + + - name: Check out target base baseline + id: baseline_checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ github.repository }} + ref: ${{ github.event.pull_request.base.sha }} + path: baseline + persist-credentials: false + + - name: Check out untrusted candidate as data + id: candidate_checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + path: candidate + persist-credentials: false + + - name: Ensure target base exists in candidate history + id: candidate_base + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + UPSTREAM_REPOSITORY: ${{ github.repository }} + run: | + [[ "$BASE_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$UPSTREAM_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] + if ! git -C candidate cat-file -e "$BASE_SHA^{commit}" 2>/dev/null; then + GIT_TERMINAL_PROMPT=0 git \ + -c credential.helper= \ + -c protocol.version=2 \ + -c protocol.file.allow=never \ + -c protocol.ext.allow=never \ + -C candidate fetch --no-tags --depth=1 \ + "https://github.com/$UPSTREAM_REPOSITORY.git" \ + "$BASE_SHA:refs/boundary/base" + fi + resolved=$(git -C candidate rev-parse --verify "$BASE_SHA^{commit}") + [ "$resolved" = "$BASE_SHA" ] + + - name: Test trusted boundary policy + id: policy_tests + run: >- + python3 -m unittest discover + -s policy/scripts/ci + -p 'test_*.py' + + - name: Require guard files to match trusted policy + id: guard_integrity + run: >- + python3 policy/scripts/ci/check_boundary_guard_integrity.py + --trusted-root policy + --candidate-root candidate + + - name: Scan complete candidate tree with trusted policy + id: candidate_scan + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + run: | + python3 policy/scripts/ci/check_public_repository_boundary.py \ + --root candidate \ + --policy policy/scripts/ci/private-content-markers.txt \ + --baseline-root baseline \ + --history-base-ref "$BASE_SHA" \ + --history-head-ref "$HEAD_SHA" \ + --ref-name "refs/heads/$HEAD_REF" + + - name: Publish candidate boundary status + if: always() + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + STATUS_STATE: ${{ steps.pending_status.outcome == 'success' && steps.policy_checkout.outcome == 'success' && steps.baseline_checkout.outcome == 'success' && steps.candidate_checkout.outcome == 'success' && steps.candidate_base.outcome == 'success' && steps.policy_tests.outcome == 'success' && steps.guard_integrity.outcome == 'success' && steps.candidate_scan.outcome == 'success' && 'success' || 'failure' }} + run: | + [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$BASE_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$PR_NUMBER" =~ ^[0-9]+$ ]] + current_head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq .head.sha) + current_base=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq .base.sha) + if [ "$current_head" != "$HEAD_SHA" ] || [ "$current_base" != "$BASE_SHA" ]; then + echo 'Skipping final status from a stale pull-request event.' + exit 0 + fi + if [ "$STATUS_STATE" = success ]; then + description='Trusted public repository boundary passed' + else + description='Trusted public repository boundary failed' + fi + gh api --method POST "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" \ + -f state="$STATUS_STATE" \ + -f context='Public repository boundary' \ + -f description="$description" \ + -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + + pushed-tree-boundary: + if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && !github.event.deleted) + name: Public repository boundary (advisory push scan) + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out trusted default-branch policy + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ github.repository }} + ref: ${{ github.event.repository.default_branch }} + path: policy + persist-credentials: false + + - name: Check out pushed tree as data + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ github.repository }} + ref: ${{ github.sha }} + fetch-depth: 0 + path: candidate + persist-credentials: false + + - name: Test trusted boundary policy + run: >- + python3 -m unittest discover + -s policy/scripts/ci + -p 'test_*.py' + + - name: Require guard files to match trusted policy + run: >- + python3 policy/scripts/ci/check_boundary_guard_integrity.py + --trusted-root policy + --candidate-root candidate + + - name: Resolve the exact pushed history + id: pushed_history + env: + AFTER_SHA: ${{ github.sha }} + BEFORE_SHA: ${{ github.event.before }} + PUSHED_REF: ${{ github.ref }} + UPSTREAM_REPOSITORY: ${{ github.repository }} + run: | + [[ "$AFTER_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$PUSHED_REF" =~ ^refs/(heads|tags)/[A-Za-z0-9._/-]+$ ]] + [[ "$UPSTREAM_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] + + fetch_exact() { + local oid=$1 + local destination=$2 + if ! git -C candidate cat-file -e "$oid" 2>/dev/null; then + GIT_TERMINAL_PROMPT=0 git \ + -c credential.helper= \ + -c protocol.version=2 \ + -c protocol.file.allow=never \ + -c protocol.ext.allow=never \ + -C candidate fetch --no-tags --depth=1 \ + "https://github.com/$UPSTREAM_REPOSITORY.git" \ + "$oid:$destination" + fi + } + + if [[ "$PUSHED_REF" == refs/tags/* ]]; then + GIT_TERMINAL_PROMPT=0 git \ + -c credential.helper= \ + -c protocol.version=2 \ + -c protocol.file.allow=never \ + -c protocol.ext.allow=never \ + -C candidate fetch --no-tags --depth=1 \ + "https://github.com/$UPSTREAM_REPOSITORY.git" \ + "+$PUSHED_REF:refs/boundary/pushed-tag" + pushed_object=$(git -C candidate rev-parse --verify refs/boundary/pushed-tag) + else + fetch_exact "$AFTER_SHA" refs/boundary/pushed + pushed_object=$AFTER_SHA + fi + object_type=$(git -C candidate cat-file -t "$pushed_object") + [ "$object_type" = commit ] || { + echo 'Annotated tags and non-commit refs require dedicated security review.' >&2 + exit 1 + } + candidate_commit=$(git -C candidate rev-parse --verify "$pushed_object^{commit}") + [ "$candidate_commit" = "$AFTER_SHA" ] + + if [[ "$BEFORE_SHA" =~ ^0{40}$ ]] || [ -z "$BEFORE_SHA" ]; then + baseline_commit=$(git -C policy rev-parse --verify HEAD) + else + [[ "$BEFORE_SHA" =~ ^[0-9a-f]{40}$ ]] + baseline_commit=$BEFORE_SHA + fi + fetch_exact "$baseline_commit" refs/boundary/baseline + resolved_baseline=$(git -C candidate rev-parse --verify "$baseline_commit^{commit}") + [ "$resolved_baseline" = "$baseline_commit" ] + + echo "history_base=$baseline_commit" >> "$GITHUB_OUTPUT" + echo "history_head=$candidate_commit" >> "$GITHUB_OUTPUT" + + - name: Scan pushed tree against trusted default branch + env: + HISTORY_BASE: ${{ steps.pushed_history.outputs.history_base }} + HISTORY_HEAD: ${{ steps.pushed_history.outputs.history_head }} + PUSHED_REF: ${{ github.ref }} + run: | + python3 policy/scripts/ci/check_public_repository_boundary.py \ + --root candidate \ + --policy policy/scripts/ci/private-content-markers.txt \ + --baseline-ref "$HISTORY_BASE" \ + --history-base-ref "$HISTORY_BASE" \ + --history-head-ref "$HISTORY_HEAD" \ + --ref-name "$PUSHED_REF" diff --git a/.gitignore b/.gitignore index 6a58ba185f..b1b8a3a3c2 100644 --- a/.gitignore +++ b/.gitignore @@ -101,6 +101,8 @@ cpu.out /tests/**/*.git/**/*.sample /node_modules /.venv +__pycache__/ +*.py[cod] /yarn.lock /yarn-error.log /npm-debug.log* @@ -153,8 +155,11 @@ prime/ /man tests/integration/api_activitypub_person_inbox_useractivity_test.go -# Agent Setup runtime (gitignored) -.agents/ +# Agent runtime is local; canonical project skills are tracked through symlinks. +/.agents/* +!/.agents/skills/ +/.agents/skills/* +!/.agents/skills/hackforger-development # User-local launcher overrides claude.local.sh @@ -165,10 +170,12 @@ claude.local.sh # Local environment / secret files (PG creds, admin password, etc.) /.env /.env.local +/CLAUDE.local.md docs/tests/e2e/*.pdf .claude/*.local.md -# Project-wide guard-rail hooks ARE committed (override the .local.md ignore) -!.claude/hookify.protect-*.local.md +.claude/projects/ +.claude/worktrees/ +.claude/scheduled_tasks.lock # E2E test screenshots (temporary artifacts, hosted on GitHub Releases if needed) tests/screenshots/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..a0e4f0673d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,5 @@ +# HackForger agent instructions + +For every development, review, documentation, CI, content, or deployment task in this repository, read and follow [`skills/hackforger-development/SKILL.md`](skills/hackforger-development/SKILL.md) before editing. + +HackForger is a public, business-neutral repository. Put branded content, customer or campaign material, real deployment facts, production runbooks, and runtime evidence in the corresponding private business repository. Keep credentials out of every Git repository. diff --git a/CODEOWNERS b/CODEOWNERS index 4934331197..bb28b9c241 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -51,3 +51,28 @@ modules/structs/.* @Cyborus routers/api/v1/.* @Cyborus routers/api/forgejo/.* @Cyborus tests/integration/api_.* @Cyborus + +# HackForger public repository boundary. +/AGENTS.md @HackForger/developer +/CLAUDE.md @HackForger/developer +/CODEOWNERS @HackForger/developer +/SECURITY.md @HackForger/developer +/.env* @HackForger/developer +/.gitattributes @HackForger/developer +/.gitignore @HackForger/developer +/.gitmodules @HackForger/developer +/.agents/skills/ @HackForger/developer +/.claude/ @HackForger/developer +/.github/ @HackForger/developer +/custom/ @HackForger/developer +/deploy/ @HackForger/developer +/docs/ @HackForger/developer +/options/hackforger-help/ @HackForger/developer +/routers/web/hackforger/ @gusted @HackForger/developer +/scripts/check-public-repository-boundary.sh @HackForger/developer +/scripts/install-public-boundary-hook.sh @HackForger/developer +/scripts/pre-push-public-boundary.sh @HackForger/developer +/scripts/ci/ @HackForger/developer +/services/hackforger/ @HackForger/developer +/skills/ @HackForger/developer +/templates/hackforger/ @beowulf @gusted @HackForger/developer diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..8bd71f8018 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,34 @@ +# Security policy + +Report vulnerabilities through [GitHub private vulnerability reporting](https://github.com/HackForger/hackforger/security/advisories/new), not a public issue. + +Do not include live credentials, private keys, real production accounts, +internal addresses, customer data, or unredacted runtime evidence in a report. +If a credential may already have been disclosed, revoke or rotate it first; +removing a file or rewriting Git history does not invalidate the credential. + +HackForger is a public, business-neutral project. Instance-specific security +facts and remediation evidence belong in the corresponding access-controlled +business repository or approved incident-management system. + +Boundary guard files (every supported `CODEOWNERS` location, `.github/workflows/**`, +`.githooks/**`, the canonical development skill, boundary wrappers, checker, +tests, and marker policy) are immutable in an ordinary pull request. A necessary +guard update requires a dedicated security-owner review and an explicit, +audited ruleset bypass; unrelated source or content changes must not share that +bypass. + +Repository rules must require the exact `Public repository boundary` commit +status, an up-to-date branch, and CODEOWNER review on both maintained release +branches. The branch/tag push workflow is only a post-disclosure audit; only the +installed pre-push hook and operator discipline can stop a first public push. +GitHub Actions workflows also share one App identity, so an organization that +does not fully trust repository writers must bind the required status to an +independent GitHub App or organization-enforced required workflow. + +Dependabot version and security updates are intentionally disabled for this +repository. Dependabot-triggered `pull_request_target` workflows receive a +read-only token and cannot publish the required candidate commit status. Before +enabling either feature, deploy and verify a trusted GitHub App or equivalent +second-stage status writer, then update this guard through the dedicated +security-owner bypass process. diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000000..7d9741fd14 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,34 @@ +# Deployment contracts + +HackForger keeps only business-neutral deployment primitives in this public +repository. Real hosts, domains, accounts, filesystem layouts, runbooks, +runtime evidence, and branded overlays belong to the corresponding private +business repository. + +For externally owned static content, use [`content/README.md`](content/README.md). +The private repository supplies a tracked configuration, a tracked content +tree, and a tracked checksum manifest; the public publisher validates and +hydrates that material without compiling or restarting HackForger. + +A business-specific release wrapper should perform the following sequence: + +1. verify both repositories are clean, pushed, and at the reviewed revisions; +2. run the public repository boundary check and application release gates; +3. back up application data using the private environment runbook; +4. publish each supported private-content mount with the generic publisher; +5. verify content hashes, public behavior, and independent deployment markers; +6. store the production report and screenshots in the private repository. + +Never copy private overlays into the public Git index. Do not use +`rsync --delete` for externally owned content, and do not treat an application +binary deployment as a substitute for content hydration. + +## Runtime overlay compatibility + +Treat a runtime template override and the application data contract it consumes +as one versioned release unit. When a public handler or template key changes, +the private overlay must be updated and staged before the new process starts; +rollback must restore both revisions together. Production templates are compiled +when the renderer initializes, so replacing the on-disk override before the +reviewed restart does not change the already-running process. Never publish a +new binary against an older, unverified private template overlay. diff --git a/deploy/content/README.md b/deploy/content/README.md new file mode 100644 index 0000000000..b8816b7271 --- /dev/null +++ b/deploy/content/README.md @@ -0,0 +1,176 @@ +# External content deployment + +This directory defines the public, business-neutral contract for publishing +runtime content that is owned by a separate private repository. It contains no +real deployment target, customer domain, infrastructure topology, or secret. + +The currently supported mount is `landing`, mapped to +`custom/public/assets/landing`. The mapping is fixed in reviewed public code; +callers cannot choose an arbitrary remote destination. + +## Private repository contract + +Keep the content tree and its manifest outside this repository, for example: + +```text +private-content/ +├── overlays/hackforger/landing/ +│ ├── index.html +│ └── assets/ +├── overlays/hackforger/SHA256SUMS +└── deploy/hackforger/production.env +``` + +Generate the manifest after every content change: + +```bash +bash /path/to/hackforger/deploy/content/generate-manifest.sh \ + overlays/hackforger/landing \ + overlays/hackforger/SHA256SUMS +``` + +The manifest contains one lowercase SHA-256 digest, two spaces, and one path +relative to the content root per line. It must exactly cover both the regular +files and the directory structure implied by those paths. Missing, extra, +duplicate, unsafe, symlinked, FIFO, socket, device, and Git LFS pointer content +fails validation. `COPYFILE_DISABLE=1` is set for archive creation and +extraction, and the finished archive is re-extracted and checked against the +manifest before upload; this prevents macOS AppleDouble members from silently +changing the release. + +## Configuration + +Copy `config.example` into the private repository and replace every example +value there. Pass the private file explicitly with `--config`. The publisher +does not accept target, URL, or root overrides on the command line and has no +production defaults. + +`REQUIRE_CLEAN_SOURCE=true` requires all of the following before even a dry run: + +- source, manifest, and deployment config are regular files in the same Git repository; +- every source file, the manifest, and config are tracked regular blobs in `HEAD`; +- every working file is byte-for-byte equal to its `HEAD` blob; +- the repository is clean, non-shallow, on a branch, and has no graft or + replacement refs; +- raw `HEAD` equals the branch tip returned by an actual + `git ls-remote --exit-code` query against explicit command-line trust + anchors; and +- no source file is a Git LFS pointer. + +For SSH publishing, `--provenance-remote` must be exactly +`git@github.com:OWNER/REPO.git`, and `--provenance-ref` must be a valid +`refs/heads/*` ref matching the checked-out branch. These anchors are CLI +arguments so a modified private config or local Git remote cannot silently +redirect verification. Local fixture mode may instead name an explicit, +absolute, canonical bare repository path. + +Raw Git commands run with replacement-object processing, environment-provided +repository overrides, global/system config, remote helpers, and the file +protocol disabled for SSH provenance. The publisher queries the branch, fetches +it into a fresh bare repository, enumerates the complete source subtree, and +materializes source, manifest, and config directly from that fetched commit. +Only regular executable or non-executable blobs are accepted. The complete +remote subtree must exactly match the manifest and the local working input, so +sparse checkout or `skip-worktree` cannot hide an extra remote file. The deploy +archive is built from the authoritative materialized source rather than the +working tree, eliminating a source-tree packaging race. The remote branch tip +is queried again under the deployment lock immediately before activation. + +SSH transport always requires this protection. Local transport may disable it +only for disposable fixture testing. `ALLOW_INITIAL_INSTALL=false` prevents an +accidental publish to a target where the live mount is absent. + +`SMOKE_ASSET` is a required, URL-safe path distinct from `index.html`. It must +be nonempty and present in the manifest. SSH activation verifies both the root +HTML and this static asset against their manifest checksums. + +The SSH target must own the configured root and provide Bash, Python 3, curl, +tar, and SHA-256 tooling. SSH configuration and credentials stay outside this +repository. + +## Publish + +The default is a read-only plan: + +```bash +bash deploy/content/publish-overlay.sh \ + --config /path/to/private/deploy/hackforger/production.env \ + --mount landing \ + --source /path/to/private/overlays/hackforger/landing \ + --manifest /path/to/private/overlays/hackforger/SHA256SUMS \ + --provenance-remote git@github.com:OWNER/PRIVATE-REPO.git \ + --provenance-ref refs/heads/main +``` + +After reviewing the target, release digest, file count, and current state, add +the explicit mutation flag: + +```bash +bash deploy/content/publish-overlay.sh ... --apply +``` + +An apply performs these steps: + +1. Revalidates exact tree coverage, checksums, inode types, remotely fetched Git + provenance, and the re-extracted authoritative archive. +2. Asks the remote transaction helper to allocate an unpredictable, deployment- + user-owned `0700` directory under `/tmp`, uploads three fixed-basename inputs, + and verifies their ownership, inode type, link count, mode and hashes before + acquiring the deployment lock. The private directory is removed after use. +3. Acquires the authoritative remote deployment lock. +4. For SSH, probes Linux `renameat2(RENAME_EXCHANGE)` on the live filesystem. + A missing Python helper, unsupported kernel/filesystem, or failed probe stops + before snapshot or live mutation. There is no non-atomic SSH fallback. +5. While still holding the lock, creates a versioned remote tar+manifest + snapshot, transfers it to persistent local storage, re-extracts it, validates + exact contents and hash, durably flushes both copies, and only then writes an + acknowledgement back under the lock. +6. Extracts the new archive into a unique directory beside live and verifies it. +7. Atomically exchanges the fixed live and staging directory names with + `RENAME_EXCHANGE`; the live path is never absent. +8. Verifies the new live tree and, over SSH, fetches both `/` and + `/assets/landing/$SMOKE_ASSET` with identity encoding and compares hashes. +9. Preserves the former live tree in the remote backup and writes the independent + `REMOTE_ROOT/.last-content-deploy` marker only after every check passes. + +Every mutating phase is recorded by atomic replacement of a fsynced `PHASE` +journal inside the versioned remote backup. Content trees, backup files, +markers, renamed directory parents, and lock creation/removal are flushed before +the corresponding phase can be acknowledged. A nonterminal journal from a +crashed transaction blocks a later lock acquisition and names the exact backup +that must be inspected. An orderly failure before activation first records the +terminal `aborted` phase and only then durably releases the lock, so a corrected +publish can retry without bypassing crash detection. + +Activation or smoke failure uses the same exchange primitive to restore the old +tree, restores the previous marker, and verifies both before reporting success. +If the exchange helper reports an error after the kernel may already have +swapped the directory names, the transaction compares both paths against the +incoming and previous manifests. A proven swap is treated as activated and is +rolled back; a proven non-swap follows pre-activation cleanup; any other state +is marked for manual recovery with the lock retained. +If any recovery action or verification fails, exit status `75` is returned, the +deployment lock is deliberately retained, and the remote backup receives a +`MANUAL_RECOVERY` file containing exact live/recovery paths. The publisher never +claims restoration in that state. + +`TRANSPORT=local` uses a clearly labelled `local-test` three-rename simulation +because macOS does not provide Linux `renameat2`. It exists only for fixtures and +is never selected for SSH/production. The publisher does not compile HackForger, +replace its binary, restart a service, or change the application's `.last-deploy` +marker. + +Run the local fixture suite with: + +```bash +bash deploy/content/tests/run.sh +``` + +The suite covers remotely fetched Git provenance, explicit trust anchors, +local-dot and replacement-ref rejection, ignored/untracked and LFS rejection, +FIFO rejection on both sides, +exact cross-platform archive contents, lock-before-backup ordering, durable +phase journals, capability-probe fail-closed behavior, post-syscall exchange +errors, verified rollback, rollback-failure lock retention, and the required +second smoke asset. It also verifies that complete authoritative subtree +enumeration defeats sparse-checkout and `skip-worktree` omissions. diff --git a/deploy/content/config.example b/deploy/content/config.example new file mode 100644 index 0000000000..1280d13e67 --- /dev/null +++ b/deploy/content/config.example @@ -0,0 +1,10 @@ +# Public example only. Keep real instance values in the corresponding private +# business repository and pass that file with --config. +TRANSPORT=ssh +DEPLOY_TARGET=deploy@example-host +PUBLIC_BASE_URL=https://code.example.com +REMOTE_ROOT=/srv/hackforger +LOCAL_BACKUP_ROOT=/absolute/local/path/to/hackforger-content-backups +SMOKE_ASSET=assets/site.css +ALLOW_INITIAL_INSTALL=false +REQUIRE_CLEAN_SOURCE=true diff --git a/deploy/content/generate-manifest.sh b/deploy/content/generate-manifest.sh new file mode 100755 index 0000000000..fc02362099 --- /dev/null +++ b/deploy/content/generate-manifest.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Generate a deterministic SHA-256 manifest for an external content tree. + +set -euo pipefail + +usage() { + echo "Usage: $0 SOURCE_DIR OUTPUT_SHA256SUMS" >&2 +} + +die() { + echo "FATAL: $*" >&2 + exit 1 +} + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +[ "$#" -eq 2 ] || { usage; exit 64; } + +SOURCE_INPUT=$1 +OUTPUT_INPUT=$2 +[ -d "$SOURCE_INPUT" ] || die "source directory does not exist: $SOURCE_INPUT" + +SOURCE=$(cd "$SOURCE_INPUT" && pwd -P) +OUTPUT_DIR=$(cd "$(dirname "$OUTPUT_INPUT")" && pwd -P) +OUTPUT="$OUTPUT_DIR/$(basename "$OUTPUT_INPUT")" + +case "$OUTPUT" in + "$SOURCE"/*) die "manifest must live outside the content source tree" ;; +esac + +special=$(find "$SOURCE" ! -type d ! -type f -print -quit) +[ -z "$special" ] || die "content tree contains a non-directory/non-regular inode: $special" +empty_directory=$(find "$SOURCE" -type d -empty -print -quit) +[ -z "$empty_directory" ] || die "content tree contains an empty directory not representable by the manifest: $empty_directory" + +WORK=$(mktemp -d "${TMPDIR:-/tmp}/content-manifest.XXXXXX") +trap 'rm -rf "$WORK"' EXIT INT TERM HUP + +( + cd "$SOURCE" + find . -type f -print | sed 's#^\./##' | LC_ALL=C sort +) > "$WORK/files" + +[ -s "$WORK/files" ] || die "content source is empty" + +: > "$WORK/SHA256SUMS" +while IFS= read -r path || [ -n "$path" ]; do + [ -n "$path" ] || die "empty path in generated file list" + if [[ "$path" =~ [[:cntrl:]] ]]; then + die "control characters are not allowed in content paths" + fi + case "$path" in + /*|./*|../*|*/../*|*/..|*/./*|*//*|*\\*) + die "unsafe content path: $path" + ;; + esac + if [ "$(sed -n '1p' "$SOURCE/$path")" = 'version https://git-lfs.github.com/spec/v1' ] \ + && grep -q '^oid sha256:[0-9a-f]\{64\}$' "$SOURCE/$path" \ + && grep -q '^size [0-9][0-9]*$' "$SOURCE/$path"; then + die "Git LFS pointer files are not deployable content: $path" + fi + printf '%s %s\n' "$(sha256_file "$SOURCE/$path")" "$path" >> "$WORK/SHA256SUMS" +done < "$WORK/files" + +mv "$WORK/SHA256SUMS" "$OUTPUT" +echo "Wrote $(wc -l < "$OUTPUT" | tr -d ' ') entries to $OUTPUT" diff --git a/deploy/content/publish-overlay.sh b/deploy/content/publish-overlay.sh new file mode 100755 index 0000000000..2f0fa516ca --- /dev/null +++ b/deploy/content/publish-overlay.sh @@ -0,0 +1,619 @@ +#!/usr/bin/env bash +# Validate and publish externally owned content without rebuilding or restarting +# HackForger. Deployment coordinates come only from an explicit config file. + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +REMOTE_HELPER="$SCRIPT_DIR/remote-transaction.sh" +EXCHANGE_HELPER="$SCRIPT_DIR/rename-exchange.py" + +usage() { + cat <<'EOF' +Usage: + publish-overlay.sh --config FILE --mount landing --source DIR --manifest FILE + [--provenance-remote REMOTE --provenance-ref refs/heads/BRANCH] [--apply] + +Default behavior is a read-only dry run. SSH apply requires a clean, pushed +source repository and Linux renameat2(RENAME_EXCHANGE). It snapshots live +content while holding the deployment lock, validates the persistent local +copy, then performs a gap-free directory exchange. There is no production +non-atomic fallback. +EOF +} + +die() { echo "FATAL: $*" >&2; exit 1; } +log() { printf '\n==> %s\n' "$*"; } + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +quote_arg() { + printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" +} + +git_sanitized() { + env \ + -u GIT_DIR -u GIT_COMMON_DIR -u GIT_WORK_TREE -u GIT_INDEX_FILE \ + -u GIT_OBJECT_DIRECTORY -u GIT_ALTERNATE_OBJECT_DIRECTORIES \ + -u GIT_NAMESPACE -u GIT_REPLACE_REF_BASE -u GIT_GRAFT_FILE -u GIT_SHALLOW_FILE \ + -u GIT_CONFIG -u GIT_CONFIG_PARAMETERS -u GIT_EXEC_PATH \ + -u GIT_CONFIG_COUNT -u GIT_CONFIG_KEY_0 -u GIT_CONFIG_VALUE_0 \ + -u GIT_SSH -u GIT_SSH_COMMAND -u GIT_SSH_VARIANT -u GIT_PROXY_COMMAND \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \ + GIT_NO_REPLACE_OBJECTS=1 GIT_LITERAL_PATHSPECS=1 GIT_OPTIONAL_LOCKS=0 \ + GIT_TERMINAL_PROMPT=0 \ + GIT_SSH_COMMAND='ssh -F /dev/null -o BatchMode=yes' \ + GIT_PROTOCOL_FROM_USER=0 git "$@" +} + +validate_inode_types() { + local tree=$1 special + special=$(find "$tree" ! -type d ! -type f -print -quit) + [ -z "$special" ] || die "content tree contains a non-directory/non-regular inode: $special" +} + +validate_manifest_path() { + local path=$1 + [ -n "$path" ] || die "empty manifest path" + if [[ "$path" =~ [[:cntrl:]] ]]; then + die "control characters are not allowed in manifest paths" + fi + case "$path" in + /*|./*|../*|*/../*|*/..|*/./*|*//*|*\\*) die "unsafe manifest path: $path" ;; + esac +} + +verify_tree() { + local tree=$1 manifest=$2 entrypoint=$3 work=$4 + local line hash separator path duplicate directory parent + validate_inode_types "$tree" + [ -f "$manifest" ] && [ ! -L "$manifest" ] || die "manifest is not a regular file" + : > "$work/expected" + while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] || die "manifest contains a blank line" + [ "${#line}" -ge 67 ] || die "malformed manifest line" + hash=${line:0:64} + separator=${line:64:2} + path=${line:66} + [[ "$hash" =~ ^[0-9a-f]{64}$ ]] || die "invalid manifest digest" + [ "$separator" = " " ] || die "manifest requires two spaces before each path" + validate_manifest_path "$path" + [ -f "$tree/$path" ] && [ ! -L "$tree/$path" ] || die "manifest file is not regular: $path" + [ "$(sha256_file "$tree/$path")" = "$hash" ] || die "checksum mismatch: $path" + printf '%s\n' "$path" >> "$work/expected" + done < "$manifest" + [ -s "$work/expected" ] || die "manifest is empty" + [ -s "$tree/$entrypoint" ] || die "entrypoint is missing or empty" + LC_ALL=C sort "$work/expected" > "$work/expected.sorted" + duplicate=$(uniq -d "$work/expected.sorted" | head -1 || true) + [ -z "$duplicate" ] || die "duplicate manifest path: $duplicate" + ( + cd "$tree" + find . -type f -print | sed 's#^\./##' | LC_ALL=C sort + ) > "$work/actual.sorted" + if ! cmp -s "$work/expected.sorted" "$work/actual.sorted"; then + echo "FATAL: manifest does not exactly cover the content tree" >&2 + diff -u "$work/expected.sorted" "$work/actual.sorted" >&2 || true + exit 1 + fi + : > "$work/expected-dirs" + printf '.\n' >> "$work/expected-dirs" + while IFS= read -r path || [ -n "$path" ]; do + directory=$(dirname "$path") + while [ "$directory" != . ]; do + printf '%s\n' "$directory" >> "$work/expected-dirs" + parent=$(dirname "$directory") + [ "$parent" != "$directory" ] || die "could not normalize manifest directory: $directory" + directory=$parent + done + done < "$work/expected.sorted" + LC_ALL=C sort -u "$work/expected-dirs" > "$work/expected-dirs.sorted" + ( + cd "$tree" + find . -type d -print | sed 's#^\./##' | LC_ALL=C sort + ) > "$work/actual-dirs.sorted" + if ! cmp -s "$work/expected-dirs.sorted" "$work/actual-dirs.sorted"; then + echo "FATAL: archive/source contains directories not implied by the manifest" >&2 + diff -u "$work/expected-dirs.sorted" "$work/actual-dirs.sorted" >&2 || true + exit 1 + fi +} + +is_lfs_pointer() { + local file=$1 + [ "$(sed -n '1p' "$file")" = 'version https://git-lfs.github.com/spec/v1' ] \ + && grep -q '^oid sha256:[0-9a-f]\{64\}$' "$file" \ + && grep -q '^size [0-9][0-9]*$' "$file" +} + +CONFIG="" +MOUNT="" +SOURCE_INPUT="" +MANIFEST_INPUT="" +PROVENANCE_REMOTE="" +PROVENANCE_REF="" +APPLY=0 +while [ "$#" -gt 0 ]; do + case "$1" in + --config) [ "$#" -ge 2 ] || die "--config needs a value"; CONFIG=$2; shift 2 ;; + --mount) [ "$#" -ge 2 ] || die "--mount needs a value"; MOUNT=$2; shift 2 ;; + --source) [ "$#" -ge 2 ] || die "--source needs a value"; SOURCE_INPUT=$2; shift 2 ;; + --manifest) [ "$#" -ge 2 ] || die "--manifest needs a value"; MANIFEST_INPUT=$2; shift 2 ;; + --provenance-remote) [ "$#" -ge 2 ] || die "--provenance-remote needs a value"; PROVENANCE_REMOTE=$2; shift 2 ;; + --provenance-ref) [ "$#" -ge 2 ] || die "--provenance-ref needs a value"; PROVENANCE_REF=$2; shift 2 ;; + --apply) APPLY=1; shift ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; die "unknown argument: $1" ;; + esac +done + +[ -n "$CONFIG" ] && [ -f "$CONFIG" ] && [ ! -L "$CONFIG" ] \ + || die "--config must name a regular non-symlink file" +[ -n "$MOUNT" ] || die "--mount is required" +[ -n "$SOURCE_INPUT" ] && [ -d "$SOURCE_INPUT" ] || die "--source must name an existing directory" +[ -n "$MANIFEST_INPUT" ] && [ -f "$MANIFEST_INPUT" ] && [ ! -L "$MANIFEST_INPUT" ] \ + || die "--manifest must name a regular non-symlink file" + +CONFIG_DIR=$(cd "$(dirname "$CONFIG")" && pwd -P) +CONFIG="$CONFIG_DIR/$(basename "$CONFIG")" +WORK=$(mktemp -d "${TMPDIR:-/tmp}/publish-content.XXXXXX") +WORK=$(cd "$WORK" && pwd -P) +trap 'rm -rf "$WORK"' EXIT INT TERM HUP +CONFIG_SNAPSHOT="$WORK/config.input" +cp "$CONFIG" "$CONFIG_SNAPSHOT" +CONFIG_READ_HASH=$(sha256_file "$CONFIG_SNAPSHOT") + +TRANSPORT="" +DEPLOY_TARGET="" +PUBLIC_BASE_URL="" +REMOTE_ROOT="" +LOCAL_BACKUP_ROOT="" +SMOKE_ASSET="" +ALLOW_INITIAL_INSTALL=false +REQUIRE_CLEAN_SOURCE=true +line_number=0 +while IFS= read -r line || [ -n "$line" ]; do + line_number=$((line_number + 1)) + case "$line" in ''|'#'*) continue ;; esac + key=${line%%=*} + [ "$key" != "$line" ] || die "config line $line_number is not KEY=value" + value=${line#*=} + [ -n "$value" ] || die "config key $key cannot be empty" + case "$key" in + TRANSPORT) TRANSPORT=$value ;; + DEPLOY_TARGET) DEPLOY_TARGET=$value ;; + PUBLIC_BASE_URL) PUBLIC_BASE_URL=$value ;; + REMOTE_ROOT) REMOTE_ROOT=$value ;; + LOCAL_BACKUP_ROOT) LOCAL_BACKUP_ROOT=$value ;; + SMOKE_ASSET) SMOKE_ASSET=$value ;; + ALLOW_INITIAL_INSTALL) ALLOW_INITIAL_INSTALL=$value ;; + REQUIRE_CLEAN_SOURCE) REQUIRE_CLEAN_SOURCE=$value ;; + *) die "unknown config key on line $line_number: $key" ;; + esac +done < "$CONFIG_SNAPSHOT" + +[ "$TRANSPORT" = ssh ] || [ "$TRANSPORT" = local ] || die "TRANSPORT must be ssh or local" +[ -n "$DEPLOY_TARGET" ] || die "DEPLOY_TARGET is required in --config" +[ -n "$PUBLIC_BASE_URL" ] || die "PUBLIC_BASE_URL is required in --config" +[ -n "$REMOTE_ROOT" ] || die "REMOTE_ROOT is required in --config" +[ -n "$LOCAL_BACKUP_ROOT" ] || die "LOCAL_BACKUP_ROOT is required in --config" +[ -n "$SMOKE_ASSET" ] || die "SMOKE_ASSET is required in --config" +[ "$ALLOW_INITIAL_INSTALL" = true ] || [ "$ALLOW_INITIAL_INSTALL" = false ] || die "ALLOW_INITIAL_INSTALL must be true or false" +[ "$REQUIRE_CLEAN_SOURCE" = true ] || [ "$REQUIRE_CLEAN_SOURCE" = false ] || die "REQUIRE_CLEAN_SOURCE must be true or false" +[[ "$DEPLOY_TARGET" =~ ^[A-Za-z0-9_.@:-]+$ ]] || die "DEPLOY_TARGET contains unsupported characters" +[[ "$PUBLIC_BASE_URL" =~ ^https?://[A-Za-z0-9._:-]+/?$ ]] || die "PUBLIC_BASE_URL must be an http(s) origin without a path" +[[ "$SMOKE_ASSET" =~ ^[A-Za-z0-9._/-]+$ ]] || die "SMOKE_ASSET must be URL-safe without encoding" +validate_manifest_path "$SMOKE_ASSET" +[ "$SMOKE_ASSET" != index.html ] || die "SMOKE_ASSET must be distinct from index.html" +case "$REMOTE_ROOT" in /*) ;; *) die "REMOTE_ROOT must be absolute" ;; esac +case "$LOCAL_BACKUP_ROOT" in /*) ;; *) die "LOCAL_BACKUP_ROOT must be absolute" ;; esac +[ "$REMOTE_ROOT" != / ] && [ "$LOCAL_BACKUP_ROOT" != / ] || die "configured roots cannot be /" +case "$REMOTE_ROOT$LOCAL_BACKUP_ROOT" in + *"'"*|*'..'*|*$'\t'*|*$'\r'*|*$'\n'*) die "configured roots contain unsafe characters" ;; +esac +if [ "$TRANSPORT" = ssh ] && [ "$REQUIRE_CLEAN_SOURCE" != true ]; then + die "ssh deployments require REQUIRE_CLEAN_SOURCE=true" +fi +if [ "$REQUIRE_CLEAN_SOURCE" = true ]; then + [ -n "$PROVENANCE_REMOTE" ] && [ -n "$PROVENANCE_REF" ] \ + || die "clean-source publishing requires --provenance-remote and --provenance-ref trust anchors" +elif [ -n "$PROVENANCE_REMOTE" ] || [ -n "$PROVENANCE_REF" ]; then + die "provenance trust anchors require REQUIRE_CLEAN_SOURCE=true" +fi + +case "$MOUNT" in + landing) MOUNT_PATH=custom/public/assets/landing; ENTRYPOINT=index.html ;; + *) die "unsupported mount: $MOUNT" ;; +esac + +SOURCE=$(cd "$SOURCE_INPUT" && pwd -P) +MANIFEST_DIR=$(cd "$(dirname "$MANIFEST_INPUT")" && pwd -P) +MANIFEST="$MANIFEST_DIR/$(basename "$MANIFEST_INPUT")" +MANIFEST_SNAPSHOT="$WORK/manifest.input" +cp "$MANIFEST" "$MANIFEST_SNAPSHOT" +MANIFEST_READ_HASH=$(sha256_file "$MANIFEST_SNAPSHOT") +case "$MANIFEST" in "$SOURCE"/*) die "manifest must live outside the content source tree" ;; esac +validate_inode_types "$SOURCE" + +REMOTE_ARCHIVE="" +REMOTE_MANIFEST="" +REMOTE_EXCHANGE="" +REMOTE_UPLOAD_DIR="" +LOCK_HELD=0 +LOCK_TOKEN="" +BACKUP_ID="" + +run_helper() { + if [ "$TRANSPORT" = local ]; then + bash "$REMOTE_HELPER" "$@" + return + fi + command="bash -s --" + for arg in "$@"; do command="$command $(quote_arg "$arg")"; done + ssh "$DEPLOY_TARGET" "$command" < "$REMOTE_HELPER" +} + +cleanup() { + rc=$? + trap - EXIT INT TERM HUP + set +e + if [ "$LOCK_HELD" -eq 1 ] && [ -n "$LOCK_TOKEN" ]; then + if [ -n "$BACKUP_ID" ]; then + run_helper abort "$REMOTE_ROOT" "$LOCK_TOKEN" "$BACKUP_ID" \ + "$APPLY_EXCHANGE" "$EXCHANGE_HASH" publisher-preactivation-failure >/dev/null 2>&1 + else + run_helper lock-release "$REMOTE_ROOT" "$LOCK_TOKEN" "$APPLY_EXCHANGE" "$EXCHANGE_HASH" >/dev/null 2>&1 + fi || \ + echo "WARN: deployment lock could not be released: $REMOTE_ROOT/.content-deploy.lock" >&2 + fi + if [ "$TRANSPORT" = ssh ]; then + [ -n "$REMOTE_UPLOAD_DIR" ] \ + && run_helper upload-clean "$REMOTE_UPLOAD_DIR" >/dev/null 2>&1 || true + fi + rm -rf "$WORK" + exit "$rc" +} +trap cleanup EXIT INT TERM HUP + +: > "$WORK/normalized.SHA256SUMS" +: > "$WORK/source-paths" +ENTRYPOINT_HASH="" +SMOKE_ASSET_HASH="" +while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] && [ "${#line}" -ge 67 ] || die "malformed or blank manifest line" + hash=$(printf '%s' "${line:0:64}" | tr A-F a-f) + separator=${line:64:2} + path=${line:66} + [[ "$hash" =~ ^[0-9a-f]{64}$ ]] || die "invalid manifest digest" + [ "$separator" = " " ] || die "manifest requires two spaces before each path" + validate_manifest_path "$path" + [ -f "$SOURCE/$path" ] && [ ! -L "$SOURCE/$path" ] || die "manifest file is not regular: $path" + [ "$(sha256_file "$SOURCE/$path")" = "$hash" ] || die "checksum mismatch: $path" + is_lfs_pointer "$SOURCE/$path" && die "Git LFS pointer files are not deployable content: $path" + printf '%s\n' "$path" >> "$WORK/source-paths" + printf '%s %s\n' "$hash" "$path" >> "$WORK/normalized.SHA256SUMS" + [ "$path" = "$ENTRYPOINT" ] && ENTRYPOINT_HASH=$hash + [ "$path" = "$SMOKE_ASSET" ] && SMOKE_ASSET_HASH=$hash +done < "$MANIFEST_SNAPSHOT" + +[ -n "$ENTRYPOINT_HASH" ] && [ -s "$SOURCE/$ENTRYPOINT" ] || die "manifest entrypoint is missing or empty" +[ -n "$SMOKE_ASSET_HASH" ] && [ -s "$SOURCE/$SMOKE_ASSET" ] || die "configured SMOKE_ASSET is missing or empty" +VERIFY_SOURCE="$WORK/verify-source" +mkdir "$VERIFY_SOURCE" +verify_tree "$SOURCE" "$WORK/normalized.SHA256SUMS" "$ENTRYPOINT" "$VERIFY_SOURCE" + +SOURCE_REVISION=unversioned +if [ "$REQUIRE_CLEAN_SOURCE" = true ]; then + [ -z "${GIT_REPLACE_REF_BASE:-}" ] && [ -z "${GIT_GRAFT_FILE:-}" ] \ + || die "replace/graft environment overrides are forbidden" + SOURCE_REPO=$(git_sanitized -C "$SOURCE" rev-parse --show-toplevel 2>/dev/null) || die "source is not inside a git repository" + MANIFEST_REPO=$(git_sanitized -C "$MANIFEST_DIR" rev-parse --show-toplevel 2>/dev/null) || die "manifest is not inside a git repository" + CONFIG_REPO=$(git_sanitized -C "$CONFIG_DIR" rev-parse --show-toplevel 2>/dev/null) || die "config is not inside a git repository" + SOURCE_REPO=$(cd "$SOURCE_REPO" && pwd -P) + MANIFEST_REPO=$(cd "$MANIFEST_REPO" && pwd -P) + CONFIG_REPO=$(cd "$CONFIG_REPO" && pwd -P) + [ "$SOURCE_REPO" = "$MANIFEST_REPO" ] && [ "$SOURCE_REPO" = "$CONFIG_REPO" ] \ + || die "source, manifest, and config must belong to the same git repository" + COMMON_DIR=$(git_sanitized -C "$SOURCE_REPO" rev-parse --git-common-dir) + case "$COMMON_DIR" in /*) ;; *) COMMON_DIR="$SOURCE_REPO/$COMMON_DIR" ;; esac + COMMON_DIR=$(cd "$COMMON_DIR" && pwd -P) + [ ! -e "$COMMON_DIR/info/grafts" ] && [ ! -L "$COMMON_DIR/info/grafts" ] \ + || die "legacy Git grafts are forbidden" + [ -z "$(git_sanitized -C "$SOURCE_REPO" for-each-ref --format='%(refname)' refs/replace)" ] || die "Git replace refs are forbidden" + [ "$(git_sanitized -C "$SOURCE_REPO" rev-parse --is-shallow-repository)" = false ] || die "shallow source repositories are forbidden" + [ -z "$(git_sanitized -C "$SOURCE_REPO" status --porcelain --untracked-files=all)" ] || die "source repository is not clean" + BRANCH=$(git_sanitized -C "$SOURCE_REPO" symbolic-ref -q --short HEAD) || die "source repository must be on a branch" + + [[ "$PROVENANCE_REF" =~ ^refs/heads/[A-Za-z0-9._/-]+$ ]] \ + && git_sanitized check-ref-format "$PROVENANCE_REF" >/dev/null 2>&1 \ + || die "--provenance-ref must be a valid refs/heads/* branch" + [ "$PROVENANCE_REF" = "refs/heads/$BRANCH" ] \ + || die "checked-out branch does not match --provenance-ref" + if [ "$TRANSPORT" = ssh ]; then + [[ "$PROVENANCE_REMOTE" =~ ^git@github\.com:[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\.git$ ]] \ + || die "SSH provenance remote must be exactly git@github.com:OWNER/REPO.git" + PROVENANCE_FILE_POLICY=never + else + case "$PROVENANCE_REMOTE" in /*) ;; *) die "local-test provenance remote must be an absolute bare repository path" ;; esac + [ -d "$PROVENANCE_REMOTE" ] && [ ! -L "$PROVENANCE_REMOTE" ] \ + || die "local-test provenance remote must be a real bare repository directory" + PROVENANCE_REMOTE_REAL=$(cd "$PROVENANCE_REMOTE" && pwd -P) + [ "$PROVENANCE_REMOTE_REAL" = "$PROVENANCE_REMOTE" ] \ + || die "local-test provenance remote must be canonical" + [ "$(git_sanitized -C "$PROVENANCE_REMOTE" rev-parse --is-bare-repository 2>/dev/null)" = true ] \ + || die "local-test provenance remote is not bare" + PROVENANCE_FILE_POLICY=always + fi + + PROVENANCE_CWD="$WORK/provenance-cwd" + mkdir "$PROVENANCE_CWD" + + git_provenance() { + git_sanitized -C "$PROVENANCE_CWD" -c protocol.ext.allow=never \ + -c "protocol.file.allow=$PROVENANCE_FILE_POLICY" \ + -c fetch.fsckObjects=true -c transfer.fsckObjects=true "$@" + } + + query_provenance_tip() { + local result tip + result=$(git_provenance ls-remote --refs --exit-code "$PROVENANCE_REMOTE" "$PROVENANCE_REF") \ + || die "could not query the provenance branch tip" + tip=$(printf '%s\n' "$result" | awk -v ref="$PROVENANCE_REF" '$2 == ref {print $1}') + [[ "$tip" =~ ^([0-9a-f]{40}|[0-9a-f]{64})$ ]] \ + || die "provenance remote returned an invalid branch tip" + printf '%s\n' "$tip" + } + + REMOTE_TIP=$(query_provenance_tip) + SOURCE_REVISION=$(git_sanitized -C "$SOURCE_REPO" rev-parse --verify 'HEAD^{commit}') + [ "$SOURCE_REVISION" = "$REMOTE_TIP" ] \ + || die "source HEAD is not equal to the provenance branch tip" + + TRUST_REPO="$WORK/provenance.git" + EMPTY_TEMPLATE="$WORK/empty-git-template" + mkdir "$EMPTY_TEMPLATE" + git_sanitized -C "$PROVENANCE_CWD" init -q --bare --template="$EMPTY_TEMPLATE" "$TRUST_REPO" + git_provenance --git-dir "$TRUST_REPO" fetch -q --no-tags --force \ + "$PROVENANCE_REMOTE" "+$PROVENANCE_REF:refs/provenance/source" + FETCHED_REVISION=$(git_sanitized --git-dir "$TRUST_REPO" rev-parse --verify 'refs/provenance/source^{commit}') + [ "$FETCHED_REVISION" = "$SOURCE_REVISION" ] \ + || die "provenance branch changed while the authoritative tree was fetched" + [ "$(git_sanitized --git-dir "$TRUST_REPO" rev-parse --is-shallow-repository)" = false ] \ + || die "authoritative provenance fetch unexpectedly produced a shallow repository" + [ ! -s "$TRUST_REPO/info/grafts" ] || die "authoritative provenance repository contains grafts" + [ -z "$(git_sanitized --git-dir "$TRUST_REPO" for-each-ref --format='%(refname)' refs/replace)" ] \ + || die "authoritative provenance repository contains replacement refs" + git_sanitized --git-dir "$TRUST_REPO" fsck --full --strict --no-reflogs "$FETCHED_REVISION" >/dev/null + + AUTHORITATIVE_ROOT="$WORK/authoritative" + AUTHORITATIVE_SOURCE="$AUTHORITATIVE_ROOT/source" + mkdir -p "$AUTHORITATIVE_SOURCE" + + materialize_authoritative_file() { + local working_file=$1 repository_path=$2 output=$3 mode + mode=$(git_sanitized --git-dir "$TRUST_REPO" ls-tree "$SOURCE_REVISION" -- "$repository_path" \ + | awk 'NR == 1 {print $1}') + case "$mode" in 100644|100755) ;; *) die "provenance commit lacks a regular file: $repository_path" ;; esac + mkdir -p "$(dirname "$output")" + git_sanitized --git-dir "$TRUST_REPO" cat-file blob "$SOURCE_REVISION:$repository_path" > "$output" + cmp -s "$output" "$working_file" || die "working file differs from provenance commit: $repository_path" + } + + case "$SOURCE" in "$SOURCE_REPO"/*) SOURCE_REL=${SOURCE#"$SOURCE_REPO"/} ;; *) die "source is outside its repository" ;; esac + case "$MANIFEST" in "$SOURCE_REPO"/*) MANIFEST_REL=${MANIFEST#"$SOURCE_REPO"/} ;; *) die "manifest is outside its repository" ;; esac + case "$CONFIG" in "$SOURCE_REPO"/*) CONFIG_REL=${CONFIG#"$SOURCE_REPO"/} ;; *) die "config is outside its repository" ;; esac + materialize_authoritative_file "$CONFIG_SNAPSHOT" "$CONFIG_REL" "$AUTHORITATIVE_ROOT/config" + materialize_authoritative_file "$MANIFEST_SNAPSHOT" "$MANIFEST_REL" "$AUTHORITATIVE_ROOT/manifest" + [ "$(sha256_file "$AUTHORITATIVE_ROOT/config")" = "$CONFIG_READ_HASH" ] \ + || die "config changed while it was being read" + [ "$(sha256_file "$AUTHORITATIVE_ROOT/manifest")" = "$MANIFEST_READ_HASH" ] \ + || die "manifest changed while it was being read" + AUTHORITATIVE_TREE="$WORK/authoritative-tree" + git_sanitized --git-dir "$TRUST_REPO" ls-tree -r -z "$SOURCE_REVISION" -- "$SOURCE_REL" \ + > "$AUTHORITATIVE_TREE" + : > "$WORK/authoritative-source-paths" + while IFS= read -r -d '' tree_entry; do + tree_metadata=${tree_entry%%$'\t'*} + repository_path=${tree_entry#*$'\t'} + read -r tree_mode tree_type tree_oid <<< "$tree_metadata" + case "$repository_path" in + "$SOURCE_REL"/*) path=${repository_path#"$SOURCE_REL"/} ;; + *) die "provenance tree contains a path outside the source subtree" ;; + esac + case "$tree_mode:$tree_type" in + 100644:blob|100755:blob) ;; + *) die "provenance source contains a non-regular Git entry: $repository_path" ;; + esac + validate_manifest_path "$path" + [ -f "$SOURCE/$path" ] && [ ! -L "$SOURCE/$path" ] \ + || die "working source lacks a provenance file: $repository_path" + mkdir -p "$(dirname "$AUTHORITATIVE_SOURCE/$path")" + git_sanitized --git-dir "$TRUST_REPO" cat-file blob "$tree_oid" \ + > "$AUTHORITATIVE_SOURCE/$path" + cmp -s "$AUTHORITATIVE_SOURCE/$path" "$SOURCE/$path" \ + || die "working source differs from provenance commit: $repository_path" + printf '%s\n' "$path" >> "$WORK/authoritative-source-paths" + done < "$AUTHORITATIVE_TREE" + [ -s "$WORK/authoritative-source-paths" ] || die "provenance source subtree is empty" + VERIFY_AUTHORITATIVE="$WORK/verify-authoritative" + mkdir "$VERIFY_AUTHORITATIVE" + verify_tree "$AUTHORITATIVE_SOURCE" "$WORK/normalized.SHA256SUMS" "$ENTRYPOINT" "$VERIFY_AUTHORITATIVE" + SOURCE=$AUTHORITATIVE_SOURCE +else + if git -C "$SOURCE" rev-parse HEAD >/dev/null 2>&1; then + SOURCE_REVISION=$(git -C "$SOURCE" rev-parse HEAD) + fi +fi + +MANIFEST_HASH=$(sha256_file "$WORK/normalized.SHA256SUMS") +RELEASE_ID=$MANIFEST_HASH +FILE_COUNT=$(wc -l < "$WORK/source-paths" | tr -d ' ') + +log "Validating deployment target" +INSPECTION=$(run_helper inspect "$REMOTE_ROOT" "$MOUNT_PATH" "$ENTRYPOINT") +printf '%s\n' "$INSPECTION" | sed 's/^/ /' +LIVE_EXISTS=$(printf '%s\n' "$INSPECTION" | sed -n 's/^live_exists=//p') +[ "$LIVE_EXISTS" = 1 ] || [ "$LIVE_EXISTS" = 0 ] || die "target inspection returned invalid state" +[ "$LIVE_EXISTS" = 1 ] || [ "$ALLOW_INITIAL_INSTALL" = true ] || die "live content is absent and initial installation is disabled" + +ACTIVATION_MODE=local-test +[ "$TRANSPORT" = ssh ] && ACTIVATION_MODE=exchange +echo +echo "Content deployment plan" +echo " transport: $TRANSPORT" +echo " activation: $ACTIVATION_MODE" +echo " target: $DEPLOY_TARGET" +echo " public URL: ${PUBLIC_BASE_URL%/}/" +echo " remote root: $REMOTE_ROOT" +echo " mount: $MOUNT_PATH" +echo " smoke asset: $SMOKE_ASSET" +echo " source revision: $SOURCE_REVISION" +echo " release id: $RELEASE_ID" +echo " files: $FILE_COUNT" +echo " local backups: $LOCAL_BACKUP_ROOT" +if [ "$APPLY" -eq 0 ]; then + echo + echo "DRY RUN: no backup, lock, staging, live, or marker files were changed." + exit 0 +fi + +log "Building and re-extracting immutable archive" +ARCHIVE="$WORK/content.tar" +COPYFILE_DISABLE=1 tar -cf "$ARCHIVE" -C "$SOURCE" . +ARCHIVE_CHECK="$WORK/archive-check" +mkdir "$ARCHIVE_CHECK" +COPYFILE_DISABLE=1 tar -xf "$ARCHIVE" -C "$ARCHIVE_CHECK" +VERIFY_ARCHIVE="$WORK/verify-archive" +mkdir "$VERIFY_ARCHIVE" +verify_tree "$ARCHIVE_CHECK" "$WORK/normalized.SHA256SUMS" "$ENTRYPOINT" "$VERIFY_ARCHIVE" +ARCHIVE_HASH=$(sha256_file "$ARCHIVE") +EXCHANGE_HASH=$(sha256_file "$EXCHANGE_HELPER") +BACKUP_ID="$(date -u +%Y%m%dT%H%M%SZ)-${RELEASE_ID:0:12}-$$" +LOCK_TOKEN=$BACKUP_ID + +if [ "$TRANSPORT" = ssh ]; then + REMOTE_UPLOAD_DIR=$(run_helper upload-create) + [[ "$REMOTE_UPLOAD_DIR" =~ ^/tmp/hackforger-content-upload\.[A-Za-z0-9]+$ ]] \ + || die "remote upload allocator returned an unsafe path" + REMOTE_ARCHIVE="$REMOTE_UPLOAD_DIR/content.tar" + REMOTE_MANIFEST="$REMOTE_UPLOAD_DIR/manifest.SHA256SUMS" + REMOTE_EXCHANGE="$REMOTE_UPLOAD_DIR/rename-exchange.py" + scp -q "$ARCHIVE" "$DEPLOY_TARGET:$REMOTE_ARCHIVE" + scp -q "$WORK/normalized.SHA256SUMS" "$DEPLOY_TARGET:$REMOTE_MANIFEST" + scp -q "$EXCHANGE_HELPER" "$DEPLOY_TARGET:$REMOTE_EXCHANGE" + run_helper upload-verify "$REMOTE_UPLOAD_DIR" \ + "$ARCHIVE_HASH" "$MANIFEST_HASH" "$EXCHANGE_HASH" + APPLY_ARCHIVE=$REMOTE_ARCHIVE + APPLY_MANIFEST=$REMOTE_MANIFEST + APPLY_EXCHANGE=$REMOTE_EXCHANGE + SMOKE_MODE=http +else + APPLY_ARCHIVE=$ARCHIVE + APPLY_MANIFEST="$WORK/normalized.SHA256SUMS" + APPLY_EXCHANGE=$EXCHANGE_HELPER + SMOKE_MODE=local +fi + +log "Acquiring deployment lock and probing activation capability" +run_helper lock-acquire "$REMOTE_ROOT" "$MOUNT_PATH" "$ENTRYPOINT" "$LOCK_TOKEN" \ + "$ACTIVATION_MODE" "$APPLY_EXCHANGE" "$EXCHANGE_HASH" "$ALLOW_INITIAL_INSTALL" +LOCK_HELD=1 + +log "Creating authoritative snapshot while lock is held" +SNAPSHOT=$(run_helper snapshot "$REMOTE_ROOT" "$MOUNT_PATH" "$ENTRYPOINT" \ + "$LOCK_TOKEN" "$BACKUP_ID" "$ALLOW_INITIAL_INSTALL" "$APPLY_EXCHANGE" "$EXCHANGE_HASH") +printf '%s\n' "$SNAPSHOT" | sed 's/^/ /' +SNAPSHOT_LIVE=$(printf '%s\n' "$SNAPSHOT" | sed -n 's/^live_exists=//p') +SNAPSHOT_ARCHIVE_HASH=$(printf '%s\n' "$SNAPSHOT" | sed -n 's/^backup_archive_sha256=//p') +SNAPSHOT_MANIFEST_HASH=$(printf '%s\n' "$SNAPSHOT" | sed -n 's/^previous_manifest_sha256=//p') +SNAPSHOT_MARKER_HASH=$(printf '%s\n' "$SNAPSHOT" | sed -n 's/^previous_marker_sha256=//p') +[ "$SNAPSHOT_LIVE" = 1 ] || [ "$SNAPSHOT_LIVE" = 0 ] || die "snapshot returned invalid live state" +REMOTE_BACKUP="$REMOTE_ROOT/.content-backups/$BACKUP_ID" +LOCAL_BACKUP="$LOCAL_BACKUP_ROOT/$BACKUP_ID" +mkdir -p "$LOCAL_BACKUP_ROOT" +[ -d "$LOCAL_BACKUP_ROOT" ] && [ ! -L "$LOCAL_BACKUP_ROOT" ] || die "LOCAL_BACKUP_ROOT is not a real directory" +LOCAL_BACKUP_PARENT=$(dirname "$LOCAL_BACKUP_ROOT") +[ -d "$LOCAL_BACKUP_PARENT" ] && [ ! -L "$LOCAL_BACKUP_PARENT" ] || die "LOCAL_BACKUP_ROOT parent is not a real directory" +[ ! -e "$LOCAL_BACKUP" ] && [ ! -L "$LOCAL_BACKUP" ] || die "local backup id already exists" +mkdir "$LOCAL_BACKUP" +printf 'version=1\ntarget=%s\nmount=%s\nnew_release_id=%s\nsource_revision=%s\ncreated_at=%s\n' \ + "$DEPLOY_TARGET" "$MOUNT_PATH" "$RELEASE_ID" "$SOURCE_REVISION" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$LOCAL_BACKUP/metadata" + +if [ "$SNAPSHOT_LIVE" = 1 ]; then + if [ "$TRANSPORT" = local ]; then + cp "$REMOTE_BACKUP/live.tar" "$LOCAL_BACKUP/live.tar" + cp "$REMOTE_BACKUP/previous.SHA256SUMS" "$LOCAL_BACKUP/previous.SHA256SUMS" + else + scp -q "$DEPLOY_TARGET:$REMOTE_BACKUP/live.tar" "$LOCAL_BACKUP/live.tar" + scp -q "$DEPLOY_TARGET:$REMOTE_BACKUP/previous.SHA256SUMS" "$LOCAL_BACKUP/previous.SHA256SUMS" + fi + [ "$(sha256_file "$LOCAL_BACKUP/live.tar")" = "$SNAPSHOT_ARCHIVE_HASH" ] || die "persistent local backup hash differs from locked snapshot" + [ "$(sha256_file "$LOCAL_BACKUP/previous.SHA256SUMS")" = "$SNAPSHOT_MANIFEST_HASH" ] || die "persistent local manifest hash differs from locked snapshot" + LOCAL_OLD_CHECK="$WORK/local-old-check" + mkdir "$LOCAL_OLD_CHECK" + COPYFILE_DISABLE=1 tar -xf "$LOCAL_BACKUP/live.tar" -C "$LOCAL_OLD_CHECK" + VERIFY_OLD="$WORK/verify-old" + mkdir "$VERIFY_OLD" + verify_tree "$LOCAL_OLD_CHECK/landing" "$LOCAL_BACKUP/previous.SHA256SUMS" "$ENTRYPOINT" "$VERIFY_OLD" + printf '%s %s\n' "$SNAPSHOT_ARCHIVE_HASH" live.tar > "$LOCAL_BACKUP/SHA256SUMS" +else + SNAPSHOT_ARCHIVE_HASH=absent + SNAPSHOT_MANIFEST_HASH=absent + : > "$LOCAL_BACKUP/live-was-absent" +fi + +if [ "$TRANSPORT" = local ]; then + if [ -f "$REMOTE_BACKUP/previous-marker" ]; then + cp "$REMOTE_BACKUP/previous-marker" "$LOCAL_BACKUP/previous-marker" + [ "$(sha256_file "$LOCAL_BACKUP/previous-marker")" = "$SNAPSHOT_MARKER_HASH" ] || die "persistent local marker hash differs from locked snapshot" + else + [ "$SNAPSHOT_MARKER_HASH" = absent ] || die "locked snapshot marker disappeared" + : > "$LOCAL_BACKUP/marker-was-absent" + fi +else + if ssh "$DEPLOY_TARGET" "test -f '$REMOTE_BACKUP/previous-marker'"; then + scp -q "$DEPLOY_TARGET:$REMOTE_BACKUP/previous-marker" "$LOCAL_BACKUP/previous-marker" + [ "$(sha256_file "$LOCAL_BACKUP/previous-marker")" = "$SNAPSHOT_MARKER_HASH" ] || die "persistent local marker hash differs from locked snapshot" + else + [ "$SNAPSHOT_MARKER_HASH" = absent ] || die "locked snapshot marker disappeared" + : > "$LOCAL_BACKUP/marker-was-absent" + fi +fi + +python3 "$EXCHANGE_HELPER" fsync-tree "$LOCAL_BACKUP" +python3 "$EXCHANGE_HELPER" fsync "$LOCAL_BACKUP" "$LOCAL_BACKUP_ROOT" "$LOCAL_BACKUP_PARENT" + +run_helper backup-ack "$REMOTE_ROOT" "$LOCK_TOKEN" "$BACKUP_ID" \ + "$SNAPSHOT_ARCHIVE_HASH" "$SNAPSHOT_MANIFEST_HASH" "$SNAPSHOT_MARKER_HASH" \ + "$APPLY_EXCHANGE" "$EXCHANGE_HASH" + +if [ "$REQUIRE_CLEAN_SOURCE" = true ]; then + FINAL_REMOTE_TIP=$(query_provenance_tip) + [ "$FINAL_REMOTE_TIP" = "$SOURCE_REVISION" ] \ + || die "provenance branch changed before activation; refusing the stale release" +fi + +log "Activating verified release" +# From here the remote transaction owns lock release. A rollback failure keeps +# the lock and writes MANUAL_RECOVERY; the outer cleanup must not remove it. +LOCK_HELD=0 +run_helper apply \ + "$REMOTE_ROOT" "$MOUNT_PATH" "$ENTRYPOINT" \ + "$APPLY_ARCHIVE" "$APPLY_MANIFEST" \ + "$ARCHIVE_HASH" "$MANIFEST_HASH" "$ENTRYPOINT_HASH" \ + "$SMOKE_ASSET" "$SMOKE_ASSET_HASH" \ + "$RELEASE_ID" "$SOURCE_REVISION" "$BACKUP_ID" "${PUBLIC_BASE_URL%/}" \ + "$SMOKE_MODE" "$ALLOW_INITIAL_INSTALL" "$LOCK_TOKEN" "$ACTIVATION_MODE" \ + "$APPLY_EXCHANGE" "$EXCHANGE_HASH" + +echo +echo "Published content release $RELEASE_ID" +echo " local backup: $LOCAL_BACKUP" +echo " remote backup: $REMOTE_BACKUP" +echo " remote marker: $REMOTE_ROOT/.last-content-deploy" diff --git a/deploy/content/remote-transaction.sh b/deploy/content/remote-transaction.sh new file mode 100755 index 0000000000..68612ca182 --- /dev/null +++ b/deploy/content/remote-transaction.sh @@ -0,0 +1,854 @@ +#!/usr/bin/env bash +# Transaction engine used locally for fixtures and over ssh for production. +# SSH activation requires Linux renameat2(RENAME_EXCHANGE); no non-atomic +# production fallback exists. + +set -euo pipefail + +die() { + echo "FATAL: $*" >&2 + exit 1 +} + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +device_id() { + if stat -c '%d' "$1" >/dev/null 2>&1; then + stat -c '%d' "$1" + else + stat -f '%d' "$1" + fi +} + +verify_durability_helper() { + local helper=$1 expected_hash=$2 + command -v python3 >/dev/null 2>&1 || die "python3 is required for durable transactions" + [ -f "$helper" ] && [ ! -L "$helper" ] || die "durability helper is not a regular file" + [ "$(sha256_file "$helper")" = "$expected_hash" ] || die "durability helper checksum changed" +} + +durable_paths() { + local helper=$1 + shift + python3 "$helper" fsync "$@" +} + +durable_tree() { + python3 "$1" fsync-tree "$2" +} + +file_mode() { + if stat -c '%a' "$1" >/dev/null 2>&1; then + stat -c '%a' "$1" + else + stat -f '%Lp' "$1" + fi +} + +link_count() { + if stat -c '%h' "$1" >/dev/null 2>&1; then + stat -c '%h' "$1" + else + stat -f '%l' "$1" + fi +} + +validate_upload_dir() { + local upload_dir=$1 + [[ "$upload_dir" =~ ^/tmp/hackforger-content-upload\.[A-Za-z0-9]+$ ]] \ + || die "upload directory is outside the managed private namespace" + [ -d "$upload_dir" ] && [ ! -L "$upload_dir" ] && [ -O "$upload_dir" ] \ + || die "upload directory is not a real directory owned by the deployment user" + [ "$(file_mode "$upload_dir")" = 700 ] \ + || die "upload directory permissions are not 0700" +} + +upload_create() { + [ "$#" -eq 0 ] || die "upload-create takes no arguments" + local upload_dir + umask 077 + upload_dir=$(mktemp -d /tmp/hackforger-content-upload.XXXXXXXXXXXX) + chmod 700 "$upload_dir" + validate_upload_dir "$upload_dir" + printf '%s\n' "$upload_dir" +} + +upload_verify() { + [ "$#" -eq 4 ] \ + || die "upload-verify expects DIR ARCHIVE_HASH MANIFEST_HASH HELPER_HASH" + local upload_dir=$1 archive_hash=$2 manifest_hash=$3 helper_hash=$4 + local archive manifest helper path count + validate_upload_dir "$upload_dir" + archive="$upload_dir/content.tar" + manifest="$upload_dir/manifest.SHA256SUMS" + helper="$upload_dir/rename-exchange.py" + count=$(find "$upload_dir" -mindepth 1 -maxdepth 1 -print | wc -l | tr -d ' ') + [ "$count" = 3 ] || die "upload directory does not contain exactly three inputs" + for path in "$archive" "$manifest" "$helper"; do + [ -f "$path" ] && [ ! -L "$path" ] && [ -O "$path" ] \ + || die "uploaded input is not a regular file owned by the deployment user" + [ "$(link_count "$path")" = 1 ] || die "uploaded input has multiple hard links" + chmod 600 "$path" + done + [ "$(sha256_file "$archive")" = "$archive_hash" ] || die "uploaded archive checksum mismatch" + [ "$(sha256_file "$manifest")" = "$manifest_hash" ] || die "uploaded manifest checksum mismatch" + verify_durability_helper "$helper" "$helper_hash" + durable_paths "$helper" "$archive" "$manifest" "$helper" "$upload_dir" +} + +upload_clean() { + [ "$#" -eq 1 ] || die "upload-clean expects DIR" + local upload_dir=$1 path + validate_upload_dir "$upload_dir" + for path in \ + "$upload_dir/content.tar" \ + "$upload_dir/manifest.SHA256SUMS" \ + "$upload_dir/rename-exchange.py"; do + if [ -e "$path" ] || [ -L "$path" ]; then + rm -f -- "$path" + fi + done + [ -z "$(find "$upload_dir" -mindepth 1 -maxdepth 1 -print -quit)" ] \ + || die "upload directory contains an unexpected entry" + rmdir -- "$upload_dir" +} + +write_phase() { + local backup=$1 helper=$2 phase=$3 temporary + shift 3 + temporary=$(mktemp "$backup/.PHASE.XXXXXX") + printf 'phase=%s\nrecorded_at=%s\n' "$phase" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$temporary" + while [ "$#" -gt 0 ]; do + printf '%s\n' "$1" >> "$temporary" + shift + done + durable_paths "$helper" "$temporary" + mv "$temporary" "$backup/PHASE" + durable_paths "$helper" "$backup/PHASE" "$backup" +} + +validate_root_and_mount() { + local root=$1 mount_path=$2 entrypoint=$3 root_real current component + case "$root" in /*) ;; *) die "remote root must be absolute" ;; esac + [ "$root" != "/" ] || die "remote root cannot be /" + [ "$mount_path" = "custom/public/assets/landing" ] || die "unsupported content mount" + [ "$entrypoint" = "index.html" ] || die "unsupported content entrypoint" + [ -d "$root" ] || die "remote root does not exist: $root" + [ ! -L "$root" ] || die "remote root cannot be a symbolic link" + root_real=$(cd "$root" && pwd -P) + [ "$root_real" = "$root" ] || die "remote root must be canonical: $root" + current=$root + for component in custom public assets; do + current="$current/$component" + if [ -e "$current" ] || [ -L "$current" ]; then + [ -d "$current" ] && [ ! -L "$current" ] || die "mount ancestor is not a real directory: $current" + else + break + fi + done +} + +validate_inode_types() { + local tree=$1 special + special=$(find "$tree" ! -type d ! -type f -print -quit) + [ -z "$special" ] || die "content tree contains a non-directory/non-regular inode: $special" +} + +validate_manifest_path() { + local path=$1 + [ -n "$path" ] || die "empty manifest path" + if [[ "$path" =~ [[:cntrl:]] ]]; then + die "control characters are not allowed in manifest paths" + fi + case "$path" in + /*|./*|../*|*/../*|*/..|*/./*|*//*|*\\*) die "unsafe manifest path: $path" ;; + esac +} + +verify_tree() { + local tree=$1 manifest=$2 entrypoint=$3 work line hash separator path + local actual_hash duplicate directory parent + work=$(mktemp -d "${TMPDIR:-/tmp}/verify-content.XXXXXX") + validate_inode_types "$tree" + [ -f "$manifest" ] && [ ! -L "$manifest" ] || { rm -rf "$work"; die "manifest is not a regular file"; } + + : > "$work/expected" + while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] || { rm -rf "$work"; die "blank manifest line"; } + [ "${#line}" -ge 67 ] || { rm -rf "$work"; die "malformed manifest line"; } + hash=${line:0:64} + separator=${line:64:2} + path=${line:66} + [[ "$hash" =~ ^[0-9a-f]{64}$ ]] || { rm -rf "$work"; die "invalid manifest digest"; } + [ "$separator" = " " ] || { rm -rf "$work"; die "manifest requires two spaces before each path"; } + validate_manifest_path "$path" + [ -f "$tree/$path" ] && [ ! -L "$tree/$path" ] || { rm -rf "$work"; die "manifest file is not regular: $path"; } + actual_hash=$(sha256_file "$tree/$path") + [ "$actual_hash" = "$hash" ] || { rm -rf "$work"; die "checksum mismatch: $path"; } + printf '%s\n' "$path" >> "$work/expected" + done < "$manifest" + + [ -s "$work/expected" ] || { rm -rf "$work"; die "manifest is empty"; } + [ -s "$tree/$entrypoint" ] || { rm -rf "$work"; die "entrypoint is missing or empty"; } + LC_ALL=C sort "$work/expected" > "$work/expected.sorted" + duplicate=$(uniq -d "$work/expected.sorted" | head -1 || true) + [ -z "$duplicate" ] || { rm -rf "$work"; die "duplicate manifest path: $duplicate"; } + ( + cd "$tree" + find . -type f -print | sed 's#^\./##' | LC_ALL=C sort + ) > "$work/actual.sorted" + if ! cmp -s "$work/expected.sorted" "$work/actual.sorted"; then + echo "FATAL: manifest does not exactly cover the content tree" >&2 + diff -u "$work/expected.sorted" "$work/actual.sorted" >&2 || true + rm -rf "$work" + exit 1 + fi + : > "$work/expected-dirs" + printf '.\n' >> "$work/expected-dirs" + while IFS= read -r path || [ -n "$path" ]; do + directory=$(dirname "$path") + while [ "$directory" != . ]; do + printf '%s\n' "$directory" >> "$work/expected-dirs" + parent=$(dirname "$directory") + [ "$parent" != "$directory" ] || { rm -rf "$work"; die "could not normalize manifest directory"; } + directory=$parent + done + done < "$work/expected.sorted" + LC_ALL=C sort -u "$work/expected-dirs" > "$work/expected-dirs.sorted" + ( + cd "$tree" + find . -type d -print | sed 's#^\./##' | LC_ALL=C sort + ) > "$work/actual-dirs.sorted" + if ! cmp -s "$work/expected-dirs.sorted" "$work/actual-dirs.sorted"; then + echo "FATAL: content tree has directories not implied by the manifest" >&2 + diff -u "$work/expected-dirs.sorted" "$work/actual-dirs.sorted" >&2 || true + rm -rf "$work" + exit 1 + fi + rm -rf "$work" +} + +generate_tree_manifest() { + local tree=$1 output=$2 work path + work=$(mktemp -d "${TMPDIR:-/tmp}/snapshot-manifest.XXXXXX") + validate_inode_types "$tree" + ( + cd "$tree" + find . -type f -print | sed 's#^\./##' | LC_ALL=C sort + ) > "$work/files" + : > "$work/manifest" + while IFS= read -r path || [ -n "$path" ]; do + validate_manifest_path "$path" + printf '%s %s\n' "$(sha256_file "$tree/$path")" "$path" >> "$work/manifest" + done < "$work/files" + mv "$work/manifest" "$output" + rm -rf "$work" +} + +validate_token() { + [[ "$1" =~ ^[A-Za-z0-9._-]+$ ]] || die "invalid lock token" +} + +assert_lock() { + local root=$1 token=$2 lock + validate_token "$token" + lock="$root/.content-deploy.lock" + [ -d "$lock" ] && [ ! -L "$lock" ] || die "content deployment lock is absent" + [ -f "$lock/token" ] && [ ! -L "$lock/token" ] || die "content deployment lock token is absent" + [ "$(cat "$lock/token")" = "$token" ] || die "content deployment lock token does not match" +} + +release_lock() { + local root=$1 token=$2 helper=$3 helper_hash=$4 lock + assert_lock "$root" "$token" + verify_durability_helper "$helper" "$helper_hash" + lock="$root/.content-deploy.lock" + rm -f "$lock/token" + durable_paths "$helper" "$lock" + rmdir "$lock" + durable_paths "$helper" "$root" +} + +exchange_paths() { + local mode=$1 helper=$2 left=$3 right=$4 temporary + [ -d "$left" ] && [ ! -L "$left" ] || return 1 + [ -d "$right" ] && [ ! -L "$right" ] || return 1 + [ "$(device_id "$left")" = "$(device_id "$right")" ] || return 1 + if [ "$mode" = "exchange" ]; then + python3 "$helper" exchange "$left" "$right" + return + fi + [ "$mode" = "local-test" ] || return 1 + temporary="${left}.local-test-exchange.$$" + [ ! -e "$temporary" ] || return 1 + mv "$left" "$temporary" || return 1 + if ! mv "$right" "$left"; then + mv "$temporary" "$left" >/dev/null 2>&1 || true + return 1 + fi + if ! mv "$temporary" "$right"; then + return 1 + fi + if [ "${CONTENT_TEST_EXCHANGE_FAIL_AFTER_SWAP:-0}" = 1 ]; then + return 70 + fi +} + +inspect() { + [ "$#" -eq 3 ] || die "inspect expects ROOT MOUNT ENTRYPOINT" + validate_root_and_mount "$1" "$2" "$3" + root=$1 + live="$root/$2" + marker="$root/.last-content-deploy" + if [ -L "$live" ]; then + die "live content path cannot be a symbolic link" + elif [ -e "$live" ] && [ ! -d "$live" ]; then + die "live content path exists but is not a directory" + elif [ -d "$live" ]; then + validate_inode_types "$live" + [ -s "$live/$3" ] || die "live content entrypoint is missing or empty" + echo "live_exists=1" + echo "live_files=$(find "$live" -type f | wc -l | tr -d ' ')" + else + echo "live_exists=0" + echo "live_files=0" + fi + if [ -e "$marker" ]; then + [ -f "$marker" ] && [ ! -L "$marker" ] || die "content marker is not a regular file" + echo "marker_present=1" + echo "marker_sha256=$(sha256_file "$marker")" + else + echo "marker_present=0" + echo "marker_sha256=none" + fi +} + +lock_acquire() { + [ "$#" -eq 8 ] || die "lock-acquire expects 8 arguments" + root=$1 + mount_path=$2 + entrypoint=$3 + token=$4 + activation_mode=$5 + exchange_helper=$6 + exchange_helper_hash=$7 + allow_initial=$8 + validate_root_and_mount "$root" "$mount_path" "$entrypoint" + validate_token "$token" + [ "$allow_initial" = true ] || [ "$allow_initial" = false ] || die "invalid initial-install setting" + verify_durability_helper "$exchange_helper" "$exchange_helper_hash" + lock="$root/.content-deploy.lock" + if [ -e "$lock" ] || [ -L "$lock" ]; then + echo "FATAL: deployment lock already exists: $lock" >&2 + echo " inspect $root/.content-backups/*/PHASE and MANUAL_RECOVERY before removing it" >&2 + exit 1 + fi + backup_root="$root/.content-backups" + if [ -d "$backup_root" ] && [ ! -L "$backup_root" ]; then + for transaction in "$backup_root"/*; do + [ -d "$transaction" ] || continue + phase_file="$transaction/PHASE" + if [ ! -f "$phase_file" ] || [ -L "$phase_file" ]; then + die "stale transaction without a durable phase journal: $transaction" + fi + phase=$(sed -n 's/^phase=//p' "$phase_file" | head -1) + case "$phase" in + completed|rolled-back|aborted) ;; + *) die "stale incomplete transaction ($phase): $transaction; inspect PHASE/MANUAL_RECOVERY" ;; + esac + done + elif [ -e "$backup_root" ] || [ -L "$backup_root" ]; then + die "remote backup root is not a real directory" + fi + if ! mkdir "$lock" 2>/dev/null; then + die "another content deployment holds $lock" + fi + printf '%s\n' "$token" > "$lock/token" + durable_paths "$exchange_helper" "$lock/token" "$lock" "$root" + + live="$root/$mount_path" + live_parent=$(dirname "$live") + probe_parent=$root + [ -d "$live_parent" ] && probe_parent=$live_parent + if [ "$activation_mode" = exchange ]; then + if [ "$(uname -s)" != Linux ] || ! command -v python3 >/dev/null 2>&1 \ + || [ ! -f "$exchange_helper" ] || [ -L "$exchange_helper" ] \ + || [ "$(sha256_file "$exchange_helper")" != "$exchange_helper_hash" ] \ + || ! python3 "$exchange_helper" probe "$probe_parent"; then + release_lock "$root" "$token" "$exchange_helper" "$exchange_helper_hash" >/dev/null 2>&1 || true + die "Linux renameat2(RENAME_EXCHANGE) capability probe failed" + fi + elif [ "$activation_mode" != local-test ]; then + release_lock "$root" "$token" "$exchange_helper" "$exchange_helper_hash" >/dev/null 2>&1 || true + die "unsupported activation mode" + fi + echo "lock_token=$token" + echo "activation_mode=$activation_mode" +} + +snapshot() { + [ "$#" -eq 8 ] || die "snapshot expects 8 arguments" + root=$1 + mount_path=$2 + entrypoint=$3 + token=$4 + backup_id=$5 + allow_initial=$6 + durability_helper=$7 + durability_helper_hash=$8 + validate_root_and_mount "$root" "$mount_path" "$entrypoint" + assert_lock "$root" "$token" + verify_durability_helper "$durability_helper" "$durability_helper_hash" + [[ "$backup_id" =~ ^[A-Za-z0-9._-]+$ ]] || die "invalid backup id" + live="$root/$mount_path" + live_parent=$(dirname "$live") + leaf=$(basename "$live") + marker="$root/.last-content-deploy" + backup_root="$root/.content-backups" + backup="$backup_root/$backup_id" + if [ -e "$backup_root" ] || [ -L "$backup_root" ]; then + [ -d "$backup_root" ] && [ ! -L "$backup_root" ] || die "remote backup root is not a real directory" + else + mkdir "$backup_root" + fi + [ ! -e "$backup" ] || die "remote backup already exists: $backup" + mkdir "$backup" + [ "$(device_id "$root")" = "$(device_id "$backup_root")" ] || die "remote backup must share the live filesystem" + + if [ -L "$live" ]; then + die "live content path cannot be a symbolic link" + elif [ -e "$live" ] && [ ! -d "$live" ]; then + die "live content path exists but is not a directory" + elif [ -d "$live" ]; then + validate_inode_types "$live" + [ -s "$live/$entrypoint" ] || die "live content entrypoint is missing or empty" + generate_tree_manifest "$live" "$backup/previous.SHA256SUMS" + COPYFILE_DISABLE=1 tar -cf "$backup/live.tar" -C "$live_parent" "$leaf" + check=$(mktemp -d "$backup/.snapshot-check.XXXXXX") + COPYFILE_DISABLE=1 tar -xf "$backup/live.tar" -C "$check" + verify_tree "$check/$leaf" "$backup/previous.SHA256SUMS" "$entrypoint" + rm -rf "$check" + echo "live_exists=1" + echo "backup_archive=$backup/live.tar" + echo "backup_archive_sha256=$(sha256_file "$backup/live.tar")" + echo "previous_manifest=$backup/previous.SHA256SUMS" + echo "previous_manifest_sha256=$(sha256_file "$backup/previous.SHA256SUMS")" + else + [ "$allow_initial" = true ] || die "live content is absent and initial installation is disabled" + : > "$backup/live-was-absent" + echo "live_exists=0" + echo "backup_archive=absent" + echo "backup_archive_sha256=absent" + echo "previous_manifest=absent" + echo "previous_manifest_sha256=absent" + fi + + if [ -e "$marker" ]; then + [ -f "$marker" ] && [ ! -L "$marker" ] || die "content marker is not a regular file" + cp "$marker" "$backup/previous-marker" + echo "previous_marker=$backup/previous-marker" + echo "previous_marker_sha256=$(sha256_file "$backup/previous-marker")" + else + : > "$backup/marker-was-absent" + echo "previous_marker=absent" + echo "previous_marker_sha256=absent" + fi + printf 'backup_id=%s\nlock_token=%s\ncreated_at=%s\n' \ + "$backup_id" "$token" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$backup/snapshot" + durable_tree "$durability_helper" "$backup" + durable_paths "$durability_helper" "$backup" "$backup_root" "$root" + write_phase "$backup" "$durability_helper" snapshot-created \ + "lock_token=$token" "live_exists=$([ -f "$backup/live-was-absent" ] && echo 0 || echo 1)" + echo "backup_dir=$backup" +} + +backup_ack() { + [ "$#" -eq 8 ] || die "backup-ack expects ROOT TOKEN BACKUP_ID ARCHIVE_HASH MANIFEST_HASH MARKER_HASH HELPER HELPER_HASH" + root=$1 + token=$2 + backup_id=$3 + expected_hash=$4 + expected_manifest_hash=$5 + expected_marker_hash=$6 + durability_helper=$7 + durability_helper_hash=$8 + validate_root_and_mount "$root" custom/public/assets/landing index.html + assert_lock "$root" "$token" + verify_durability_helper "$durability_helper" "$durability_helper_hash" + [[ "$backup_id" =~ ^[A-Za-z0-9._-]+$ ]] || die "invalid backup id" + backup="$root/.content-backups/$backup_id" + [ -d "$backup" ] || die "authoritative remote backup is absent" + if [ "$expected_hash" = absent ]; then + [ -f "$backup/live-was-absent" ] || die "initial-install backup state does not match" + [ "$expected_manifest_hash" = absent ] || die "initial-install manifest state does not match" + else + [[ "$expected_hash" =~ ^[0-9a-f]{64}$ ]] || die "invalid backup hash" + [ -f "$backup/live.tar" ] && [ "$(sha256_file "$backup/live.tar")" = "$expected_hash" ] \ + || die "authoritative remote backup hash changed" + [[ "$expected_manifest_hash" =~ ^[0-9a-f]{64}$ ]] || die "invalid previous manifest hash" + [ -f "$backup/previous.SHA256SUMS" ] \ + && [ "$(sha256_file "$backup/previous.SHA256SUMS")" = "$expected_manifest_hash" ] \ + || die "authoritative previous manifest hash changed" + fi + if [ "$expected_marker_hash" = absent ]; then + [ -f "$backup/marker-was-absent" ] || die "previous marker absence state changed" + else + [[ "$expected_marker_hash" =~ ^[0-9a-f]{64}$ ]] || die "invalid previous marker hash" + [ -f "$backup/previous-marker" ] \ + && [ "$(sha256_file "$backup/previous-marker")" = "$expected_marker_hash" ] \ + || die "authoritative previous marker hash changed" + fi + printf 'archive_sha256=%s\nmanifest_sha256=%s\nmarker_sha256=%s\nvalidated_at=%s\n' \ + "$expected_hash" "$expected_manifest_hash" "$expected_marker_hash" \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$backup/local-backup-validated" + durable_paths "$durability_helper" "$backup/local-backup-validated" + write_phase "$backup" "$durability_helper" local-backup-validated \ + "archive_sha256=$expected_hash" "manifest_sha256=$expected_manifest_hash" "marker_sha256=$expected_marker_hash" + echo "local_backup_ack=$expected_hash" +} + +abort_transaction() { + [ "$#" -eq 6 ] || die "abort expects ROOT TOKEN BACKUP_ID HELPER HELPER_HASH REASON" + root=$1 + token=$2 + backup_id=$3 + durability_helper=$4 + durability_helper_hash=$5 + reason=$6 + validate_root_and_mount "$root" custom/public/assets/landing index.html + assert_lock "$root" "$token" + verify_durability_helper "$durability_helper" "$durability_helper_hash" + [[ "$backup_id" =~ ^[A-Za-z0-9._-]+$ ]] || die "invalid backup id" + [[ "$reason" =~ ^[A-Za-z0-9._-]+$ ]] || die "invalid abort reason" + backup="$root/.content-backups/$backup_id" + if [ -d "$backup" ] && [ ! -L "$backup" ]; then + write_phase "$backup" "$durability_helper" aborted "reason=$reason" + elif [ -e "$backup" ] || [ -L "$backup" ]; then + die "remote backup is not a real directory" + fi + release_lock "$root" "$token" "$durability_helper" "$durability_helper_hash" +} + +apply_transaction() { + [ "$#" -eq 20 ] || die "apply expects 20 arguments" + root=$1 + mount_path=$2 + entrypoint=$3 + archive=$4 + manifest=$5 + expected_archive_hash=$6 + expected_manifest_hash=$7 + expected_entrypoint_hash=$8 + smoke_asset=$9 + expected_asset_hash=${10} + release_id=${11} + source_revision=${12} + backup_id=${13} + public_base_url=${14} + smoke_mode=${15} + allow_initial=${16} + token=${17} + activation_mode=${18} + exchange_helper=${19} + exchange_helper_hash=${20} + + validate_root_and_mount "$root" "$mount_path" "$entrypoint" + assert_lock "$root" "$token" + validate_manifest_path "$smoke_asset" + [[ "$release_id" =~ ^[0-9a-f]{64}$ ]] || die "invalid release id" + [[ "$source_revision" =~ ^([0-9a-f]{40,64}|unversioned)$ ]] || die "invalid source revision" + [[ "$backup_id" =~ ^[A-Za-z0-9._-]+$ ]] || die "invalid backup id" + [ "$allow_initial" = true ] || [ "$allow_initial" = false ] || die "invalid initial-install setting" + [ "$smoke_mode" = http ] || [ "$smoke_mode" = local ] || die "invalid smoke mode" + [ "$activation_mode" = exchange ] || [ "$activation_mode" = local-test ] || die "invalid activation mode" + + live="$root/$mount_path" + live_parent=$(dirname "$live") + leaf=$(basename "$live") + marker="$root/.last-content-deploy" + backup="$root/.content-backups/$backup_id" + stage="$live_parent/.${leaf}.staging-${backup_id}" + marker_tmp="" + smoke_index="" + smoke_asset_file="" + activated=0 + committed=0 + ambiguous=0 + had_live=0 + recovery_peer=$stage + + write_manual_recovery() { + local manual_status=${1:-ROLLBACK_FAILED} + mkdir -p "$backup" 2>/dev/null || true + printf '%s\n' \ + "status=$manual_status" \ + "live=$live" \ + "staging_peer=$stage" \ + "backup_peer=$backup/live" \ + "failed_peer=$backup/failed-live" \ + "previous_manifest=$backup/previous.SHA256SUMS" \ + "lock=$root/.content-deploy.lock" \ + "recorded_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$backup/MANUAL_RECOVERY" 2>/dev/null || true + durable_paths "$exchange_helper" "$backup/MANUAL_RECOVERY" "$backup" >/dev/null 2>&1 || true + write_phase "$backup" "$exchange_helper" manual-recovery \ + "status=$manual_status" "live=$live" "staging_peer=$stage" \ + "backup_peer=$backup/live" >/dev/null 2>&1 || true + } + + matches_incoming() { + ( verify_tree "$1" "$manifest" "$entrypoint" ) >/dev/null 2>&1 + } + + matches_previous() { + local candidate=$1 + if [ "$had_live" -eq 1 ]; then + ( verify_tree "$candidate" "$backup/previous.SHA256SUMS" "$entrypoint" ) >/dev/null 2>&1 + else + [ -d "$candidate" ] && [ -z "$(find "$candidate" -mindepth 1 -print -quit)" ] + fi + } + + restore_marker_verified() { + if [ -f "$backup/previous-marker" ]; then + cp "$backup/previous-marker" "$marker_tmp" || return 1 + durable_paths "$exchange_helper" "$marker_tmp" || return 1 + mv "$marker_tmp" "$marker" || return 1 + durable_paths "$exchange_helper" "$marker" "$root" || return 1 + [ "$(sha256_file "$marker")" = "$(sha256_file "$backup/previous-marker")" ] || return 1 + elif [ -f "$backup/marker-was-absent" ]; then + rm -f "$marker" || return 1 + durable_paths "$exchange_helper" "$root" || return 1 + [ ! -e "$marker" ] || return 1 + else + return 1 + fi + } + + rollback_verified() { + local exchange_result + if [ "${CONTENT_TEST_FAIL_ROLLBACK:-0}" = 1 ]; then + return 1 + fi + if [ ! -d "$recovery_peer" ]; then + if [ -d "$backup/live" ]; then + recovery_peer="$backup/live" + elif [ -d "$stage" ]; then + recovery_peer=$stage + else + return 1 + fi + fi + set +e + exchange_paths "$activation_mode" "$exchange_helper" "$live" "$recovery_peer" + exchange_result=$? + set -e + if [ "$exchange_result" -ne 0 ]; then + if matches_previous "$live" && matches_incoming "$recovery_peer"; then + : # The exchange completed even though its helper reported an error. + else + return 1 + fi + fi + durable_paths "$exchange_helper" "$live_parent" || return 1 + if [ "$had_live" -eq 1 ]; then + ( verify_tree "$live" "$backup/previous.SHA256SUMS" "$entrypoint" ) || return 1 + else + [ -z "$(find "$live" -mindepth 1 -print -quit)" ] || return 1 + rmdir "$live" || return 1 + [ ! -e "$live" ] || return 1 + fi + restore_marker_verified || return 1 + ( verify_tree "$recovery_peer" "$manifest" "$entrypoint" ) || return 1 + [ ! -e "$backup/failed-live" ] || return 1 + mv "$recovery_peer" "$backup/failed-live" || return 1 + [ -d "$backup/failed-live" ] || return 1 + return 0 + } + + finish() { + rc=$? + trap - EXIT INT TERM HUP + set +e + [ -n "$smoke_index" ] && rm -f "$smoke_index" + [ -n "$smoke_asset_file" ] && rm -f "$smoke_asset_file" + rm -f "$marker_tmp" + if [ "$rc" -ne 0 ] && [ "$committed" -eq 1 ]; then + write_manual_recovery COMMIT_FINALIZATION_FAILED + echo "Content was committed but transaction finalization failed; further deploys are blocked. See $backup/MANUAL_RECOVERY" >&2 + exit 75 + fi + if [ "$rc" -ne 0 ] && [ "$committed" -eq 0 ]; then + if [ "$ambiguous" -eq 1 ]; then + write_manual_recovery ATOMIC_EXCHANGE_AMBIGUOUS + echo "ATOMIC EXCHANGE STATE AMBIGUOUS; lock retained. See $backup/MANUAL_RECOVERY" >&2 + exit 75 + fi + if [ "$activated" -eq 1 ]; then + if rollback_verified; then + printf 'rolled_back_at=%s\nexit_code=%s\nverified=true\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$rc" > "$backup/ROLLBACK" + durable_paths "$exchange_helper" "$backup/ROLLBACK" + write_phase "$backup" "$exchange_helper" rolled-back "exit_code=$rc" + if ( release_lock "$root" "$token" "$exchange_helper" "$exchange_helper_hash" ); then + echo "Content activation failed; previous live content and marker were restored and verified." >&2 + exit "$rc" + fi + fi + write_manual_recovery + echo "ROLLBACK FAILED; lock retained. See $backup/MANUAL_RECOVERY" >&2 + exit 75 + fi + if [ -d "$stage" ] && [ -d "$backup" ] && [ ! -e "$backup/failed-staging" ]; then + if mv "$stage" "$backup/failed-staging" >/dev/null 2>&1; then + durable_paths "$exchange_helper" "$live_parent" "$backup" >/dev/null 2>&1 || true + fi + fi + write_phase "$backup" "$exchange_helper" aborted "exit_code=$rc" >/dev/null 2>&1 || true + if ( release_lock "$root" "$token" "$exchange_helper" "$exchange_helper_hash" ); then + exit "$rc" + fi + write_manual_recovery PREACTIVATION_CLEANUP_FAILED + echo "Pre-activation failure could not release the lock; manual recovery required." >&2 + exit 75 + fi + exit "$rc" + } + trap finish EXIT INT TERM HUP + + [ -d "$backup" ] || die "authoritative remote backup is absent" + [ -f "$backup/local-backup-validated" ] || die "validated local backup acknowledgement is absent" + verify_durability_helper "$exchange_helper" "$exchange_helper_hash" + [ -f "$archive" ] && [ ! -L "$archive" ] || die "uploaded archive is not regular" + [ -f "$manifest" ] && [ ! -L "$manifest" ] || die "uploaded manifest is not regular" + [ "$(sha256_file "$archive")" = "$expected_archive_hash" ] || die "uploaded archive checksum mismatch" + [ "$(sha256_file "$manifest")" = "$expected_manifest_hash" ] || die "uploaded manifest checksum mismatch" + durable_paths "$exchange_helper" "$archive" "$manifest" + cp "$manifest" "$backup/incoming.SHA256SUMS" + if [ "$activation_mode" = exchange ]; then + [ "$(uname -s)" = Linux ] && command -v python3 >/dev/null 2>&1 \ + && [ -f "$exchange_helper" ] && [ ! -L "$exchange_helper" ] \ + && [ "$(sha256_file "$exchange_helper")" = "$exchange_helper_hash" ] \ + || die "atomic exchange helper is unavailable or changed" + fi + marker_tmp=$(mktemp "$root/.last-content-deploy.tmp.XXXXXX") + + mkdir -p "$live_parent" + [ ! -e "$stage" ] || die "staging path already exists: $stage" + mkdir "$stage" + COPYFILE_DISABLE=1 tar -xf "$archive" -C "$stage" + verify_tree "$stage" "$manifest" "$entrypoint" + [ "$(sha256_file "$stage/$entrypoint")" = "$expected_entrypoint_hash" ] || die "entrypoint digest argument does not match" + [ -f "$stage/$smoke_asset" ] && [ ! -L "$stage/$smoke_asset" ] || die "smoke asset is missing" + [ "$(sha256_file "$stage/$smoke_asset")" = "$expected_asset_hash" ] || die "smoke asset digest argument does not match" + find "$stage" -type d -exec chmod 755 {} + + find "$stage" -type f -exec chmod 644 {} + + durable_tree "$exchange_helper" "$stage" + durable_paths "$exchange_helper" "$live_parent" + write_phase "$backup" "$exchange_helper" staging-durable \ + "stage=$stage" "archive_sha256=$expected_archive_hash" + [ "$(device_id "$live_parent")" = "$(device_id "$backup")" ] || die "live, staging, and backup must share one filesystem" + + if [ -d "$live" ] && [ ! -L "$live" ]; then + had_live=1 + verify_tree "$live" "$backup/previous.SHA256SUMS" "$entrypoint" + elif [ ! -e "$live" ] && [ "$allow_initial" = true ] && [ -f "$backup/live-was-absent" ]; then + mkdir "$live" + else + die "live content state changed after the authoritative snapshot" + fi + + set +e + exchange_paths "$activation_mode" "$exchange_helper" "$live" "$stage" + exchange_result=$? + set -e + recovery_peer=$stage + if [ "$exchange_result" -eq 0 ]; then + activated=1 + elif matches_incoming "$live" && matches_previous "$stage"; then + activated=1 + durable_paths "$exchange_helper" "$live_parent" + write_phase "$backup" "$exchange_helper" activated \ + "exchange_reported_error=$exchange_result" "live=$live" "recovery_peer=$stage" + die "atomic exchange completed but helper returned $exchange_result; rolling back" + elif matches_previous "$live" && matches_incoming "$stage"; then + die "atomic exchange failed before changing live (exit $exchange_result)" + else + ambiguous=1 + die "atomic exchange returned $exchange_result and resulting paths are ambiguous" + fi + durable_paths "$exchange_helper" "$live_parent" + write_phase "$backup" "$exchange_helper" activated \ + "exchange_reported_error=0" "live=$live" "recovery_peer=$stage" + verify_tree "$live" "$manifest" "$entrypoint" + if [ "${CONTENT_TEST_FAIL_AFTER_SWAP:-0}" = 1 ]; then + die "injected post-swap failure" + fi + + if [ "$smoke_mode" = http ]; then + case "$public_base_url" in http://*|https://*) ;; *) die "public base URL must use http or https" ;; esac + smoke_index=$(mktemp "${TMPDIR:-/tmp}/content-smoke-index.XXXXXX") + smoke_asset_file=$(mktemp "${TMPDIR:-/tmp}/content-smoke-asset.XXXXXX") + curl -fsSL --max-time 30 -H 'Accept-Encoding: identity' "${public_base_url%/}/" -o "$smoke_index" + [ "$(sha256_file "$smoke_index")" = "$expected_entrypoint_hash" ] || die "public entrypoint checksum mismatch" + curl -fsSL --max-time 30 -H 'Accept-Encoding: identity' \ + "${public_base_url%/}/assets/landing/$smoke_asset" -o "$smoke_asset_file" + [ "$(sha256_file "$smoke_asset_file")" = "$expected_asset_hash" ] || die "public smoke asset checksum mismatch" + fi + write_phase "$backup" "$exchange_helper" smoke-verified \ + "entrypoint_sha256=$expected_entrypoint_hash" "asset=$smoke_asset" "asset_sha256=$expected_asset_hash" + + [ ! -e "$backup/live" ] || die "remote recovery directory already exists" + mv "$stage" "$backup/live" + recovery_peer="$backup/live" + durable_paths "$exchange_helper" "$live_parent" "$backup" + if [ "$had_live" -eq 1 ]; then + verify_tree "$backup/live" "$backup/previous.SHA256SUMS" "$entrypoint" + else + [ -z "$(find "$backup/live" -mindepth 1 -print -quit)" ] || die "initial-install recovery placeholder is not empty" + fi + write_phase "$backup" "$exchange_helper" old-live-backed-up "recovery_peer=$recovery_peer" + + printf '%s\n' \ + version=1 \ + "mount=$mount_path" \ + "release_id=$release_id" \ + "source_revision=$source_revision" \ + "manifest_sha256=$expected_manifest_hash" \ + "backup_id=$backup_id" \ + "deployed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$marker_tmp" + durable_paths "$exchange_helper" "$marker_tmp" + mv "$marker_tmp" "$marker" + durable_paths "$exchange_helper" "$marker" "$root" + [ -f "$marker" ] && grep -q "^release_id=$release_id$" "$marker" || die "new content marker verification failed" + write_phase "$backup" "$exchange_helper" marker-committed "marker=$marker" + committed=1 + write_phase "$backup" "$exchange_helper" completed "release_id=$release_id" + if ! ( release_lock "$root" "$token" "$exchange_helper" "$exchange_helper_hash" ); then + die "content committed but durable lock release failed" + fi + echo "release_id=$release_id" + echo "backup_id=$backup_id" + echo "marker=$marker" +} + +ACTION=${1:-} +shift || true +case "$ACTION" in + upload-create) upload_create "$@" ;; + upload-verify) upload_verify "$@" ;; + upload-clean) upload_clean "$@" ;; + inspect) inspect "$@" ;; + lock-acquire) lock_acquire "$@" ;; + snapshot) snapshot "$@" ;; + backup-ack) backup_ack "$@" ;; + abort) abort_transaction "$@" ;; + lock-release) [ "$#" -eq 4 ] || die "lock-release expects ROOT TOKEN HELPER HELPER_HASH"; release_lock "$1" "$2" "$3" "$4" ;; + apply) apply_transaction "$@" ;; + *) die "unknown transaction action: $ACTION" ;; +esac diff --git a/deploy/content/rename-exchange.py b/deploy/content/rename-exchange.py new file mode 100755 index 0000000000..9ceb94f8c3 --- /dev/null +++ b/deploy/content/rename-exchange.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Fail-closed Linux renameat2(RENAME_EXCHANGE) helper. + +Production activation uses this helper so the fixed live directory is never +absent. There is deliberately no non-atomic fallback here. +""" + +from __future__ import annotations + +import ctypes +import errno +import os +import shutil +import sys +import tempfile +import stat + + +AT_FDCWD = -100 +RENAME_EXCHANGE = 2 + + +def fail(message: str, code: int = 1) -> "None": + print(f"FATAL: {message}", file=sys.stderr) + raise SystemExit(code) + + +def rename_exchange(left: str, right: str) -> None: + if sys.platform != "linux": + fail("rename exchange is supported only on Linux") + if not os.path.isdir(left) or os.path.islink(left): + fail(f"exchange operand is not a real directory: {left}") + if not os.path.isdir(right) or os.path.islink(right): + fail(f"exchange operand is not a real directory: {right}") + if os.stat(left).st_dev != os.stat(right).st_dev: + fail("exchange operands are not on the same filesystem") + + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = getattr(libc, "renameat2", None) + if renameat2 is None: + fail("libc does not expose renameat2; refusing a non-atomic fallback") + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + result = renameat2( + AT_FDCWD, + os.fsencode(left), + AT_FDCWD, + os.fsencode(right), + RENAME_EXCHANGE, + ) + if result != 0: + error = ctypes.get_errno() + detail = os.strerror(error) if error else "unknown error" + fail(f"renameat2(RENAME_EXCHANGE) failed: {detail} (errno={error})") + + if os.environ.get("CONTENT_TEST_RENAME_FAIL_AFTER_SYSCALL") == "1": + fail("injected failure after renameat2 syscall", 70) + + parents = {os.path.dirname(os.path.abspath(left)), os.path.dirname(os.path.abspath(right))} + for parent in parents: + try: + descriptor = os.open(parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + except OSError as exc: + if exc.errno not in (errno.EINVAL, errno.ENOTSUP, errno.EROFS): + raise + + +def fsync_path(path: str) -> None: + metadata = os.lstat(path) + if stat.S_ISLNK(metadata.st_mode): + fail(f"refusing to fsync a symbolic link: {path}") + if not (stat.S_ISREG(metadata.st_mode) or stat.S_ISDIR(metadata.st_mode)): + fail(f"refusing to fsync a special inode: {path}") + flags = os.O_RDONLY + if stat.S_ISDIR(metadata.st_mode): + flags |= getattr(os, "O_DIRECTORY", 0) + descriptor = os.open(path, flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def fsync_tree(root: str) -> None: + if not os.path.isdir(root) or os.path.islink(root): + fail(f"durable tree root is not a real directory: {root}") + directories = [] + for current, dirnames, filenames in os.walk(root, topdown=True, followlinks=False): + directories.append(current) + for name in dirnames: + candidate = os.path.join(current, name) + if os.path.islink(candidate): + fail(f"durable tree contains a symbolic link: {candidate}") + for name in filenames: + candidate = os.path.join(current, name) + fsync_path(candidate) + for directory in reversed(directories): + fsync_path(directory) + + +def probe(parent: str) -> None: + if not os.path.isdir(parent) or os.path.islink(parent): + fail(f"probe parent is not a real directory: {parent}") + probe_root = tempfile.mkdtemp(prefix=".rename-exchange-probe-", dir=parent) + left = os.path.join(probe_root, "left") + right = os.path.join(probe_root, "right") + try: + os.mkdir(left) + os.mkdir(right) + with open(os.path.join(left, "left-marker"), "wb") as handle: + handle.write(b"left\n") + with open(os.path.join(right, "right-marker"), "wb") as handle: + handle.write(b"right\n") + rename_exchange(left, right) + if not os.path.isfile(os.path.join(left, "right-marker")): + fail("rename exchange probe did not move the right directory atomically") + if not os.path.isfile(os.path.join(right, "left-marker")): + fail("rename exchange probe did not move the left directory atomically") + rename_exchange(left, right) + if not os.path.isfile(os.path.join(left, "left-marker")): + fail("rename exchange probe could not restore the original order") + finally: + shutil.rmtree(probe_root, ignore_errors=True) + + +def main() -> None: + if len(sys.argv) == 3 and sys.argv[1] == "probe": + probe(sys.argv[2]) + return + if len(sys.argv) == 4 and sys.argv[1] == "exchange": + rename_exchange(sys.argv[2], sys.argv[3]) + return + if len(sys.argv) >= 3 and sys.argv[1] == "fsync": + for path in sys.argv[2:]: + fsync_path(path) + return + if len(sys.argv) == 3 and sys.argv[1] == "fsync-tree": + fsync_tree(sys.argv[2]) + return + fail("usage: rename-exchange.py probe PARENT | exchange LEFT RIGHT | fsync PATH... | fsync-tree ROOT", 64) + + +if __name__ == "__main__": + main() diff --git a/deploy/content/tests/run.sh b/deploy/content/tests/run.sh new file mode 100755 index 0000000000..3141a662e3 --- /dev/null +++ b/deploy/content/tests/run.sh @@ -0,0 +1,585 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CONTENT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) +PUBLISH="$CONTENT_DIR/publish-overlay.sh" +GENERATE="$CONTENT_DIR/generate-manifest.sh" +REMOTE_HELPER="$CONTENT_DIR/remote-transaction.sh" +EXCHANGE_HELPER="$CONTENT_DIR/rename-exchange.py" +TMP=$(mktemp -d "${TMPDIR:-/tmp}/content-publisher-tests.XXXXXX") +TMP=$(cd "$TMP" && pwd -P) +trap 'rm -rf "$TMP"' EXIT INT TERM HUP + +PASS=0 + +ok() { + PASS=$((PASS + 1)) + echo "ok $PASS - $*" +} + +fail() { + echo "not ok - $*" >&2 + exit 1 +} + +assert_file_text() { + file=$1 + expected=$2 + actual=$(cat "$file") + [ "$actual" = "$expected" ] || fail "$file: expected '$expected', got '$actual'" +} + +expect_failure() { + if "$@" >"$TMP/expected-failure.out" 2>&1; then + cat "$TMP/expected-failure.out" >&2 + fail "command unexpectedly succeeded: $*" + fi +} + +write_config() { + config=$1 + remote_root=$2 + backup_root=$3 + allow_initial=${4:-false} + require_clean=${5:-false} + smoke_asset=${6:-assets/site.css} + { + echo 'TRANSPORT=local' + echo 'DEPLOY_TARGET=local-fixture' + echo 'PUBLIC_BASE_URL=http://local.invalid' + echo "REMOTE_ROOT=$remote_root" + echo "LOCAL_BACKUP_ROOT=$backup_root" + echo "SMOKE_ASSET=$smoke_asset" + echo "ALLOW_INITIAL_INSTALL=$allow_initial" + echo "REQUIRE_CLEAN_SOURCE=$require_clean" + } > "$config" +} + +test_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +file_mode() { + if stat -c '%a' "$1" >/dev/null 2>&1; then + stat -c '%a' "$1" + else + stat -f '%Lp' "$1" + fi +} + +write_manifest_unchecked() { + source_dir=$1 + output=$2 + ( + cd "$source_dir" + find . -type f -print | sed 's#^\./##' | LC_ALL=C sort + ) > "$output.paths" + : > "$output" + while IFS= read -r path || [ -n "$path" ]; do + printf '%s %s\n' "$(test_sha256 "$source_dir/$path")" "$path" >> "$output" + done < "$output.paths" + rm -f "$output.paths" +} + +init_pushed_repo() { + repo=$1 + bare=$2 + git init -q "$repo" + git -C "$repo" checkout -q -b main + git -C "$repo" config user.name 'Content Test' + git -C "$repo" config user.email content-test@example.invalid + git init -q --bare "$bare" + git -C "$repo" remote add origin "$bare" +} + +commit_and_push() { + repo=$1 + message=$2 + git -C "$repo" add -A + git -C "$repo" commit -q -m "$message" + if git -C "$repo" rev-parse --verify '@{upstream}' >/dev/null 2>&1; then + git -C "$repo" push -q + else + git -C "$repo" push -q -u origin main + fi +} + +make_source() { + source_dir=$1 + label=$2 + mkdir -p "$source_dir/assets" + printf '%s\n' "$label" > "$source_dir/index.html" + printf '%s\n' "$label-asset" > "$source_dir/assets/site.css" + printf '%s\n' "$label-image" > "$source_dir/assets/banner 中文 name.webp" +} + +echo '1..24' + +# 1. Default invocation is a read-only plan. +CASE1="$TMP/case1" +REMOTE1="$CASE1/remote" +LIVE1="$REMOTE1/custom/public/assets/landing" +SOURCE1="$CASE1/source" +BACKUPS1="$CASE1/local-backups" +mkdir -p "$LIVE1" +printf 'old-live\n' > "$LIVE1/index.html" +printf 'obsolete\n' > "$LIVE1/obsolete.txt" +make_source "$SOURCE1" new-live +bash "$GENERATE" "$SOURCE1" "$CASE1/SHA256SUMS" >/dev/null +write_config "$CASE1/config" "$REMOTE1" "$BACKUPS1" +DRY_OUTPUT=$(bash "$PUBLISH" --config "$CASE1/config" --mount landing --source "$SOURCE1" --manifest "$CASE1/SHA256SUMS") +printf '%s\n' "$DRY_OUTPUT" | grep -q 'DRY RUN' || fail 'dry-run marker missing' +assert_file_text "$LIVE1/index.html" old-live +[ ! -e "$REMOTE1/.last-content-deploy" ] || fail 'dry run wrote remote marker' +[ ! -e "$REMOTE1/.content-backups" ] || fail 'dry run created remote backup' +[ ! -e "$BACKUPS1" ] || fail 'dry run created local backup' +ok 'default invocation is read-only' + +# 2. Apply swaps the exact tree and creates both backups and a marker. +bash "$PUBLISH" --config "$CASE1/config" --mount landing --source "$SOURCE1" --manifest "$CASE1/SHA256SUMS" --apply >/dev/null +assert_file_text "$LIVE1/index.html" new-live +[ ! -e "$LIVE1/obsolete.txt" ] || fail 'old unlisted file survived exact directory swap' +[ -f "$REMOTE1/.last-content-deploy" ] || fail 'content marker missing' +BACKUP_ID=$(sed -n 's/^backup_id=//p' "$REMOTE1/.last-content-deploy") +[ -n "$BACKUP_ID" ] || fail 'marker lacks backup id' +assert_file_text "$REMOTE1/.content-backups/$BACKUP_ID/live/index.html" old-live +[ -f "$BACKUPS1/$BACKUP_ID/live.tar" ] || fail 'local live tar backup missing' +tar -tf "$BACKUPS1/$BACKUP_ID/live.tar" >/dev/null || fail 'local live tar is invalid' +OLD_FROM_TAR=$(tar -xOf "$BACKUPS1/$BACKUP_ID/live.tar" landing/index.html) +[ "$OLD_FROM_TAR" = old-live ] || fail 'local backup does not contain prior live content' +grep -q '^phase=completed$' "$REMOTE1/.content-backups/$BACKUP_ID/PHASE" || fail 'completed deployment lacks a durable terminal phase' +[ ! -e "$REMOTE1/.content-deploy.lock" ] || fail 'completed deployment retained its lock' +ok 'apply performs exact swap with local and remote backups' + +# 3. A modified file invalidates the manifest before target mutation. +CASE3="$TMP/case3" +REMOTE3="$CASE3/remote" +LIVE3="$REMOTE3/custom/public/assets/landing" +SOURCE3="$CASE3/source" +mkdir -p "$LIVE3" +printf 'stable\n' > "$LIVE3/index.html" +make_source "$SOURCE3" candidate +bash "$GENERATE" "$SOURCE3" "$CASE3/SHA256SUMS" >/dev/null +printf 'tampered\n' > "$SOURCE3/index.html" +write_config "$CASE3/config" "$REMOTE3" "$CASE3/backups" +expect_failure bash "$PUBLISH" --config "$CASE3/config" --mount landing --source "$SOURCE3" --manifest "$CASE3/SHA256SUMS" +assert_file_text "$LIVE3/index.html" stable +ok 'checksum mismatch fails closed' + +# 4. An unlisted extra file invalidates exact coverage. +CASE4="$TMP/case4" +REMOTE4="$CASE4/remote" +LIVE4="$REMOTE4/custom/public/assets/landing" +SOURCE4="$CASE4/source" +mkdir -p "$LIVE4" +printf 'stable\n' > "$LIVE4/index.html" +make_source "$SOURCE4" candidate +bash "$GENERATE" "$SOURCE4" "$CASE4/SHA256SUMS" >/dev/null +printf 'extra\n' > "$SOURCE4/not-in-manifest.txt" +write_config "$CASE4/config" "$REMOTE4" "$CASE4/backups" +expect_failure bash "$PUBLISH" --config "$CASE4/config" --mount landing --source "$SOURCE4" --manifest "$CASE4/SHA256SUMS" +assert_file_text "$LIVE4/index.html" stable +ok 'extra source file fails exact manifest coverage' + +# 5. Symlinked content is rejected. +CASE5="$TMP/case5" +REMOTE5="$CASE5/remote" +LIVE5="$REMOTE5/custom/public/assets/landing" +SOURCE5="$CASE5/source" +mkdir -p "$LIVE5" +printf 'stable\n' > "$LIVE5/index.html" +make_source "$SOURCE5" candidate +bash "$GENERATE" "$SOURCE5" "$CASE5/SHA256SUMS" >/dev/null +ln -s index.html "$SOURCE5/linked.html" +write_config "$CASE5/config" "$REMOTE5" "$CASE5/backups" +expect_failure bash "$PUBLISH" --config "$CASE5/config" --mount landing --source "$SOURCE5" --manifest "$CASE5/SHA256SUMS" +assert_file_text "$LIVE5/index.html" stable +ok 'symbolic links fail closed' + +# 6. Missing config coordinates fail before inspection. +CASE6="$TMP/case6" +mkdir -p "$CASE6/remote/custom/public/assets/landing" +printf 'stable\n' > "$CASE6/remote/custom/public/assets/landing/index.html" +make_source "$CASE6/source" candidate +bash "$GENERATE" "$CASE6/source" "$CASE6/SHA256SUMS" >/dev/null +{ + echo 'TRANSPORT=local' + echo 'DEPLOY_TARGET=local-fixture' + echo "REMOTE_ROOT=$CASE6/remote" + echo "LOCAL_BACKUP_ROOT=$CASE6/backups" + echo 'SMOKE_ASSET=assets/site.css' + echo 'REQUIRE_CLEAN_SOURCE=false' +} > "$CASE6/config" +expect_failure bash "$PUBLISH" --config "$CASE6/config" --mount landing --source "$CASE6/source" --manifest "$CASE6/SHA256SUMS" +ok 'missing public URL in config fails closed' + +# 7. A post-swap failure restores both content and marker. +CASE7="$TMP/case7" +REMOTE7="$CASE7/remote" +LIVE7="$REMOTE7/custom/public/assets/landing" +SOURCE7="$CASE7/source" +mkdir -p "$LIVE7" +printf 'before-failure\n' > "$LIVE7/index.html" +printf 'version=1\nrelease_id=previous\n' > "$REMOTE7/.last-content-deploy" +make_source "$SOURCE7" after-failure +bash "$GENERATE" "$SOURCE7" "$CASE7/SHA256SUMS" >/dev/null +write_config "$CASE7/config" "$REMOTE7" "$CASE7/backups" +if CONTENT_TEST_FAIL_AFTER_SWAP=1 bash "$PUBLISH" --config "$CASE7/config" --mount landing --source "$SOURCE7" --manifest "$CASE7/SHA256SUMS" --apply >"$CASE7/output" 2>&1; then + cat "$CASE7/output" >&2 + fail 'injected post-swap failure unexpectedly succeeded' +fi +assert_file_text "$LIVE7/index.html" before-failure +grep -q '^release_id=previous$' "$REMOTE7/.last-content-deploy" || fail 'previous marker was not restored' +FAILED_NEW=$(find "$REMOTE7/.content-backups" -path '*/failed-live/index.html' -print -quit) +[ -n "$FAILED_NEW" ] || fail 'failed replacement tree was not preserved' +assert_file_text "$FAILED_NEW" after-failure +BACKUP7=$(dirname "$(dirname "$FAILED_NEW")") +grep -q '^phase=rolled-back$' "$BACKUP7/PHASE" || fail 'verified rollback lacks a durable terminal phase' +ok 'post-swap failure automatically restores prior live state' + +# 8. Initial installation is denied unless explicitly enabled. +CASE8="$TMP/case8" +REMOTE8="$CASE8/remote" +SOURCE8="$CASE8/source" +mkdir -p "$REMOTE8" +make_source "$SOURCE8" initial +bash "$GENERATE" "$SOURCE8" "$CASE8/SHA256SUMS" >/dev/null +write_config "$CASE8/config" "$REMOTE8" "$CASE8/backups" false +expect_failure bash "$PUBLISH" --config "$CASE8/config" --mount landing --source "$SOURCE8" --manifest "$CASE8/SHA256SUMS" --apply +[ ! -e "$REMOTE8/custom/public/assets/landing" ] || fail 'denied initial install created live content' +ok 'initial installation requires explicit config approval' + +# 9. FIFO and other special inodes are rejected in source and live trees. +CASE9="$TMP/case9" +REMOTE9="$CASE9/remote" +LIVE9="$REMOTE9/custom/public/assets/landing" +SOURCE9="$CASE9/source" +mkdir -p "$LIVE9" +printf 'stable\n' > "$LIVE9/index.html" +make_source "$SOURCE9" fifo-candidate +bash "$GENERATE" "$SOURCE9" "$CASE9/SHA256SUMS" >/dev/null +mkfifo "$SOURCE9/assets/pipe" +write_config "$CASE9/config" "$REMOTE9" "$CASE9/backups" +expect_failure bash "$PUBLISH" --config "$CASE9/config" --mount landing --source "$SOURCE9" --manifest "$CASE9/SHA256SUMS" +rm "$SOURCE9/assets/pipe" +mkfifo "$LIVE9/live-pipe" +expect_failure bash "$PUBLISH" --config "$CASE9/config" --mount landing --source "$SOURCE9" --manifest "$CASE9/SHA256SUMS" +ok 'source and remote FIFO inodes fail closed' + +# 10. Strict provenance accepts a clean, pushed, same-repository source. +CASE10="$TMP/case10" +REPO10="$CASE10/private" +BARE10="$CASE10/origin.git" +mkdir -p "$CASE10" +init_pushed_repo "$REPO10" "$BARE10" +make_source "$REPO10/overlays/landing" strict-clean +bash "$GENERATE" "$REPO10/overlays/landing" "$REPO10/overlays/SHA256SUMS" >/dev/null +REMOTE10="$CASE10/remote" +mkdir -p "$REMOTE10/custom/public/assets/landing" +printf 'stable\n' > "$REMOTE10/custom/public/assets/landing/index.html" +mkdir -p "$REPO10/deploy" +CONFIG10="$REPO10/deploy/production.env" +write_config "$CONFIG10" "$REMOTE10" "$CASE10/backups" false true +commit_and_push "$REPO10" 'initial content and deployment config' +bash "$PUBLISH" --config "$CONFIG10" --mount landing \ + --provenance-remote "$BARE10" --provenance-ref refs/heads/main \ + --source "$REPO10/overlays/landing" --manifest "$REPO10/overlays/SHA256SUMS" >/dev/null +ok 'strict provenance accepts clean pushed HEAD content' + +# 11. Strict provenance requires source and manifest in the same repository. +mkdir -p "$CASE10/outside" +cp "$REPO10/overlays/SHA256SUMS" "$CASE10/outside/SHA256SUMS" +expect_failure bash "$PUBLISH" --config "$CONFIG10" --mount landing \ + --provenance-remote "$BARE10" --provenance-ref refs/heads/main \ + --source "$REPO10/overlays/landing" --manifest "$CASE10/outside/SHA256SUMS" +cp "$CONFIG10" "$CASE10/outside/production.env" +expect_failure bash "$PUBLISH" --config "$CASE10/outside/production.env" --mount landing \ + --provenance-remote "$BARE10" --provenance-ref refs/heads/main \ + --source "$REPO10/overlays/landing" --manifest "$REPO10/overlays/SHA256SUMS" +ok 'strict provenance rejects a manifest or config outside the source repository' + +# 12. An ignored source file is rejected even when status is clean and the +# updated manifest itself is committed and pushed. +printf 'overlays/landing/ignored.bin\n' > "$REPO10/.gitignore" +commit_and_push "$REPO10" 'ignore fixture file' +printf 'ignored-but-present\n' > "$REPO10/overlays/landing/ignored.bin" +bash "$GENERATE" "$REPO10/overlays/landing" "$REPO10/overlays/SHA256SUMS" >/dev/null +git -C "$REPO10" add overlays/SHA256SUMS +git -C "$REPO10" commit -q -m 'manifest references ignored file' +git -C "$REPO10" push -q +[ -z "$(git -C "$REPO10" status --porcelain --untracked-files=all)" ] || fail 'ignored-file fixture repository is unexpectedly dirty' +expect_failure bash "$PUBLISH" --config "$CONFIG10" --mount landing \ + --provenance-remote "$BARE10" --provenance-ref refs/heads/main \ + --source "$REPO10/overlays/landing" --manifest "$REPO10/overlays/SHA256SUMS" +ok 'strict provenance rejects ignored or otherwise untracked source files' + +# 13. A tracked Git LFS pointer is never accepted as deployable content. +CASE13="$TMP/case13" +REPO13="$CASE13/private" +BARE13="$CASE13/origin.git" +mkdir -p "$CASE13" +init_pushed_repo "$REPO13" "$BARE13" +make_source "$REPO13/overlays/landing" lfs +printf '%s\n' \ + 'version https://git-lfs.github.com/spec/v1' \ + 'oid sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' \ + 'size 1234' > "$REPO13/overlays/landing/assets/site.css" +write_manifest_unchecked "$REPO13/overlays/landing" "$REPO13/overlays/SHA256SUMS" +REMOTE13="$CASE13/remote" +mkdir -p "$REMOTE13/custom/public/assets/landing" +printf 'stable\n' > "$REMOTE13/custom/public/assets/landing/index.html" +mkdir -p "$REPO13/deploy" +CONFIG13="$REPO13/deploy/production.env" +write_config "$CONFIG13" "$REMOTE13" "$CASE13/backups" false true +commit_and_push "$REPO13" 'tracked lfs pointer fixture and deployment config' +expect_failure bash "$PUBLISH" --config "$CONFIG13" --mount landing \ + --provenance-remote "$BARE13" --provenance-ref refs/heads/main \ + --source "$REPO13/overlays/landing" --manifest "$REPO13/overlays/SHA256SUMS" +ok 'tracked Git LFS pointers fail closed' + +# 14. A pre-existing lock blocks apply before any authoritative or local backup. +CASE14="$TMP/case14" +REMOTE14="$CASE14/remote" +LIVE14="$REMOTE14/custom/public/assets/landing" +SOURCE14="$CASE14/source" +mkdir -p "$LIVE14" "$REMOTE14/.content-deploy.lock" +printf 'stable\n' > "$LIVE14/index.html" +printf 'foreign-token\n' > "$REMOTE14/.content-deploy.lock/token" +make_source "$SOURCE14" locked +bash "$GENERATE" "$SOURCE14" "$CASE14/SHA256SUMS" >/dev/null +write_config "$CASE14/config" "$REMOTE14" "$CASE14/backups" +expect_failure bash "$PUBLISH" --config "$CASE14/config" --mount landing \ + --source "$SOURCE14" --manifest "$CASE14/SHA256SUMS" --apply +[ ! -e "$REMOTE14/.content-backups" ] || fail 'blocked apply created a remote backup outside the lock' +[ ! -e "$CASE14/backups" ] || fail 'blocked apply created a local backup outside the lock' +grep -q '^foreign-token$' "$REMOTE14/.content-deploy.lock/token" || fail 'publisher disturbed a foreign lock' +ok 'deployment lock covers authoritative and persistent backup creation' + +# 15. If recovery itself fails, retain the lock and exact manual paths and +# never claim that the old tree was restored. +CASE15="$TMP/case15" +REMOTE15="$CASE15/remote" +LIVE15="$REMOTE15/custom/public/assets/landing" +SOURCE15="$CASE15/source" +mkdir -p "$LIVE15" +printf 'before-unrecoverable\n' > "$LIVE15/index.html" +printf 'version=1\nrelease_id=previous\n' > "$REMOTE15/.last-content-deploy" +make_source "$SOURCE15" after-unrecoverable +bash "$GENERATE" "$SOURCE15" "$CASE15/SHA256SUMS" >/dev/null +write_config "$CASE15/config" "$REMOTE15" "$CASE15/backups" +set +e +CONTENT_TEST_FAIL_AFTER_SWAP=1 CONTENT_TEST_FAIL_ROLLBACK=1 \ + bash "$PUBLISH" --config "$CASE15/config" --mount landing \ + --source "$SOURCE15" --manifest "$CASE15/SHA256SUMS" --apply > "$CASE15/output" 2>&1 +STATUS15=$? +set -e +[ "$STATUS15" -eq 75 ] || { cat "$CASE15/output" >&2; fail "rollback failure returned $STATUS15 instead of 75"; } +grep -q 'ROLLBACK FAILED; lock retained' "$CASE15/output" || fail 'rollback failure did not report retained lock' +if grep -q 'restored and verified' "$CASE15/output"; then fail 'rollback failure falsely claimed restoration'; fi +[ -d "$REMOTE15/.content-deploy.lock" ] || fail 'rollback failure did not retain deployment lock' +MANUAL15=$(find "$REMOTE15/.content-backups" -name MANUAL_RECOVERY -print -quit) +[ -n "$MANUAL15" ] || fail 'rollback failure did not write manual recovery paths' +grep -q '^phase=manual-recovery$' "$(dirname "$MANUAL15")/PHASE" || fail 'manual recovery lacks a durable blocking phase' +assert_file_text "$LIVE15/index.html" after-unrecoverable +ok 'rollback failure is explicit, verifiable, and keeps the manual lock' + +# 16. Production exchange capability is probed before snapshot or live mutation. +CASE16="$TMP/case16" +REMOTE16="$CASE16/remote" +mkdir -p "$REMOTE16/custom/public/assets/landing" +printf 'stable\n' > "$REMOTE16/custom/public/assets/landing/index.html" +EXCHANGE_HASH16=$(test_sha256 "$EXCHANGE_HELPER") +if [ "$(uname -s)" = Linux ]; then + bash "$REMOTE_HELPER" lock-acquire "$REMOTE16" custom/public/assets/landing index.html \ + atomic-probe exchange "$EXCHANGE_HELPER" "$EXCHANGE_HASH16" false >/dev/null + [ -d "$REMOTE16/.content-deploy.lock" ] || fail 'successful exchange probe did not retain its lock' + bash "$REMOTE_HELPER" lock-release "$REMOTE16" atomic-probe "$EXCHANGE_HELPER" "$EXCHANGE_HASH16" +else + expect_failure bash "$REMOTE_HELPER" lock-acquire "$REMOTE16" custom/public/assets/landing index.html \ + atomic-probe exchange "$EXCHANGE_HELPER" "$EXCHANGE_HASH16" false + [ ! -e "$REMOTE16/.content-deploy.lock" ] || fail 'failed exchange capability probe left a lock' +fi +[ ! -e "$REMOTE16/.content-backups" ] || fail 'exchange probe created a backup before capability was proven' +assert_file_text "$REMOTE16/custom/public/assets/landing/index.html" stable +ok 'production atomic exchange capability probe is fail-closed before mutation' + +# 17. A second deterministic URL-safe smoke asset is mandatory and manifest-backed. +CASE17="$TMP/case17" +REMOTE17="$CASE17/remote" +LIVE17="$REMOTE17/custom/public/assets/landing" +SOURCE17="$CASE17/source" +mkdir -p "$LIVE17" +printf 'stable\n' > "$LIVE17/index.html" +make_source "$SOURCE17" smoke +bash "$GENERATE" "$SOURCE17" "$CASE17/SHA256SUMS" >/dev/null +write_config "$CASE17/config" "$REMOTE17" "$CASE17/backups" false false assets/not-present.css +expect_failure bash "$PUBLISH" --config "$CASE17/config" --mount landing \ + --source "$SOURCE17" --manifest "$CASE17/SHA256SUMS" +assert_file_text "$LIVE17/index.html" stable +ok 'second smoke asset must be URL-safe, nonempty, and present in the manifest' + +# 18. A branch configured with the local-dot pseudo-remote is not proof that +# content exists in an independently hosted upstream. +expect_failure bash "$PUBLISH" --config "$CONFIG10" --mount landing \ + --provenance-remote . --provenance-ref refs/heads/main \ + --source "$REPO10/overlays/landing" --manifest "$REPO10/overlays/SHA256SUMS" +grep -q 'absolute bare repository path' "$TMP/expected-failure.out" || fail 'local-dot provenance anchor failed for the wrong reason' +ok 'strict provenance rejects a local-dot upstream' + +# 19. Replacement refs are rejected before object provenance is evaluated. +REPLACED_COMMIT=$(git -C "$REPO10" rev-parse HEAD) +REPLACEMENT_COMMIT=$(git -C "$REPO10" rev-parse HEAD^) +git -C "$REPO10" replace "$REPLACED_COMMIT" "$REPLACEMENT_COMMIT" +expect_failure bash "$PUBLISH" --config "$CONFIG10" --mount landing \ + --provenance-remote "$BARE10" --provenance-ref refs/heads/main \ + --source "$REPO10/overlays/landing" --manifest "$REPO10/overlays/SHA256SUMS" +grep -q 'Git replace refs are forbidden' "$TMP/expected-failure.out" || fail 'replacement ref failed for the wrong reason' +git -C "$REPO10" replace -d "$REPLACED_COMMIT" >/dev/null +ok 'strict provenance rejects Git replacement refs' + +# 20. A nonzero exchange result is classified by both directory manifests. If +# the swap happened, the verified rollback runs and releases the lock. +CASE20="$TMP/case20" +REMOTE20="$CASE20/remote" +LIVE20="$REMOTE20/custom/public/assets/landing" +SOURCE20="$CASE20/source" +mkdir -p "$LIVE20" +printf 'before-exchange-error\n' > "$LIVE20/index.html" +printf 'version=1\nrelease_id=previous\n' > "$REMOTE20/.last-content-deploy" +make_source "$SOURCE20" after-exchange-error +bash "$GENERATE" "$SOURCE20" "$CASE20/SHA256SUMS" >/dev/null +write_config "$CASE20/config" "$REMOTE20" "$CASE20/backups" +set +e +CONTENT_TEST_EXCHANGE_FAIL_AFTER_SWAP=1 \ + bash "$PUBLISH" --config "$CASE20/config" --mount landing \ + --source "$SOURCE20" --manifest "$CASE20/SHA256SUMS" --apply > "$CASE20/output" 2>&1 +STATUS20=$? +set -e +[ "$STATUS20" -ne 0 ] || fail 'post-syscall exchange error unexpectedly committed' +assert_file_text "$LIVE20/index.html" before-exchange-error +grep -q '^release_id=previous$' "$REMOTE20/.last-content-deploy" || fail 'exchange-error rollback did not restore the marker' +[ ! -e "$REMOTE20/.content-deploy.lock" ] || fail 'verified exchange-error rollback retained its lock' +PHASE20=$(find "$REMOTE20/.content-backups" -name PHASE -print -quit) +[ -n "$PHASE20" ] || fail 'exchange-error rollback lacks a phase journal' +grep -q '^phase=rolled-back$' "$PHASE20" || fail 'exchange-error rollback did not reach its verified terminal phase' +ok 'post-syscall exchange errors are classified and rolled back safely' + +# 21. Linux-only fault injection proves that renameat2 may mutate both names +# before the helper can report success to its caller. +CASE21="$TMP/case21" +mkdir -p "$CASE21/left" "$CASE21/right" +printf 'left\n' > "$CASE21/left/marker" +printf 'right\n' > "$CASE21/right/marker" +if [ "$(uname -s)" = Linux ]; then + set +e + CONTENT_TEST_RENAME_FAIL_AFTER_SYSCALL=1 \ + python3 "$EXCHANGE_HELPER" exchange "$CASE21/left" "$CASE21/right" > "$CASE21/output" 2>&1 + STATUS21=$? + set -e + [ "$STATUS21" -eq 70 ] || { cat "$CASE21/output" >&2; fail "post-syscall injection returned $STATUS21 instead of 70"; } + assert_file_text "$CASE21/left/marker" right + assert_file_text "$CASE21/right/marker" left + python3 "$EXCHANGE_HELPER" exchange "$CASE21/left" "$CASE21/right" +else + assert_file_text "$CASE21/left/marker" left + assert_file_text "$CASE21/right/marker" right +fi +ok 'rename helper exposes a post-syscall failure injection point' + +# 22. A crashed transaction journal blocks a new deployment even when its +# lock directory is no longer present, while an orderly pre-activation abort +# records a durable terminal phase before releasing the lock. +CASE22="$TMP/case22" +ABORT22="$CASE22/aborted-remote" +mkdir -p "$ABORT22/custom/public/assets/landing" +printf 'stable\n' > "$ABORT22/custom/public/assets/landing/index.html" +EXCHANGE_HASH22=$(test_sha256 "$EXCHANGE_HELPER") +bash "$REMOTE_HELPER" lock-acquire "$ABORT22" \ + custom/public/assets/landing index.html abort-lock local-test \ + "$EXCHANGE_HELPER" "$EXCHANGE_HASH22" false >/dev/null +bash "$REMOTE_HELPER" snapshot "$ABORT22" custom/public/assets/landing index.html \ + abort-lock abort-backup false "$EXCHANGE_HELPER" "$EXCHANGE_HASH22" >/dev/null +bash "$REMOTE_HELPER" abort "$ABORT22" abort-lock abort-backup \ + "$EXCHANGE_HELPER" "$EXCHANGE_HASH22" test-preactivation >/dev/null +grep -q '^phase=aborted$' "$ABORT22/.content-backups/abort-backup/PHASE" || fail 'pre-activation abort lacks a durable terminal phase' +[ ! -e "$ABORT22/.content-deploy.lock" ] || fail 'pre-activation abort retained its lock' +bash "$REMOTE_HELPER" lock-acquire "$ABORT22" \ + custom/public/assets/landing index.html after-abort local-test \ + "$EXCHANGE_HELPER" "$EXCHANGE_HASH22" false >/dev/null +bash "$REMOTE_HELPER" lock-release "$ABORT22" after-abort "$EXCHANGE_HELPER" "$EXCHANGE_HASH22" + +REMOTE22="$CASE22/remote" +mkdir -p "$REMOTE22/custom/public/assets/landing" "$REMOTE22/.content-backups/crashed" +printf 'stable\n' > "$REMOTE22/custom/public/assets/landing/index.html" +printf 'phase=activated\n' > "$REMOTE22/.content-backups/crashed/PHASE" +expect_failure bash "$REMOTE_HELPER" lock-acquire "$REMOTE22" \ + custom/public/assets/landing index.html stale-journal local-test \ + "$EXCHANGE_HELPER" "$EXCHANGE_HASH22" false +grep -q 'stale incomplete transaction' "$TMP/expected-failure.out" || fail 'stale journal failed for the wrong reason' +[ ! -e "$REMOTE22/.content-deploy.lock" ] || fail 'stale journal check created a new lock' +ok 'durable aborts allow retry while stale nonterminal journals block re-entry' + +# 23. The authoritative remote subtree, not a sparse/skip-worktree local view, +# defines exact source coverage. A remote extra file omitted by the manifest +# must fail even when the local working tree hides it and reports clean. +CASE23="$TMP/case23" +REPO23="$CASE23/private" +BARE23="$CASE23/origin.git" +mkdir -p "$CASE23" +init_pushed_repo "$REPO23" "$BARE23" +make_source "$REPO23/overlays/landing" authoritative-extra +bash "$GENERATE" "$REPO23/overlays/landing" "$REPO23/overlays/SHA256SUMS" >/dev/null +printf 'tracked-but-not-manifested\n' > "$REPO23/overlays/landing/remote-extra.txt" +REMOTE23="$CASE23/remote" +mkdir -p "$REMOTE23/custom/public/assets/landing" "$REPO23/deploy" +printf 'stable\n' > "$REMOTE23/custom/public/assets/landing/index.html" +CONFIG23="$REPO23/deploy/production.env" +write_config "$CONFIG23" "$REMOTE23" "$CASE23/backups" false true +commit_and_push "$REPO23" 'remote source contains an extra tracked file' +git -C "$REPO23" update-index --skip-worktree overlays/landing/remote-extra.txt +rm "$REPO23/overlays/landing/remote-extra.txt" +[ -z "$(git -C "$REPO23" status --porcelain --untracked-files=all)" ] \ + || fail 'skip-worktree provenance fixture is unexpectedly dirty' +expect_failure bash "$PUBLISH" --config "$CONFIG23" --mount landing \ + --provenance-remote "$BARE23" --provenance-ref refs/heads/main \ + --source "$REPO23/overlays/landing" --manifest "$REPO23/overlays/SHA256SUMS" +grep -q 'working source lacks a provenance file' "$TMP/expected-failure.out" \ + || fail 'authoritative extra-file fixture failed for the wrong reason' +ok 'authoritative remote subtree defeats sparse or skip-worktree omissions' + +# 24. SSH inputs live in an unpredictable, mode-0700 directory and are checked +# before the deployment lock or any live-content operation can use them. +UPLOAD24=$(bash "$REMOTE_HELPER" upload-create) +[[ "$UPLOAD24" =~ ^/tmp/hackforger-content-upload\.[A-Za-z0-9]+$ ]] \ + || fail 'upload allocator returned an unsafe path' +[ "$(file_mode "$UPLOAD24")" = 700 ] || fail 'upload directory is not mode 0700' +cp "$CASE1/SHA256SUMS" "$UPLOAD24/manifest.SHA256SUMS" +cp "$EXCHANGE_HELPER" "$UPLOAD24/rename-exchange.py" +tar -cf "$UPLOAD24/content.tar" -C "$SOURCE1" . +UPLOAD_ARCHIVE_HASH24=$(test_sha256 "$UPLOAD24/content.tar") +UPLOAD_MANIFEST_HASH24=$(test_sha256 "$UPLOAD24/manifest.SHA256SUMS") +UPLOAD_HELPER_HASH24=$(test_sha256 "$UPLOAD24/rename-exchange.py") +bash "$REMOTE_HELPER" upload-verify "$UPLOAD24" \ + "$UPLOAD_ARCHIVE_HASH24" "$UPLOAD_MANIFEST_HASH24" "$UPLOAD_HELPER_HASH24" +[ "$(file_mode "$UPLOAD24/content.tar")" = 600 ] || fail 'verified upload is not mode 0600' +bash "$REMOTE_HELPER" upload-clean "$UPLOAD24" +[ ! -e "$UPLOAD24" ] || fail 'verified upload directory was not removed' + +UPLOAD_SYMLINK24=$(bash "$REMOTE_HELPER" upload-create) +VICTIM24="$TMP/upload-symlink-victim" +printf 'unchanged\n' > "$VICTIM24" +ln -s "$VICTIM24" "$UPLOAD_SYMLINK24/content.tar" +cp "$CASE1/SHA256SUMS" "$UPLOAD_SYMLINK24/manifest.SHA256SUMS" +cp "$EXCHANGE_HELPER" "$UPLOAD_SYMLINK24/rename-exchange.py" +expect_failure bash "$REMOTE_HELPER" upload-verify "$UPLOAD_SYMLINK24" \ + "$UPLOAD_ARCHIVE_HASH24" "$UPLOAD_MANIFEST_HASH24" "$UPLOAD_HELPER_HASH24" +assert_file_text "$VICTIM24" unchanged +bash "$REMOTE_HELPER" upload-clean "$UPLOAD_SYMLINK24" +ok 'private upload directory rejects symlink pre-placement before lock acquisition' + +echo "All $PASS content publisher tests passed." diff --git a/scripts/check-public-repository-boundary.sh b/scripts/check-public-repository-boundary.sh new file mode 100755 index 0000000000..8790c77700 --- /dev/null +++ b/scripts/check-public-repository-boundary.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BASELINE_REF=${HACKFORGER_BOUNDARY_BASELINE_REF:-refs/remotes/origin/v0.1-dev/hackforger} +[[ "$BASELINE_REF" =~ ^refs/(heads|remotes)/[A-Za-z0-9._/-]+$ \ + || "$BASELINE_REF" =~ ^[0-9a-f]{40}$ \ + || "$BASELINE_REF" =~ ^[0-9a-f]{64}$ ]] \ + || { echo "FATAL: boundary baseline ref has an unsafe form" >&2; exit 2; } +BASELINE_COMMIT=$(git -C "$ROOT" rev-parse --verify "$BASELINE_REF^{commit}" 2>/dev/null) \ + || { echo "FATAL: boundary baseline $BASELINE_REF is unavailable; fetch origin first" >&2; exit 2; } +HEAD_COMMIT=$(git -C "$ROOT" rev-parse --verify 'HEAD^{commit}') +CURRENT_REF=$(git -C "$ROOT" symbolic-ref -q HEAD || printf 'detached/%s\n' "$HEAD_COMMIT") +exec python3 "$ROOT/scripts/ci/check_public_repository_boundary.py" \ + --root "$ROOT" \ + --baseline-ref "$BASELINE_REF" \ + --history-base-ref "$BASELINE_COMMIT" \ + --history-head-ref "$HEAD_COMMIT" \ + --ref-name "$CURRENT_REF" \ + "$@" diff --git a/scripts/ci/boundary_guard_policy.py b/scripts/ci/boundary_guard_policy.py new file mode 100644 index 0000000000..22f1685b0d --- /dev/null +++ b/scripts/ci/boundary_guard_policy.py @@ -0,0 +1,24 @@ +"""Canonical path policy for self-protected public-boundary enforcement.""" + +from __future__ import annotations + + +GUARDED_FILES = { + ".github/CODEOWNERS", + ".github/dependabot.yaml", + ".github/dependabot.yml", + ".agents/skills/hackforger-development", + ".claude/skills/hackforger-development", + "AGENTS.md", + "CODEOWNERS", + "docs/CODEOWNERS", + "scripts/check-public-repository-boundary.sh", + "scripts/install-public-boundary-hook.sh", + "scripts/pre-push-public-boundary.sh", + "skills/hackforger-development/SKILL.md", +} +GUARDED_PREFIXES = (".githooks/", ".github/workflows/", "scripts/ci/") + + +def is_guarded_path(path: str) -> bool: + return path in GUARDED_FILES or path.startswith(GUARDED_PREFIXES) diff --git a/scripts/ci/check_boundary_guard_integrity.py b/scripts/ci/check_boundary_guard_integrity.py new file mode 100644 index 0000000000..569070c360 --- /dev/null +++ b/scripts/ci/check_boundary_guard_integrity.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Require boundary-enforcement files to match the trusted default branch.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +from boundary_guard_policy import is_guarded_path + + + +def tracked_guard_entries(root: Path) -> dict[str, tuple[str, str]]: + proc = subprocess.run( + ["git", "-C", str(root), "ls-files", "--stage", "-z"], + check=True, + stdout=subprocess.PIPE, + ) + entries: dict[str, tuple[str, str]] = {} + for raw in proc.stdout.split(b"\0"): + if not raw: + continue + metadata, path_bytes = raw.split(b"\t", 1) + mode, oid, stage = metadata.decode("ascii").split() + path = path_bytes.decode("utf-8", "surrogateescape") + if stage != "0": + continue + if is_guarded_path(path): + entries[path] = (mode, oid) + return entries + + +def changed_paths(trusted_root: Path, candidate_root: Path) -> list[str]: + trusted = tracked_guard_entries(trusted_root) + candidate = tracked_guard_entries(candidate_root) + return sorted( + path + for path in trusted.keys() | candidate.keys() + if trusted.get(path) != candidate.get(path) + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--trusted-root", type=Path, required=True) + parser.add_argument("--candidate-root", type=Path, required=True) + args = parser.parse_args() + changes = changed_paths(args.trusted_root.resolve(), args.candidate_root.resolve()) + if changes: + print("public boundary guard integrity: FAIL", file=sys.stderr) + for _path in changes: + print("GUARD_CHANGE\tguard-path-redacted", file=sys.stderr) + print( + "Boundary guard changes require a dedicated security-owner review and ruleset bypass.", + file=sys.stderr, + ) + return 1 + print("public boundary guard integrity: PASS") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/check_public_repository_boundary.py b/scripts/ci/check_public_repository_boundary.py new file mode 100755 index 0000000000..49cc5d3e6c --- /dev/null +++ b/scripts/ci/check_public_repository_boundary.py @@ -0,0 +1,972 @@ +#!/usr/bin/env python3 +"""Fail when the public HackForger tree contains private or business content.""" + +from __future__ import annotations + +import argparse +import hashlib +import ipaddress +import os +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +from boundary_guard_policy import is_guarded_path + + +POLICY_REL = "scripts/ci/private-content-markers.txt" + +FORBIDDEN_PATH_PREFIXES = ( + ".claude/projects/", + ".claude/worktrees/", + ".worktrees/", + "custom/public/assets/landing/", + "docs/landing-page/", + "docs/ops/", +) +FORBIDDEN_PATH_EXACT = { + ".claude/scheduled_tasks.lock", + "CLAUDE.local.md", + "scripts/cleanup-invalid-hackathons.sql", +} +FORBIDDEN_BINARY_SUFFIXES = { + ".7z", + ".avi", + ".bin", + ".bmp", + ".bz2", + ".class", + ".db", + ".dmg", + ".doc", + ".docx", + ".eot", + ".gif", + ".gz", + ".ico", + ".jar", + ".jpeg", + ".jpg", + ".mov", + ".mp4", + ".o", + ".otf", + ".pdf", + ".png", + ".rar", + ".sqlite", + ".svg", + ".tar", + ".tgz", + ".ttf", + ".wav", + ".webm", + ".webp", + ".woff", + ".woff2", + ".xls", + ".xlsx", + ".zip", +} +PROJECT_OWNED_PREFIXES = ( + ".agents/", + ".claude/", + ".github/", + "custom/", + "deploy/", + "docs/", + "options/hackforger-help/", + "scripts/", + "skills/", + "templates/hackforger/", +) +REVIEWED_PUBLIC_BINARY_HASHES = { + # Neutral HackForger brand assets. Any byte change requires boundary review. + "custom/public/assets/fonts/manrope-medium.woff2": "19874318747181a650eda439c37955220b849d9c4797c9e0718ee67d4bf929bc", + "custom/public/assets/fonts/manrope-regular.woff2": "849290ef12a2eeb9af5c11924120d11aa4ae8b435ed3347d7fc8bc240c293ca3", + "custom/public/assets/fonts/manrope-semibold.woff2": "f7ac6258da20ab7541939b59851155753d1d24f1b30cbcb949077a3faa3d1593", + "custom/public/assets/fonts/poppins-bold.woff2": "9338e65fc077355c7a87ae0d64cc101e23b9bf8ad78ae65f0f319c857311b526", + "custom/public/assets/fonts/poppins-medium.woff2": "cd36de204aca2d5fa263a731f7c20009b5e3d754ba1f1e03c33e93a48f3e7446", + "custom/public/assets/fonts/poppins-regular.woff2": "7d93459d86585bfcdbb7e0376056226adb25821ee54b96236fe2123e9560929f", + "custom/public/assets/fonts/poppins-semibold.woff2": "f4e80d9dfd374d02989b87a27b5ed4cb78fbb177c27f1478e9a8b0afb7513149", + "custom/public/assets/img/favicon.png": "3a147442d1f4cd1fcb8c6038f994b6efd87be6530e1e92277de28139efc28562", + "custom/public/assets/img/favicon.svg": "9e90f2d49e9d0e03a2ec39f75919ab94774e3d550d8c888223683960261996e6", + "custom/public/assets/img/logo-dark.svg": "9c338a73f1b374aee25ecf4bfcf1b80f5e179ae3e9319bd3a1818885803d91e9", + "custom/public/assets/img/logo-light.svg": "02c8c0e652a76262dc2492b600dd1c69451a5e006aac0d1680d3317ba5499405", + "custom/public/assets/img/logo.png": "71a2a2d7c4cb6ae8742b452680e2ddb04d63476c3afecd086bffe36234a52cc0", + "custom/public/assets/img/logo.svg": "02c8c0e652a76262dc2492b600dd1c69451a5e006aac0d1680d3317ba5499405", +} +IPV4_RE = re.compile( + r"(? tuple[dict[str, IndexEntry], set[str]]: + proc = subprocess.run( + ["git", "-C", str(root), "ls-files", "--stage", "-z"], + check=True, + stdout=subprocess.PIPE, + ) + entries: dict[str, IndexEntry] = {} + unmerged: set[str] = set() + for raw in proc.stdout.split(b"\0"): + if not raw: + continue + metadata, path_bytes = raw.split(b"\t", 1) + mode, oid, stage = metadata.decode("ascii").split() + path = path_bytes.decode("utf-8", "surrogateescape") + if stage != "0": + unmerged.add(path) + continue + entries[path] = IndexEntry(mode, oid) + return entries, unmerged + + +def git_untracked_files(root: Path) -> set[str]: + proc = subprocess.run( + ["git", "-C", str(root), "ls-files", "-z", "--others", "--exclude-standard"], + check=True, + stdout=subprocess.PIPE, + ) + return { + item.decode("utf-8", "surrogateescape") + for item in proc.stdout.split(b"\0") + if item + } + + +def git_object_format(root: Path) -> str: + object_format = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--show-object-format"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + if object_format not in {"sha1", "sha256"}: + raise ValueError(f"unsupported Git object format: {object_format}") + return object_format + + +def read_git_blob(root: Path, oid: str) -> bytes: + if not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", oid): + raise ValueError("invalid Git blob object id") + return subprocess.run( + ["git", "-C", str(root), "cat-file", "blob", oid], + check=True, + stdout=subprocess.PIPE, + ).stdout + + +def load_markers(path: Path) -> tuple[MarkerFingerprint, ...]: + markers: list[MarkerFingerprint] = [] + for raw in path.read_text(encoding="utf-8").splitlines(): + value = raw.strip() + if not value or value.startswith("#"): + continue + parts = value.split() + if len(parts) != 3: + raise ValueError(f"malformed marker fingerprint in {path}") + length_text, rolling_text, digest = parts + if not length_text.isdigit() or int(length_text) < 1: + raise ValueError(f"invalid marker length in {path}") + if not re.fullmatch(r"[0-9a-f]{16}", rolling_text): + raise ValueError(f"invalid rolling hash in {path}") + if not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError(f"invalid SHA-256 in {path}") + markers.append(MarkerFingerprint(int(length_text), int(rolling_text, 16), digest)) + if not markers: + raise ValueError(f"marker policy is empty: {path}") + return tuple(markers) + + +def contains_marker(value: str, markers: tuple[MarkerFingerprint, ...]) -> bool: + data = value.lower().encode("utf-8", "surrogateescape") + mask = (1 << 64) - 1 + base = 257 + by_length: dict[int, dict[int, set[str]]] = {} + for marker in markers: + by_length.setdefault(marker.length, {}).setdefault(marker.rolling_hash, set()).add(marker.sha256) + + for length, candidates in by_length.items(): + if len(data) < length: + continue + factor = pow(base, length - 1, 1 << 64) + rolling = 0 + for byte in data[:length]: + rolling = ((rolling * base) + byte) & mask + for start in range(0, len(data) - length + 1): + if start: + rolling = (rolling - (data[start - 1] * factor)) & mask + rolling = ((rolling * base) + data[start + length - 1]) & mask + digests = candidates.get(rolling) + if digests and hashlib.sha256(data[start : start + length]).hexdigest() in digests: + return True + return False + + +def allowed_ip(value: str) -> bool: + try: + address = ipaddress.ip_address(value) + except ValueError: + return False + allowed_networks = ( + ipaddress.ip_network("0.0.0.0/32"), + ipaddress.ip_network("127.0.0.0/8"), + ipaddress.ip_network("192.0.2.0/24"), + ipaddress.ip_network("198.51.100.0/24"), + ipaddress.ip_network("203.0.113.0/24"), + ) + return any(address in network for network in allowed_networks) + + +def ipv6_candidates(text: str) -> set[str]: + candidates = {match.group(1) for match in IPV6_BRACKET_RE.finditer(text)} + candidates.update(match.group(1) for match in IPV6_BARE_RE.finditer(text)) + return {candidate for candidate in candidates if candidate.count(":") >= 2} + + +def allowed_ipv6(value: str) -> bool: + address_text = value.split("%", 1)[0] + try: + address = ipaddress.ip_address(address_text) + except ValueError: + return True + if not isinstance(address, ipaddress.IPv6Address): + return True + allowed_networks = ( + ipaddress.ip_network("::/128"), + ipaddress.ip_network("::1/128"), + ipaddress.ip_network("2001:db8::/32"), + ) + return any(address in network for network in allowed_networks) + + +def path_findings(path: str, markers: tuple[MarkerFingerprint, ...]) -> list[Finding]: + findings: list[Finding] = [] + lower = path.lower() + parts = Path(lower).parts + if lower in {value.lower() for value in FORBIDDEN_PATH_EXACT} or lower.startswith( + tuple(value.lower() for value in FORBIDDEN_PATH_PREFIXES) + ): + findings.append(Finding("BOUNDARY_PATH", path)) + if ".local." in lower or any(part.endswith(".local") for part in parts): + findings.append(Finding("LOCAL_FILE", path)) + if ( + ".internal." in lower + or any(part.endswith(".internal") for part in parts) + or lower.startswith("private-content/") + or "/private-content/" in lower + ): + findings.append(Finding("PRIVATE_FILE", path)) + if contains_marker(path, markers): + findings.append(Finding("BUSINESS_MARKER", path)) + name = Path(path).name.lower() + if name.startswith(".env") and not name.endswith((".example", ".template")): + findings.append(Finding("ENV_FILE", path)) + if lower.startswith("docs/tests/") and Path(lower).suffix in FORBIDDEN_BINARY_SUFFIXES: + findings.append(Finding("RUNTIME_EVIDENCE", path)) + if "__pycache__" in parts or Path(lower).suffix in {".pyc", ".pyo"}: + findings.append(Finding("GENERATED_ARTIFACT", path)) + return findings + + +SENSITIVE_PATH_RULES = {"BUSINESS_MARKER", "PRIVATE_FILE"} + + +def path_metadata_findings( + path: str, markers: tuple[MarkerFingerprint, ...] +) -> tuple[set[Finding], str]: + """Scan a Git path as metadata and return findings plus a safe display name.""" + direct = path_findings(path, markers) + global_findings = global_content_findings("path-name.md", path, markers) + redact = bool(global_findings) or any( + finding.rule in SENSITIVE_PATH_RULES for finding in direct + ) + display = "path-redacted" if redact else path + findings = {Finding(finding.rule, display) for finding in direct} + findings.update( + Finding(f"PATH_{finding.rule}", "path-redacted") + for finding in global_findings + ) + return findings, display + + +def credential_is_safe(value: str) -> bool: + normalized = value.strip().lower() + return normalized.startswith(SAFE_CREDENTIAL_PREFIXES) or normalized in SAFE_CREDENTIAL_VALUES + + +def is_lfs_pointer(data: bytes) -> bool: + if len(data) > 1024: + return False + lines = data.decode("ascii", "ignore").splitlines() + return ( + len(lines) >= 3 + and lines[0] == "version https://git-lfs.github.com/spec/v1" + and re.fullmatch(r"oid sha256:[0-9a-f]{64}", lines[1]) is not None + and re.fullmatch(r"size [0-9]+", lines[2]) is not None + ) + + +def should_scan_credentials(path: str, text_hash: str) -> bool: + lower = path.lower() + if lower.startswith("options/locale/"): + return False + expected_fixture_hash = CREDENTIAL_EXEMPT_HASHES.get(path) + if expected_fixture_hash == text_hash: + return False + name = Path(lower).name + return ( + name in CREDENTIAL_SCAN_NAMES + or name.startswith(".env") + or Path(lower).suffix in CREDENTIAL_SCAN_SUFFIXES + ) + + +def locale_internal_key_context(path: str, text: str, start: int, end: int) -> bool: + if path in {"path-name.md", "ref-name.md", "commit-metadata.md"}: + return False + value = text[start:end] + if not re.fullmatch( + r"(?i)[a-z0-9_.-]+\.(?:error|desc)\.internal", value + ): + return False + line_start = text.rfind("\n", 0, start) + 1 + line_end = text.find("\n", end) + if line_end < 0: + line_end = len(text) + line = text[line_start:line_end] + relative_start = start - line_start + relative_end = end - line_start + if path.startswith("options/locale/") and re.fullmatch( + rf"\s*{re.escape(value)}\s*=.*", line + ): + return True + before = line[:relative_start] + after = line[relative_end:] + quoted = bool(before) and bool(after) and before[-1] in {'"', "'"} and after[0] == before[-1] + return quoted and re.search( + r"(?:\bTr(?:String)?\s*\(\s*|\bLocale\.Tr\s+)[\"']$", before + ) is not None + + +def contains_internal_host(path: str, text: str) -> bool: + for match in INTERNAL_HOST_RE.finditer(text): + if locale_internal_key_context(path, text, match.start(), match.end()): + continue + return True + return False + + +def global_content_findings( + path: str, text: str, markers: tuple[MarkerFingerprint, ...] +) -> list[Finding]: + findings: list[Finding] = [] + text_hash = hashlib.sha256(text.encode("utf-8", "surrogateescape")).hexdigest() + if contains_marker(text, markers): + findings.append(Finding("BUSINESS_MARKER", path)) + if PRIVATE_KEY_RE.search(text) or TOKEN_RE.search(text) or SLACK_WEBHOOK_RE.search(text): + expected_fixture_hash = SECRET_MATERIAL_EXEMPT_HASHES.get(path) + if expected_fixture_hash != text_hash: + findings.append(Finding("SECRET_MATERIAL", path)) + if PERSONAL_HOME_RE.search(text): + expected_personal_fixture_hash = PERSONAL_PATH_EXEMPT_HASHES.get(path) + if expected_personal_fixture_hash != text_hash: + findings.append(Finding("PERSONAL_PATH", path)) + if contains_internal_host(path, text): + expected_internal_fixture_hash = INTERNAL_HOST_EXEMPT_HASHES.get(path) + if expected_internal_fixture_hash != text_hash: + findings.append(Finding("INTERNAL_HOST", path)) + if should_scan_credentials(path, text_hash): + for match in CREDENTIAL_RE.finditer(text): + if not credential_is_safe(match.group(1)): + findings.append(Finding("LITERAL_CREDENTIAL", path)) + break + expected_ip_fixture_hash = IP_EXEMPT_HASHES.get(path) + ip_exempt = Path(path).suffix.lower() == ".svg" or expected_ip_fixture_hash == text_hash + if not ip_exempt: + for match in IPV4_RE.finditer(text): + if not allowed_ip(match.group(0)): + findings.append(Finding("NON_DOCUMENTATION_IP", path)) + break + expected_ipv6_fixture_hash = IPV6_EXEMPT_HASHES.get(path) + if expected_ipv6_fixture_hash != text_hash: + for candidate in ipv6_candidates(text): + if not allowed_ipv6(candidate): + findings.append(Finding("NON_DOCUMENTATION_IPV6", path)) + break + return findings + + +def load_git_baseline(root: Path, ref: str) -> tuple[dict[str, IndexEntry], str]: + if not re.fullmatch( + r"(?:refs/(?:heads|remotes)/[A-Za-z0-9._/-]+|[0-9a-f]{40}|[0-9a-f]{64})", + ref, + ): + raise ValueError("baseline ref must be a full heads/remotes ref or commit object id") + commit = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--verify", f"{ref}^{{commit}}"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + object_format = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--show-object-format"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + if object_format not in {"sha1", "sha256"}: + raise ValueError(f"unsupported Git object format: {object_format}") + tree = subprocess.run( + ["git", "-C", str(root), "ls-tree", "-r", "-z", commit], + check=True, + stdout=subprocess.PIPE, + ).stdout + blobs: dict[str, IndexEntry] = {} + for raw in tree.split(b"\0"): + if not raw: + continue + metadata, path_bytes = raw.split(b"\t", 1) + mode, object_type, oid = metadata.decode("ascii").split() + if object_type == "blob": + blobs[path_bytes.decode("utf-8", "surrogateescape")] = IndexEntry(mode, oid) + return blobs, object_format + + +def git_blob_oid(data: bytes, object_format: str) -> str: + framed = f"blob {len(data)}\0".encode("ascii") + data + return hashlib.new(object_format, framed).hexdigest() + + +def baseline_differs( + baseline_root: Path | None, + baseline_blobs: dict[str, IndexEntry] | None, + baseline_object_format: str | None, + rel: str, + data: bytes, +) -> bool: + if baseline_root is not None: + baseline = baseline_root / rel + if baseline.is_symlink() or not baseline.is_file(): + return True + try: + return baseline.read_bytes() != data + except OSError: + return True + if baseline_blobs is None or baseline_object_format is None: + raise ValueError("an opaque-file baseline is required") + entry = baseline_blobs.get(rel) + return entry is None or entry.oid != git_blob_oid(data, baseline_object_format) + + +def blob_findings( + path: str, + data: bytes, + markers: tuple[MarkerFingerprint, ...], + baseline_root: Path | None, + baseline_blobs: dict[str, IndexEntry] | None, + baseline_object_format: str | None, +) -> set[Finding]: + findings: set[Finding] = set() + lower = path.lower() + content_hash = hashlib.sha256(data).hexdigest() + if is_lfs_pointer(data): + findings.add(Finding("LFS_POINTER", path)) + try: + text = data.decode("utf-8") + invalid_utf8 = False + except UnicodeDecodeError: + text = "" + invalid_utf8 = True + opaque = ( + b"\0" in data + or invalid_utf8 + or Path(lower).suffix in FORBIDDEN_BINARY_SUFFIXES + ) + owned_opaque = lower.startswith(PROJECT_OWNED_PREFIXES) and opaque + changed_opaque = opaque and baseline_differs( + baseline_root, + baseline_blobs, + baseline_object_format, + path, + data, + ) + if owned_opaque or changed_opaque: + expected = REVIEWED_PUBLIC_BINARY_HASHES.get(lower) + if expected is None or content_hash != expected: + findings.add(Finding("UNREVIEWED_BINARY", path)) + if not opaque: + findings.update(global_content_findings(path, text, markers)) + return findings + + +def resolve_commit(root: Path, value: str) -> str: + if not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", value): + raise ValueError("history endpoints must be commit object ids") + commit = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--verify", f"{value}^{{commit}}"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + return commit + + +def history_range(root: Path, base: str, head: str) -> list[str]: + base_commit = resolve_commit(root, base) + head_commit = resolve_commit(root, head) + output = subprocess.run( + ["git", "-C", str(root), "rev-list", "--reverse", f"{base_commit}..{head_commit}"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout + return [line for line in output.splitlines() if line] + + +def changed_commit_paths(root: Path, commit: str) -> list[str]: + parents = subprocess.run( + ["git", "-C", str(root), "rev-list", "--parents", "-n", "1", commit], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.split() + command = [ + "git", + "-C", + str(root), + "diff-tree", + "--no-commit-id", + "--no-renames", + "--name-only", + "-r", + "-z", + ] + if len(parents) > 1: + command.extend((parents[1], commit)) + else: + command.extend(("--root", commit)) + output = subprocess.run(command, check=True, stdout=subprocess.PIPE).stdout + return [ + item.decode("utf-8", "surrogateescape") + for item in output.split(b"\0") + if item + ] + + +def commit_tree_entry(root: Path, commit: str, path: str) -> IndexEntry | None: + output = subprocess.run( + ["git", "-C", str(root), "ls-tree", "-z", commit, "--", f":(literal){path}"], + check=True, + stdout=subprocess.PIPE, + ).stdout + records = [record for record in output.split(b"\0") if record] + if not records: + return None + metadata, _path = records[0].split(b"\t", 1) + _mode, object_type, oid = metadata.decode("ascii").split() + if object_type not in {"blob", "commit"}: + raise ValueError(f"unsupported Git object type in history: {object_type}") + return IndexEntry(_mode, oid) + + +def commit_blob(root: Path, entry: IndexEntry | None) -> bytes | None: + if entry is None or entry.mode == "160000": + return None + return read_git_blob(root, entry.oid) + + +def filesystem_git_entry(root: Path, path: str, object_format: str) -> IndexEntry | None: + full = root / path + try: + if full.is_symlink(): + data = os.readlink(full).encode("utf-8", "surrogateescape") + mode = "120000" + elif full.is_file(): + data = full.read_bytes() + mode = "100755" if full.stat().st_mode & 0o111 else "100644" + else: + return None + except OSError: + return None + return IndexEntry(mode, git_blob_oid(data, object_format)) + + +def guard_entry_matches_baseline( + path: str, + entry: IndexEntry | None, + root_object_format: str, + baseline_root: Path | None, + baseline_blobs: dict[str, IndexEntry] | None, +) -> bool: + if baseline_root is not None: + expected = filesystem_git_entry(baseline_root, path, root_object_format) + elif baseline_blobs is not None: + expected = baseline_blobs.get(path) + else: + raise ValueError("an opaque-file baseline is required") + return entry == expected + + +def raw_commit_metadata(root: Path, commit: str) -> str: + raw = subprocess.run( + ["git", "-C", str(root), "cat-file", "commit", commit], + check=True, + stdout=subprocess.PIPE, + ).stdout + return raw.decode("utf-8", "surrogateescape") + + +def scan_history( + root: Path, + commits: list[str], + markers: tuple[MarkerFingerprint, ...], + baseline_root: Path | None, + baseline_blobs: dict[str, IndexEntry] | None, + baseline_object_format: str | None, +) -> set[Finding]: + findings: set[Finding] = set() + root_object_format = git_object_format(root) + for commit in commits: + short = commit[:12] + metadata = raw_commit_metadata(root, commit) + for finding in global_content_findings("commit-metadata.md", metadata, markers): + findings.add(Finding(f"HISTORY_{finding.rule}", f"{short}:COMMIT_METADATA")) + for path in changed_commit_paths(root, commit): + entry = commit_tree_entry(root, commit, path) + if is_guarded_path(path) and not guard_entry_matches_baseline( + path, + entry, + root_object_format, + baseline_root, + baseline_blobs, + ): + findings.add( + Finding("HISTORY_GUARD_CHANGE", f"{short}:guard-path-redacted") + ) + # Deleting a path reduces public exposure. If it was added earlier + # in this outgoing range, that earlier blob/path is scanned at its + # own commit; a deletion must not block cleanup of legacy content. + if entry is None: + continue + metadata_findings, display = path_metadata_findings(path, markers) + for finding in metadata_findings: + findings.add(Finding(f"HISTORY_{finding.rule}", f"{short}:{display}")) + data = commit_blob(root, entry) + if data is None: + continue + for finding in blob_findings( + path, + data, + markers, + baseline_root, + baseline_blobs, + baseline_object_format, + ): + findings.add(Finding(f"HISTORY_{finding.rule}", f"{short}:{display}")) + return findings + + +def scan( + root: Path, + policy: Path, + baseline_root: Path | None = None, + baseline_blobs: dict[str, IndexEntry] | None = None, + baseline_object_format: str | None = None, + history_commits: list[str] | None = None, +) -> list[Finding]: + markers = load_markers(policy) + findings: set[Finding] = set() + object_format = git_object_format(root) + index_entries, unmerged = git_index_entries(root) + untracked = git_untracked_files(root) + for rel in unmerged: + metadata_findings, display = path_metadata_findings(rel, markers) + findings.update(metadata_findings) + findings.add(Finding("UNMERGED_INDEX", display)) + for rel in sorted(index_entries.keys() | untracked): + metadata_findings, display = path_metadata_findings(rel, markers) + findings.update(metadata_findings) + full = root / rel + worktree_data: bytes | None = None + if full.is_symlink(): + worktree_data = os.readlink(full).encode("utf-8", "surrogateescape") + elif full.is_file(): + try: + worktree_data = full.read_bytes() + except OSError: + findings.add(Finding("UNREADABLE_FILE", display)) + elif full.exists(): + entry = index_entries.get(rel) + if entry is None or entry.mode != "160000": + findings.add(Finding("NON_REGULAR_WORKTREE", display)) + + versions: list[bytes] = [] + entry = index_entries.get(rel) + if entry is not None: + if entry.mode in {"100644", "100755", "120000"}: + if worktree_data is None or git_blob_oid(worktree_data, object_format) != entry.oid: + versions.append(read_git_blob(root, entry.oid)) + elif entry.mode != "160000": + findings.add(Finding("NON_BLOB_INDEX", display)) + if worktree_data is not None: + versions.append(worktree_data) + + seen_versions: set[str] = set() + for data in versions: + digest = hashlib.sha256(data).hexdigest() + if digest in seen_versions: + continue + seen_versions.add(digest) + findings.update( + Finding(finding.rule, display) + for finding in blob_findings( + rel, + data, + markers, + baseline_root, + baseline_blobs, + baseline_object_format, + ) + ) + if history_commits: + findings.update( + scan_history( + root, + history_commits, + markers, + baseline_root, + baseline_blobs, + baseline_object_format, + ) + ) + return sorted(findings) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--policy", type=Path) + baseline = parser.add_mutually_exclusive_group(required=True) + baseline.add_argument("--baseline-root", type=Path) + baseline.add_argument("--baseline-ref") + parser.add_argument("--history-base-ref") + parser.add_argument("--history-head-ref") + parser.add_argument("--history-commit-list", type=Path) + parser.add_argument("--history-only", action="store_true") + parser.add_argument("--ref-name", action="append", default=[]) + args = parser.parse_args() + root = args.root.resolve() + policy = (args.policy or root / POLICY_REL).resolve() + baseline_root = args.baseline_root.resolve() if args.baseline_root else None + baseline_blobs = None + baseline_object_format = None + if args.baseline_ref: + try: + baseline_blobs, baseline_object_format = load_git_baseline(root, args.baseline_ref) + except (ValueError, subprocess.CalledProcessError) as error: + print( + f"public repository boundary: invalid baseline ({type(error).__name__})", + file=sys.stderr, + ) + return 2 + history_commits: list[str] = [] + try: + if args.history_commit_list: + if args.history_base_ref or args.history_head_ref: + raise ValueError("history commit list cannot be combined with a history range") + raw_commits = args.history_commit_list.read_text(encoding="ascii").splitlines() + history_commits = list(dict.fromkeys(resolve_commit(root, value) for value in raw_commits if value)) + elif args.history_base_ref or args.history_head_ref: + if not args.history_base_ref or not args.history_head_ref: + raise ValueError("history base and head refs must be provided together") + history_commits = history_range(root, args.history_base_ref, args.history_head_ref) + if args.history_only: + if not (args.history_commit_list or args.history_base_ref): + raise ValueError("history-only scanning requires a commit list or range") + findings = sorted( + scan_history( + root, + history_commits, + load_markers(policy), + baseline_root, + baseline_blobs, + baseline_object_format, + ) + ) + else: + findings = scan( + root, + policy, + baseline_root, + baseline_blobs, + baseline_object_format, + history_commits, + ) + metadata_findings = set(findings) + markers = load_markers(policy) + for ref_name in args.ref_name: + if not ref_name or len(ref_name) > 1024 or re.search(r"[\x00-\x20\x7f]", ref_name): + metadata_findings.add(Finding("UNSAFE_REF_NAME", "ref-name-redacted")) + continue + for finding in path_findings(ref_name, markers): + metadata_findings.add(Finding(f"REF_{finding.rule}", "ref-name-redacted")) + for finding in global_content_findings("ref-name.md", ref_name, markers): + metadata_findings.add(Finding(f"REF_{finding.rule}", "ref-name-redacted")) + findings = sorted(metadata_findings) + except (OSError, ValueError, subprocess.CalledProcessError) as error: + print( + f"public repository boundary: scan error ({type(error).__name__})", + file=sys.stderr, + ) + return 2 + if findings: + print("public repository boundary: FAIL", file=sys.stderr) + for finding in findings: + print(f"{finding.rule}\t{finding.path}", file=sys.stderr) + print(f"total findings: {len(findings)}", file=sys.stderr) + return 1 + print("public repository boundary: PASS") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/fingerprint_private_content_marker.py b/scripts/ci/fingerprint_private_content_marker.py new file mode 100755 index 0000000000..828a5cb944 --- /dev/null +++ b/scripts/ci/fingerprint_private_content_marker.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Create a non-plaintext marker fingerprint for the boundary policy.""" + +from __future__ import annotations + +import getpass +import hashlib + + +def main() -> int: + marker = getpass.getpass("Private/business marker (input hidden): ").strip().lower() + if not marker or any(ord(character) > 127 for character in marker): + raise SystemExit("marker must be non-empty ASCII") + data = marker.encode("ascii") + rolling = 0 + for byte in data: + rolling = ((rolling * 257) + byte) & ((1 << 64) - 1) + print(f"{len(data)} {rolling:016x} {hashlib.sha256(data).hexdigest()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/private-content-markers.txt b/scripts/ci/private-content-markers.txt new file mode 100644 index 0000000000..a45a0b2b2d --- /dev/null +++ b/scripts/ci/private-content-markers.txt @@ -0,0 +1,10 @@ +# Fingerprints of case-insensitive business markers forbidden in paths/content. +# Format: byte_length rolling_hash_64 sha256. Plaintext business or topology +# values must never be stored in this public policy file. Use the sibling +# fingerprint_private_content_marker.py helper, which reads the value hidden. +10 98c78d2b9aa4ea63 3a4e46b4b9f7bfcbf978a33a4f187897a52ac5ed5676a6d3069e502ccdef6e16 +10 ca5e735826d0ffc1 b44a8c9b15c4741a07b874b9f1ce88fd36e01b97b390219d2113faa576041294 +8 6b1436dcc22d661e 250b879bc5db23dab916633b7e476761993ed7bee763e28547f227828a47d39f +10 24b329b668b98844 e4f6167405a8233dcf22b46fa6ca994393eabda31823c16389e6c774ad40b342 +8 728535f5c6667639 7639c3ff04ac901c8feb34ceb7c122771569c5d3c183dd5659f065c1717d063a +16 cb44c0d79f961299 a73820a009aaefcc7e9fabe8247ec6ccf599e1dccbdf37ab086f81a82d115fe3 diff --git a/scripts/ci/test_boundary_guard_integrity.py b/scripts/ci/test_boundary_guard_integrity.py new file mode 100644 index 0000000000..088eb6261d --- /dev/null +++ b/scripts/ci/test_boundary_guard_integrity.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SCRIPT_DIR = Path(__file__).resolve().parent +CHECKER = SCRIPT_DIR / "check_boundary_guard_integrity.py" + + +class BoundaryGuardIntegrityTest(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.trusted = self.root / "trusted" + self.candidate = self.root / "candidate" + for repo in (self.trusted, self.candidate): + subprocess.run(["git", "init", "-q", str(repo)], check=True) + self.write(repo, "CODEOWNERS", "* @example/reviewer\n") + self.write(repo, ".github/workflows/boundary.yml", "name: boundary\n") + self.write(repo, "scripts/check-public-repository-boundary.sh", "exit 0\n") + self.write(repo, "scripts/ci/checker.py", "print('checked')\n") + + def tearDown(self) -> None: + self.temp.cleanup() + + @staticmethod + def write(repo: Path, rel: str, content: str) -> None: + path = repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + subprocess.run(["git", "-C", str(repo), "add", rel], check=True) + + def run_checker(self) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "python3", + str(CHECKER), + "--trusted-root", + str(self.trusted), + "--candidate-root", + str(self.candidate), + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + def test_identical_guard_and_unrelated_changes_pass(self) -> None: + self.write(self.candidate, "README.md", "candidate documentation\n") + self.assertEqual(0, self.run_checker().returncode) + + def test_modified_deleted_or_added_guard_file_fails(self) -> None: + self.write(self.candidate, "scripts/ci/checker.py", "print('disabled')\n") + result = self.run_checker() + self.assertIn("GUARD_CHANGE\tguard-path-redacted", result.stderr) + + subprocess.run( + ["git", "-C", str(self.candidate), "rm", "-f", "CODEOWNERS"], + check=True, + stdout=subprocess.DEVNULL, + ) + self.write(self.candidate, ".github/workflows/spoof.yml", "name: spoof\n") + address = "8" + ".8.8.8" + self.write( + self.candidate, + f".github/workflows/prod-{address}.yml", + "name: sensitive metadata\n", + ) + self.write(self.candidate, ".github/CODEOWNERS", "* @attacker\n") + self.write(self.candidate, ".github/dependabot.yml", "version: 2\n") + self.write(self.candidate, ".github/dependabot.yaml", "version: 2\n") + self.write(self.candidate, ".githooks/pre-push", "exit 0\n") + self.write(self.candidate, "scripts/pre-push-public-boundary.sh", "exit 0\n") + self.write(self.candidate, "docs/CODEOWNERS", "* @attacker\n") + result = self.run_checker() + self.assertGreaterEqual( + result.stderr.count("GUARD_CHANGE\tguard-path-redacted"), + 8, + ) + self.assertNotIn("spoof.yml", result.stderr) + self.assertNotIn("dependabot.yml", result.stderr) + self.assertNotIn(address, result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/test_hook_installer.py b/scripts/ci/test_hook_installer.py new file mode 100755 index 0000000000..6e681b63bd --- /dev/null +++ b/scripts/ci/test_hook_installer.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import os +import shlex +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SCRIPT_DIR = Path(__file__).resolve().parent +SOURCE_ROOT = SCRIPT_DIR.parent.parent +INSTALLER = SOURCE_ROOT / "scripts/install-public-boundary-hook.sh" +PRE_PUSH = SOURCE_ROOT / "scripts/pre-push-public-boundary.sh" +CHECKER = SCRIPT_DIR / "check_public_repository_boundary.py" +GUARD_POLICY = SCRIPT_DIR / "boundary_guard_policy.py" +POLICY = SCRIPT_DIR / "private-content-markers.txt" +CANONICAL_ORIGIN = "git@github.com:HackForger/hackforger.git" + + +class HookInstallerTest(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.base = Path(self.temp.name) + self.repo = self.base / "repo" + self.remote = self.base / "origin.git" + subprocess.run(["git", "init", "-q", "--bare", str(self.remote)], check=True) + subprocess.run(["git", "init", "-q", str(self.repo)], check=True) + subprocess.run(["git", "-C", str(self.repo), "checkout", "-q", "-b", "main"], check=True) + subprocess.run(["git", "-C", str(self.repo), "remote", "add", "origin", CANONICAL_ORIGIN], check=True) + fake_ssh = self.base / "fake-ssh" + fake_ssh.write_text( + "#!/bin/sh\n" + "case \"$*\" in\n" + f" *git-upload-pack*) exec git-upload-pack {shlex.quote(str(self.remote))} ;;\n" + f" *git-receive-pack*) exec git-receive-pack {shlex.quote(str(self.remote))} ;;\n" + " *) exit 97 ;;\n" + "esac\n", + encoding="utf-8", + ) + fake_ssh.chmod(0o755) + self.env = os.environ.copy() + self.env["GIT_SSH_COMMAND"] = str(fake_ssh) + self.write("README.md", "neutral pre-hook worktree\n") + self.pre_hook = self.commit("pre-hook baseline") + self.copy_source(INSTALLER, "scripts/install-public-boundary-hook.sh", 0o755) + self.copy_source(PRE_PUSH, "scripts/pre-push-public-boundary.sh", 0o755) + self.copy_source(CHECKER, "scripts/ci/check_public_repository_boundary.py", 0o755) + self.copy_source(GUARD_POLICY, "scripts/ci/boundary_guard_policy.py", 0o644) + self.copy_source(POLICY, "scripts/ci/private-content-markers.txt", 0o644) + self.baseline = self.commit("trusted default") + subprocess.run( + ["git", "-C", str(self.repo), "push", "-q", "origin", "main"], + check=True, + env=self.env, + ) + subprocess.run( + ["git", "-C", str(self.remote), "symbolic-ref", "HEAD", "refs/heads/main"], + check=True, + ) + self.common_dir = Path( + subprocess.run( + ["git", "-C", str(self.repo), "rev-parse", "--path-format=absolute", "--git-common-dir"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + ) + + def tearDown(self) -> None: + self.temp.cleanup() + + def copy_source(self, source: Path, rel: str, mode: int) -> None: + target = self.repo / rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + target.chmod(mode) + subprocess.run(["git", "-C", str(self.repo), "add", rel], check=True) + + def write(self, rel: str, content: str) -> None: + target = self.repo / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + subprocess.run(["git", "-C", str(self.repo), "add", rel], check=True) + + def commit(self, message: str) -> str: + subprocess.run( + [ + "git", + "-C", + str(self.repo), + "-c", + "user.name=Boundary Test", + "-c", + "user.email=boundary@example.invalid", + "commit", + "-q", + "-m", + message, + ], + check=True, + ) + return subprocess.run( + ["git", "-C", str(self.repo), "rev-parse", "HEAD"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + + def run_installer(self, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [str(self.repo / "scripts/install-public-boundary-hook.sh")], + cwd=self.repo, + env=env or self.env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + def hooks_path(self) -> Path: + return Path( + subprocess.run( + ["git", "-C", str(self.repo), "config", "--get", "core.hooksPath"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + ) + + def test_installs_remote_default_bundle_not_checkout_controlled_hook(self) -> None: + checkout_hook = self.repo / "scripts/pre-push-public-boundary.sh" + checkout_hook.write_text("#!/bin/sh\nexit 77\n", encoding="utf-8") + result = self.run_installer() + self.assertEqual(0, result.returncode, result.stderr) + + hooks_path = self.hooks_path() + self.assertTrue(hooks_path.is_absolute()) + self.assertEqual(self.common_dir / "hackforger-boundary-hooks" / self.baseline, hooks_path) + effective = subprocess.run( + ["git", "-C", str(self.repo), "rev-parse", "--path-format=absolute", "--git-path", "hooks"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + self.assertEqual(str(hooks_path), effective) + self.assertTrue((hooks_path / "pre-push").is_file()) + self.assertTrue((hooks_path / "pre-push").stat().st_mode & 0o111) + self.assertEqual(0, hooks_path.stat().st_mode & 0o222) + self.assertEqual(0, (hooks_path / "pre-push").stat().st_mode & 0o222) + trusted_hook = subprocess.run( + ["git", "-C", str(self.repo), "show", f"{self.baseline}:scripts/pre-push-public-boundary.sh"], + check=True, + stdout=subprocess.PIPE, + ).stdout + self.assertEqual(trusted_hook, (hooks_path / "pre-push").read_bytes()) + self.assertNotEqual(checkout_hook.read_bytes(), (hooks_path / "pre-push").read_bytes()) + + def test_installer_ignores_git_environment_repository_overlays(self) -> None: + decoy = self.base / "decoy" + subprocess.run(["git", "init", "-q", str(decoy)], check=True) + env = self.env.copy() + env.update( + { + "GIT_DIR": str(decoy / ".git"), + "GIT_WORK_TREE": str(decoy), + "GIT_IMPLICIT_WORK_TREE": "0", + "GIT_OBJECT_DIRECTORY": str(decoy / ".git/objects"), + "GIT_ALTERNATE_OBJECT_DIRECTORIES": str(decoy / ".git/objects"), + "GIT_INDEX_FILE": str(decoy / ".git/index"), + "GIT_NAMESPACE": "decoy", + "GIT_NO_REPLACE_OBJECTS": "0", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "remote.origin.url", + "GIT_CONFIG_VALUE_0": "https://github.com/example/other.git", + "GIT_CONFIG_GLOBAL": str(decoy / "config"), + "GIT_EXEC_PATH": str(decoy), + "GIT_INTERNAL_SUPER_PREFIX": "decoy/", + } + ) + result = self.run_installer(env) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(self.common_dir / "hackforger-boundary-hooks" / self.baseline, self.hooks_path()) + + def test_explicit_canonical_sample_only_hooks_path_can_switch(self) -> None: + canonical = self.common_dir / "hooks" + subprocess.run( + ["git", "-C", str(self.repo), "config", "core.hooksPath", str(canonical)], + check=True, + ) + result = self.run_installer() + self.assertEqual(0, result.returncode, result.stderr) + self.assertNotEqual(canonical, self.hooks_path()) + + def test_installed_hook_blocks_sensitive_push_from_pre_hook_worktree(self) -> None: + result = self.run_installer() + self.assertEqual(0, result.returncode, result.stderr) + legacy = self.base / "legacy-worktree" + subprocess.run( + ["git", "-C", str(self.repo), "worktree", "add", "-q", "-b", "legacy", str(legacy), self.pre_hook], + check=True, + ) + marker = "synno" + "vator" + sensitive = legacy / "legacy-note.md" + sensitive.write_text(f"tenant: {marker}\n", encoding="utf-8") + subprocess.run(["git", "-C", str(legacy), "add", "legacy-note.md"], check=True) + subprocess.run( + [ + "git", + "-C", + str(legacy), + "-c", + "user.name=Boundary Test", + "-c", + "user.email=boundary@example.invalid", + "commit", + "-q", + "-m", + "legacy sensitive change", + ], + check=True, + ) + push = subprocess.run( + ["git", "-C", str(legacy), "push", "origin", "legacy"], + env=self.env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertNotEqual(0, push.returncode) + self.assertIn("must contain the installed boundary source commit", push.stderr) + remote_ref = subprocess.run( + ["git", "-C", str(self.remote), "show-ref", "--verify", "refs/heads/legacy"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertNotEqual(0, remote_ref.returncode) + + def test_guard_change_fails_closed_until_atomic_reinstall(self) -> None: + first_install = self.run_installer() + self.assertEqual(0, first_install.returncode, first_install.stderr) + old_hooks_path = self.hooks_path() + + checker = self.repo / "scripts/ci/check_public_repository_boundary.py" + checker.write_text(checker.read_text(encoding="utf-8") + "\n# guard revision\n", encoding="utf-8") + subprocess.run(["git", "-C", str(self.repo), "add", str(checker.relative_to(self.repo))], check=True) + guard_commit = self.commit("revise guard") + subprocess.run( + ["git", "-C", str(self.repo), "push", "--no-verify", "-q", "origin", "main"], + check=True, + env=self.env, + ) + self.write("neutral.txt", "neutral follow-up\n") + self.commit("neutral follow-up") + stale_push = subprocess.run( + ["git", "-C", str(self.repo), "push", "origin", "main"], + env=self.env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertNotEqual(0, stale_push.returncode) + self.assertIn("installed boundary guard is stale", stale_push.stderr) + + reinstall = self.run_installer() + self.assertEqual(0, reinstall.returncode, reinstall.stderr) + self.assertEqual(self.common_dir / "hackforger-boundary-hooks" / guard_commit, self.hooks_path()) + self.assertNotEqual(old_hooks_path, self.hooks_path()) + recovered_push = subprocess.run( + ["git", "-C", str(self.repo), "push", "origin", "main"], + env=self.env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(0, recovered_push.returncode, recovered_push.stderr) + + def test_real_canonical_hook_is_not_replaced(self) -> None: + real_hook = self.common_dir / "hooks/pre-push" + real_hook.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + real_hook.chmod(0o755) + result = self.run_installer() + self.assertEqual(1, result.returncode) + self.assertIn("canonical hooks directory contains a real hook", result.stderr) + self.assertEqual("#!/bin/sh\nexit 0\n", real_hook.read_text(encoding="utf-8")) + configured = subprocess.run( + ["git", "-C", str(self.repo), "config", "--get", "core.hooksPath"], + text=True, + stdout=subprocess.PIPE, + ) + self.assertEqual(1, configured.returncode) + + def test_conflicting_hooks_path_is_not_replaced(self) -> None: + other = self.base / "team-hooks" + other.mkdir() + subprocess.run( + ["git", "-C", str(self.repo), "config", "core.hooksPath", str(other)], + check=True, + ) + result = self.run_installer() + self.assertEqual(1, result.returncode) + self.assertIn("refusing to replace it", result.stderr) + self.assertEqual(other, self.hooks_path()) + + def test_noncanonical_origin_is_rejected(self) -> None: + subprocess.run( + ["git", "-C", str(self.repo), "remote", "set-url", "origin", "git@github.com:example/other.git"], + check=True, + ) + result = self.run_installer() + self.assertEqual(1, result.returncode) + self.assertIn("origin must be the canonical HackForger/hackforger", result.stderr) + + def test_effective_hook_rejects_origin_changed_after_install(self) -> None: + result = self.run_installer() + self.assertEqual(0, result.returncode, result.stderr) + subprocess.run( + ["git", "-C", str(self.repo), "remote", "set-url", "origin", "https://github.com/example/other.git"], + check=True, + ) + self.write("neutral.txt", "neutral update\n") + head = self.commit("neutral update") + hook = subprocess.run( + [str(self.hooks_path() / "pre-push"), "origin", "unused"], + cwd=self.repo, + env=self.env, + input=f"refs/heads/main {head} refs/heads/main {self.baseline}\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(2, hook.returncode) + self.assertIn("origin must be the canonical HackForger/hackforger", hook.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/test_pre_push_boundary.py b/scripts/ci/test_pre_push_boundary.py new file mode 100755 index 0000000000..e5841efbbd --- /dev/null +++ b/scripts/ci/test_pre_push_boundary.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import os +import shlex +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SCRIPT_DIR = Path(__file__).resolve().parent +CHECKER = SCRIPT_DIR / "check_public_repository_boundary.py" +POLICY = SCRIPT_DIR / "private-content-markers.txt" +GUARD_POLICY = SCRIPT_DIR / "boundary_guard_policy.py" +PRE_PUSH = SCRIPT_DIR.parent / "pre-push-public-boundary.sh" +CANONICAL_ORIGIN = "git@github.com:HackForger/hackforger.git" + + +class PrePushBoundaryTest(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.base = Path(self.temp.name) + self.repo = self.base / "repo" + self.remote = self.base / "origin.git" + subprocess.run(["git", "init", "-q", "--bare", str(self.remote)], check=True) + subprocess.run(["git", "init", "-q", str(self.repo)], check=True) + subprocess.run(["git", "-C", str(self.repo), "checkout", "-q", "-b", "main"], check=True) + subprocess.run(["git", "-C", str(self.repo), "remote", "add", "origin", CANONICAL_ORIGIN], check=True) + fake_ssh = self.base / "fake-ssh" + fake_ssh.write_text( + "#!/bin/sh\n" + "case \"$*\" in\n" + f" *git-upload-pack*) exec git-upload-pack {shlex.quote(str(self.remote))} ;;\n" + f" *git-receive-pack*) exec git-receive-pack {shlex.quote(str(self.remote))} ;;\n" + " *) exit 97 ;;\n" + "esac\n", + encoding="utf-8", + ) + fake_ssh.chmod(0o755) + self.env = os.environ.copy() + self.env["GIT_SSH_COMMAND"] = str(fake_ssh) + target_ci = self.repo / "scripts/ci" + target_ci.mkdir(parents=True) + shutil.copy2(CHECKER, target_ci / CHECKER.name) + shutil.copy2(POLICY, target_ci / POLICY.name) + shutil.copy2(GUARD_POLICY, target_ci / GUARD_POLICY.name) + shutil.copy2(PRE_PUSH, self.repo / "scripts" / PRE_PUSH.name) + subprocess.run( + [ + "git", + "-C", + str(self.repo), + "add", + "scripts/pre-push-public-boundary.sh", + "scripts/ci/check_public_repository_boundary.py", + "scripts/ci/boundary_guard_policy.py", + "scripts/ci/private-content-markers.txt", + ], + check=True, + ) + self.write("README.md", "neutral baseline\n") + self.baseline = self.commit("baseline") + subprocess.run( + ["git", "-C", str(self.repo), "push", "-q", "-u", "origin", "main"], + check=True, + env=self.env, + ) + subprocess.run( + ["git", "-C", str(self.remote), "symbolic-ref", "HEAD", "refs/heads/main"], + check=True, + ) + self.bundle = self.base / "hooks" / self.baseline + self.bundle.mkdir(parents=True) + shutil.copy2(CHECKER, self.bundle / "check_public_repository_boundary.py") + shutil.copy2(POLICY, self.bundle / "private-content-markers.txt") + shutil.copy2(GUARD_POLICY, self.bundle / "boundary_guard_policy.py") + shutil.copy2(PRE_PUSH, self.bundle / "pre-push") + (self.bundle / "source-commit").write_text(f"{self.baseline}\n", encoding="ascii") + (self.bundle / "check_public_repository_boundary.py").chmod(0o555) + (self.bundle / "pre-push").chmod(0o555) + + def tearDown(self) -> None: + self.temp.cleanup() + + def write(self, rel: str, content: str) -> None: + path = self.repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + subprocess.run(["git", "-C", str(self.repo), "add", "-f", rel], check=True) + + def commit(self, message: str) -> str: + subprocess.run( + [ + "git", + "-C", + str(self.repo), + "-c", + "user.name=Boundary Test", + "-c", + "user.email=boundary@example.invalid", + "commit", + "-q", + "-m", + message, + ], + check=True, + ) + return subprocess.run( + ["git", "-C", str(self.repo), "rev-parse", "HEAD"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + + def test_outgoing_add_then_delete_is_rejected_despite_replace_ref(self) -> None: + marker = "synno" + "vator" + self.write("temporary.md", f"tenant: {marker}\n") + sensitive = self.commit("temporary content") + subprocess.run( + ["git", "-C", str(self.repo), "rm", "temporary.md"], + check=True, + stdout=subprocess.DEVNULL, + ) + head = self.commit("remove temporary content") + tree = subprocess.run( + ["git", "-C", str(self.repo), "rev-parse", f"{head}^{{tree}}"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + replacement = subprocess.run( + [ + "git", + "-C", + str(self.repo), + "-c", + "user.name=Boundary Test", + "-c", + "user.email=boundary@example.invalid", + "commit-tree", + tree, + "-p", + self.baseline, + "-m", + "neutral replacement", + ], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + subprocess.run( + ["git", "-C", str(self.repo), "replace", head, replacement], + check=True, + ) + result = subprocess.run( + [str(self.bundle / "pre-push"), "origin", str(self.remote)], + cwd=self.repo, + env=self.env, + input=f"refs/heads/main {head} refs/heads/main {self.baseline}\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(1, result.returncode) + self.assertIn( + f"HISTORY_BUSINESS_MARKER\t{sensitive[:12]}:temporary.md", + result.stderr, + ) + self.assertNotIn(marker, result.stderr.lower()) + + def test_git_environment_overlays_cannot_redirect_history_scan(self) -> None: + marker = "synno" + "vator" + self.write("temporary.md", f"tenant: {marker}\n") + sensitive = self.commit("temporary content") + subprocess.run( + ["git", "-C", str(self.repo), "rm", "temporary.md"], + check=True, + stdout=subprocess.DEVNULL, + ) + head = self.commit("remove temporary content") + decoy = self.base / "decoy" + subprocess.run(["git", "init", "-q", str(decoy)], check=True) + env = self.env.copy() + env.update( + { + "GIT_DIR": str(decoy / ".git"), + "GIT_WORK_TREE": str(decoy), + "GIT_IMPLICIT_WORK_TREE": "0", + "GIT_OBJECT_DIRECTORY": str(decoy / ".git/objects"), + "GIT_ALTERNATE_OBJECT_DIRECTORIES": str(decoy / ".git/objects"), + "GIT_INDEX_FILE": str(decoy / ".git/index"), + "GIT_NAMESPACE": "decoy", + "GIT_NO_REPLACE_OBJECTS": "0", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "remote.origin.url", + "GIT_CONFIG_VALUE_0": "https://github.com/example/other.git", + "GIT_CONFIG_GLOBAL": str(decoy / "config"), + "GIT_EXEC_PATH": str(decoy), + "GIT_INTERNAL_SUPER_PREFIX": "decoy/", + } + ) + result = subprocess.run( + [str(self.bundle / "pre-push"), "origin", str(self.remote)], + cwd=self.repo, + env=env, + input=f"refs/heads/main {head} refs/heads/main {self.baseline}\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(1, result.returncode, result.stderr) + self.assertIn( + f"HISTORY_BUSINESS_MARKER\t{sensitive[:12]}:temporary.md", + result.stderr, + ) + self.assertNotIn(marker, result.stderr.lower()) + + def test_legacy_graft_cannot_hide_sensitive_add_then_delete(self) -> None: + marker = "synno" + "vator" + self.write("temporary.md", f"tenant: {marker}\n") + self.commit("temporary content") + subprocess.run( + ["git", "-C", str(self.repo), "rm", "temporary.md"], + check=True, + stdout=subprocess.DEVNULL, + ) + head = self.commit("remove temporary content") + grafts = Path( + subprocess.run( + [ + "git", + "-C", + str(self.repo), + "rev-parse", + "--path-format=absolute", + "--git-path", + "info/grafts", + ], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + ) + grafts.parent.mkdir(parents=True, exist_ok=True) + grafts.write_text(f"{head} {self.baseline}\n", encoding="ascii") + result = subprocess.run( + [str(self.bundle / "pre-push"), "origin", str(self.remote)], + cwd=self.repo, + env=self.env, + input=f"refs/heads/main {head} refs/heads/main {self.baseline}\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(2, result.returncode) + self.assertIn("legacy Git grafts are not allowed", result.stderr) + self.assertNotIn(marker, result.stderr.lower()) + + def test_shallow_boundary_cannot_hide_sensitive_add_then_delete(self) -> None: + marker = "synno" + "vator" + self.write("temporary.md", f"tenant: {marker}\n") + self.commit("temporary content") + subprocess.run( + ["git", "-C", str(self.repo), "rm", "temporary.md"], + check=True, + stdout=subprocess.DEVNULL, + ) + head = self.commit("remove temporary content") + shallow = Path( + subprocess.run( + [ + "git", + "-C", + str(self.repo), + "rev-parse", + "--path-format=absolute", + "--git-path", + "shallow", + ], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + ) + shallow.write_text(f"{head}\n", encoding="ascii") + result = subprocess.run( + [str(self.bundle / "pre-push"), "origin", str(self.remote)], + cwd=self.repo, + env=self.env, + input=f"refs/heads/main {head} refs/heads/main {self.baseline}\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(2, result.returncode) + self.assertIn("shallow repositories are not allowed", result.stderr) + self.assertNotIn(marker, result.stderr.lower()) + + def test_annotated_tag_is_rejected_before_metadata_can_escape(self) -> None: + marker = "synno" + "vator" + subprocess.run( + [ + "git", + "-C", + str(self.repo), + "-c", + "user.name=Boundary Test", + "-c", + "user.email=boundary@example.invalid", + "tag", + "-a", + "release-test", + "-m", + f"tenant: {marker}", + ], + check=True, + ) + tag_oid = subprocess.run( + ["git", "-C", str(self.repo), "rev-parse", "refs/tags/release-test"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + result = subprocess.run( + [str(self.bundle / "pre-push"), "origin", str(self.remote)], + cwd=self.repo, + env=self.env, + input=f"refs/tags/release-test {tag_oid} refs/tags/release-test {'0' * 40}\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(2, result.returncode) + self.assertIn("annotated tags or non-commit refs require dedicated security review", result.stderr) + self.assertNotIn(marker, result.stderr.lower()) + + def test_sensitive_ref_name_is_rejected_even_without_new_commits(self) -> None: + address = "8" + ".8.8.8" + host = "api.corp." + "internal" + ref_name = f"refs/heads/prod-{address}-{host}" + result = subprocess.run( + [str(self.bundle / "pre-push"), "origin", str(self.remote)], + cwd=self.repo, + env=self.env, + input=f"{ref_name} {self.baseline} {ref_name} {self.baseline}\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(1, result.returncode) + self.assertIn("REF_INTERNAL_HOST\tref-name-redacted", result.stderr) + self.assertIn("REF_NON_DOCUMENTATION_IP\tref-name-redacted", result.stderr) + self.assertNotIn(address, result.stderr) + self.assertNotIn(host, result.stderr) + + def test_ref_deletion_skips_annotated_object_and_sensitive_name(self) -> None: + address = "8" + ".8.8.8" + host = "api.corp." + "internal" + ref_name = f"refs/tags/prod-{address}-{host}" + subprocess.run( + [ + "git", + "-C", + str(self.repo), + "-c", + "user.name=Boundary Test", + "-c", + "user.email=boundary@example.invalid", + "tag", + "-a", + "delete-me", + "-m", + "neutral tag", + ], + check=True, + ) + tag_oid = subprocess.run( + ["git", "-C", str(self.repo), "rev-parse", "refs/tags/delete-me"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + grafts = Path( + subprocess.run( + ["git", "-C", str(self.repo), "rev-parse", "--path-format=absolute", "--git-path", "info/grafts"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + ) + grafts.parent.mkdir(parents=True, exist_ok=True) + grafts.write_text(f"{self.baseline} {self.baseline}\n", encoding="ascii") + shallow = Path( + subprocess.run( + ["git", "-C", str(self.repo), "rev-parse", "--path-format=absolute", "--git-path", "shallow"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + ) + shallow.write_text(f"{self.baseline}\n", encoding="ascii") + result = subprocess.run( + [str(self.bundle / "pre-push"), "origin", str(self.remote)], + cwd=self.repo, + env=self.env, + input=f"(delete) {'0' * 40} {ref_name} {tag_oid}\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("PASS (ref deletions only)", result.stdout) + self.assertNotIn(address, result.stderr) + self.assertNotIn(host, result.stderr) + + def test_clean_update_to_existing_default_ref_passes(self) -> None: + self.write("README.md", "neutral baseline\nneutral update\n") + head = self.commit("neutral update") + result = subprocess.run( + [str(self.bundle / "pre-push"), "origin", str(self.remote)], + cwd=self.repo, + env=self.env, + input=f"refs/heads/main {head} refs/heads/main {self.baseline}\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("public repository boundary: PASS", result.stdout) + + def test_checkout_baseline_environment_override_is_ignored(self) -> None: + self.write("README.md", "neutral baseline\nneutral update\n") + head = self.commit("neutral update") + env = self.env.copy() + env["HACKFORGER_BOUNDARY_BASELINE_REF"] = "refs/heads/not-a-trusted-baseline" + result = subprocess.run( + [str(self.bundle / "pre-push"), "origin", str(self.remote)], + cwd=self.repo, + env=env, + input=f"refs/heads/main {head} refs/heads/main {self.baseline}\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("public repository boundary: PASS", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/test_public_repository_boundary.py b/scripts/ci/test_public_repository_boundary.py new file mode 100755 index 0000000000..b6dfb2cf07 --- /dev/null +++ b/scripts/ci/test_public_repository_boundary.py @@ -0,0 +1,593 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import subprocess +import tempfile +import unittest +import os +from pathlib import Path + + +SCRIPT_DIR = Path(__file__).resolve().parent +CHECKER = SCRIPT_DIR / "check_public_repository_boundary.py" +POLICY = SCRIPT_DIR / "private-content-markers.txt" + + +class BoundaryCheckerTest(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + subprocess.run(["git", "init", "-q", str(self.root)], check=True) + + def tearDown(self) -> None: + self.temp.cleanup() + + def write(self, path: str, content: str = "neutral\n") -> None: + target = self.root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + subprocess.run(["git", "-C", str(self.root), "add", "-f", path], check=True) + + def write_bytes(self, path: str, content: bytes) -> None: + target = self.root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + subprocess.run(["git", "-C", str(self.root), "add", "-f", path], check=True) + + def commit(self, message: str, *, author_name: str = "Boundary Test") -> str: + subprocess.run( + [ + "git", + "-C", + str(self.root), + "-c", + f"user.name={author_name}", + "-c", + "user.email=boundary@example.invalid", + "commit", + "-q", + "-m", + message, + ], + check=True, + ) + return subprocess.run( + ["git", "-C", str(self.root), "rev-parse", "HEAD"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + + @staticmethod + def real_ipv6() -> str: + return "2606" + ":4700" + ":4700" + ":" + ":1111" + + def run_checker( + self, + baseline: Path | None = None, + baseline_ref: str | None = None, + history_base: str | None = None, + history_head: str | None = None, + ref_name: str | None = None, + ) -> subprocess.CompletedProcess[str]: + command = ["python3", str(CHECKER), "--root", str(self.root), "--policy", str(POLICY)] + if baseline_ref is not None: + command.extend(["--baseline-ref", baseline_ref]) + else: + baseline = baseline or self.root + command.extend(["--baseline-root", str(baseline)]) + if history_base is not None or history_head is not None: + self.assertIsNotNone(history_base) + self.assertIsNotNone(history_head) + command.extend(["--history-base-ref", history_base, "--history-head-ref", history_head]) + if ref_name is not None: + command.extend(["--ref-name", ref_name]) + return subprocess.run( + command, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + def test_neutral_content_passes(self) -> None: + self.write("docs/guide.md", "See https://example.invalid and 203.0.113.10\n") + self.assertEqual(0, self.run_checker().returncode) + + def test_business_marker_fails_without_echoing_content(self) -> None: + marker = "synno" + "vator" + self.write("README.md", f"tenant: {marker}\n") + result = self.run_checker() + self.assertEqual(1, result.returncode) + self.assertIn("BUSINESS_MARKER\tREADME.md", result.stderr) + self.assertNotIn(marker, result.stderr.lower()) + + path = "docs/" + marker.title() + "-runbook.md" + self.write(path) + result = self.run_checker() + self.assertIn("BUSINESS_MARKER\tpath-redacted", result.stderr) + self.assertIn("PATH_BUSINESS_MARKER\tpath-redacted", result.stderr) + self.assertNotIn(marker, result.stderr.lower()) + + def test_sensitive_file_name_metadata_fails_without_echo(self) -> None: + address = "8" + ".8.8.8" + host = "api.corp." + "internal" + material = "gh" + "p_" + "a" * 24 + path = f"docs/release-{address}-{host}-{material}.md" + self.write(path, f"host={host}\n") + result = self.run_checker() + self.assertIn("PATH_NON_DOCUMENTATION_IP\tpath-redacted", result.stderr) + self.assertIn("PATH_INTERNAL_HOST\tpath-redacted", result.stderr) + self.assertIn("PATH_SECRET_MATERIAL\tpath-redacted", result.stderr) + self.assertNotIn(address, result.stderr) + self.assertNotIn(host, result.stderr) + self.assertNotIn(material, result.stderr) + + def test_ip_before_filename_extension_and_sentence_period_fails(self) -> None: + address = "8" + ".8.8.8" + path = f"docs/prod-{address}.md" + self.write(path) + result = self.run_checker() + self.assertIn("PATH_NON_DOCUMENTATION_IP\tpath-redacted", result.stderr) + self.assertNotIn(address, result.stderr) + + subprocess.run( + ["git", "-C", str(self.root), "rm", "-f", path], + check=True, + stdout=subprocess.DEVNULL, + ) + self.write("docs/host.md", f"host={address}.\n") + result = self.run_checker() + self.assertIn("NON_DOCUMENTATION_IP\tdocs/host.md", result.stderr) + + def test_five_component_numeric_sequence_is_not_an_ipv4_submatch(self) -> None: + self.write("docs/version.md", "version=1.8.8.8.8\n") + self.assertEqual(0, self.run_checker().returncode) + + def test_real_ipv6_content_path_and_ref_fail_without_echo(self) -> None: + address = self.real_ipv6() + self.write("README.md", f"host=[{address}]\n") + result = self.run_checker() + self.assertIn("NON_DOCUMENTATION_IPV6\tREADME.md", result.stderr) + self.assertNotIn(address, result.stderr) + + path = f"docs/prod-[{address}].md" + self.write(path) + result = self.run_checker() + self.assertIn("PATH_NON_DOCUMENTATION_IPV6\tpath-redacted", result.stderr) + self.assertNotIn(address, result.stderr) + + result = self.run_checker(ref_name=f"refs/heads/prod-[{address}]") + self.assertIn("REF_NON_DOCUMENTATION_IPV6\tref-name-redacted", result.stderr) + self.assertNotIn(address, result.stderr) + + def test_documentation_and_loopback_ipv6_addresses_pass(self) -> None: + self.write( + "docs/network.md", + "unspecified=::\nloopback=::1\ndocumentation=2001:db8::1234\n", + ) + self.assertEqual(0, self.run_checker().returncode) + + def test_intermediate_ipv6_content_is_scanned_after_deletion(self) -> None: + self.write("README.md", "neutral baseline\n") + base = self.commit("baseline") + address = self.real_ipv6() + self.write("temporary.md", f"host=[{address}]\n") + sensitive_commit = self.commit("temporary network metadata") + subprocess.run( + ["git", "-C", str(self.root), "rm", "temporary.md"], + check=True, + stdout=subprocess.DEVNULL, + ) + head = self.commit("remove temporary metadata") + result = self.run_checker(history_base=base, history_head=head) + self.assertIn( + f"HISTORY_NON_DOCUMENTATION_IPV6\t{sensitive_commit[:12]}:temporary.md", + result.stderr, + ) + self.assertNotIn(address, result.stderr) + + def test_ipv6_in_raw_commit_metadata_is_scanned(self) -> None: + self.write("README.md", "neutral baseline\n") + base = self.commit("baseline") + address = self.real_ipv6() + self.write("README.md", "neutral update\n") + sensitive_commit = self.commit("neutral message", author_name=f"operator [{address}]") + result = self.run_checker(history_base=base, history_head=sensitive_commit) + self.assertIn( + f"HISTORY_NON_DOCUMENTATION_IPV6\t{sensitive_commit[:12]}:COMMIT_METADATA", + result.stderr, + ) + self.assertNotIn(address, result.stderr) + + def test_business_marker_in_ref_name_fails(self) -> None: + self.write("README.md") + marker = "synno" + "vator" + result = self.run_checker(ref_name=f"refs/heads/chore/{marker}-content") + self.assertIn("REF_BUSINESS_MARKER", result.stderr) + self.assertNotIn(marker, result.stderr.lower()) + + def test_infrastructure_facts_in_ref_name_fail_without_echo(self) -> None: + self.write("README.md") + address = "8" + ".8.8.8" + host = "api.corp." + "internal" + ref_name = f"refs/heads/prod-{address}-{host}" + result = self.run_checker(ref_name=ref_name) + self.assertIn("REF_INTERNAL_HOST\tref-name-redacted", result.stderr) + self.assertIn("REF_NON_DOCUMENTATION_IP\tref-name-redacted", result.stderr) + self.assertNotIn(address, result.stderr) + self.assertNotIn(host, result.stderr) + + def test_staged_blob_is_scanned_when_worktree_bytes_differ(self) -> None: + marker = "synno" + "vator" + self.write("README.md", f"tenant: {marker}\n") + (self.root / "README.md").write_text("neutral worktree\n", encoding="utf-8") + result = self.run_checker() + self.assertIn("BUSINESS_MARKER\tREADME.md", result.stderr) + self.assertNotIn(marker, result.stderr.lower()) + + def test_intermediate_commit_is_scanned_after_file_is_deleted(self) -> None: + self.write("README.md", "neutral baseline\n") + base = self.commit("baseline") + marker = "synno" + "vator" + self.write("temporary.md", f"tenant: {marker}\n") + sensitive_commit = self.commit("temporary content") + subprocess.run( + ["git", "-C", str(self.root), "rm", "temporary.md"], + check=True, + stdout=subprocess.DEVNULL, + ) + head = self.commit("remove temporary content") + result = self.run_checker(history_base=base, history_head=head) + self.assertIn( + f"HISTORY_BUSINESS_MARKER\t{sensitive_commit[:12]}:temporary.md", + result.stderr, + ) + + def test_deleting_legacy_forbidden_path_is_allowed(self) -> None: + self.write("docs/ops/legacy-runbook.md", "legacy public content\n") + base = self.commit("legacy baseline") + subprocess.run( + ["git", "-C", str(self.root), "rm", "docs/ops/legacy-runbook.md"], + check=True, + stdout=subprocess.DEVNULL, + ) + head = self.commit("remove legacy runbook") + result = self.run_checker(history_base=base, history_head=head) + self.assertEqual(0, result.returncode, result.stderr) + + def test_sensitive_deleted_file_name_is_scanned_without_echo(self) -> None: + self.write("README.md", "neutral baseline\n") + base = self.commit("baseline") + address = "8" + ".8.8.8" + host = "api.corp." + "internal" + material = "gh" + "p_" + "a" * 24 + path = f"docs/release-{address}-{host}-{material}.md" + self.write(path) + sensitive_commit = self.commit("temporary path metadata") + subprocess.run( + ["git", "-C", str(self.root), "rm", path], + check=True, + stdout=subprocess.DEVNULL, + ) + head = self.commit("remove temporary path") + result = self.run_checker(history_base=base, history_head=head) + self.assertIn( + f"HISTORY_PATH_NON_DOCUMENTATION_IP\t{sensitive_commit[:12]}:path-redacted", + result.stderr, + ) + self.assertIn( + f"HISTORY_PATH_INTERNAL_HOST\t{sensitive_commit[:12]}:path-redacted", + result.stderr, + ) + self.assertIn( + f"HISTORY_PATH_SECRET_MATERIAL\t{sensitive_commit[:12]}:path-redacted", + result.stderr, + ) + self.assertNotIn(address, result.stderr) + self.assertNotIn(host, result.stderr) + self.assertNotIn(material, result.stderr) + + def test_raw_commit_metadata_is_scanned_without_echo(self) -> None: + self.write("README.md", "neutral baseline\n") + base = self.commit("baseline") + marker = "synno" + "vator" + self.write("README.md", "neutral update\n") + sensitive_commit = self.commit("neutral message", author_name=marker) + result = self.run_checker(history_base=base, history_head=sensitive_commit) + self.assertIn( + f"HISTORY_BUSINESS_MARKER\t{sensitive_commit[:12]}:COMMIT_METADATA", + result.stderr, + ) + self.assertNotIn(marker, result.stderr.lower()) + + def test_guard_modify_then_restore_is_rejected_from_history(self) -> None: + self.write(".github/workflows/boundary.yml", "name: trusted\n") + base = self.commit("baseline") + self.write(".github/workflows/boundary.yml", "name: disabled\n") + modified = self.commit("temporarily modify guard") + self.write(".github/workflows/boundary.yml", "name: trusted\n") + restored = self.commit("restore guard") + result = self.run_checker(history_base=base, history_head=restored) + self.assertIn( + f"HISTORY_GUARD_CHANGE\t{modified[:12]}:guard-path-redacted", + result.stderr, + ) + + def test_merge_of_current_trusted_guard_version_is_allowed(self) -> None: + self.write(".github/workflows/boundary.yml", "name: trusted-v1\n") + old_base = self.commit("old trusted baseline") + subprocess.run( + ["git", "-C", str(self.root), "checkout", "-q", "-b", "feature"], + check=True, + ) + self.write("README.md", "feature work\n") + self.commit("feature work") + + subprocess.run( + ["git", "-C", str(self.root), "checkout", "-q", "-b", "trusted", old_base], + check=True, + ) + self.write(".github/workflows/boundary.yml", "name: trusted-v2\n") + target_base = self.commit("audited guard update") + + subprocess.run( + ["git", "-C", str(self.root), "checkout", "-q", "feature"], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(self.root), + "-c", + "user.name=Boundary Test", + "-c", + "user.email=boundary@example.invalid", + "merge", + "-q", + "--no-ff", + "trusted", + "-m", + "merge current trusted policy", + ], + check=True, + ) + head = subprocess.run( + ["git", "-C", str(self.root), "rev-parse", "HEAD"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + result = self.run_checker(history_base=target_base, history_head=head) + self.assertNotIn("HISTORY_GUARD_CHANGE", result.stderr) + + def test_secret_material_fails_outside_scoped_directories(self) -> None: + material = "gh" + "p_" + "a" * 24 + self.write("README.md", f"credential: {material}\n") + result = self.run_checker() + self.assertIn("SECRET_MATERIAL\tREADME.md", result.stderr) + self.assertNotIn(material, result.stderr) + + webhook = "https://hooks." + "slack.com/services/T000/B000/secret-part" + self.write("README.md", webhook) + result = self.run_checker() + self.assertIn("SECRET_MATERIAL\tREADME.md", result.stderr) + self.assertNotIn(webhook, result.stderr) + + def test_private_paths_fail(self) -> None: + self.write("custom/public/assets/landing/index.html") + self.assertIn("BOUNDARY_PATH", self.run_checker().stderr) + + self.write("Docs/Ops/runbook.md") + self.assertIn("BOUNDARY_PATH\tDocs/Ops/runbook.md", self.run_checker().stderr) + + self.write(".worktrees/runtime/data.bin") + self.assertIn("BOUNDARY_PATH\t.worktrees/runtime/data.bin", self.run_checker().stderr) + + self.write("scripts/cleanup-invalid-hackathons.sql") + self.assertIn( + "BOUNDARY_PATH\tscripts/cleanup-invalid-hackathons.sql", + self.run_checker().stderr, + ) + + def test_project_owned_opaque_and_generated_files_fail(self) -> None: + self.write("docs/guide.md", "neutral\x00hidden\n") + result = self.run_checker() + self.assertIn("UNREVIEWED_BINARY\tdocs/guide.md", result.stderr) + + self.write("scripts/ci/__pycache__/checker.pyc", "opaque\x00bytes") + result = self.run_checker() + self.assertIn("GENERATED_ARTIFACT\tscripts/ci/__pycache__/checker.pyc", result.stderr) + self.assertIn("UNREVIEWED_BINARY\tscripts/ci/__pycache__/checker.pyc", result.stderr) + + def test_lfs_pointer_fails_in_any_directory(self) -> None: + self.write( + "public/assets/customer.dat", + "\n".join( + ( + "version https://git-lfs.github.com/spec/v1", + "oid sha256:" + "a" * 64, + "size 1234", + "", + ) + ), + ) + self.assertIn( + "LFS_POINTER\tpublic/assets/customer.dat", + self.run_checker().stderr, + ) + + def test_changed_opaque_file_outside_owned_tree_requires_review(self) -> None: + with tempfile.TemporaryDirectory() as baseline_dir: + baseline = Path(baseline_dir) + target = baseline / "modules/example/fixture.bin" + target.parent.mkdir(parents=True) + target.write_bytes(b"baseline\x00bytes") + self.write("modules/example/fixture.bin", "baseline\x00bytes") + self.assertEqual(0, self.run_checker(baseline).returncode) + + self.write("modules/example/fixture.bin", "changed\x00bytes") + result = self.run_checker(baseline) + self.assertIn("UNREVIEWED_BINARY\tmodules/example/fixture.bin", result.stderr) + + def test_delayed_nul_and_invalid_utf8_are_opaque(self) -> None: + with tempfile.TemporaryDirectory() as baseline_dir: + baseline = Path(baseline_dir) + self.write_bytes("assets/customer.data", b"A" * 8192 + b"\x00private") + result = self.run_checker(baseline) + self.assertIn("UNREVIEWED_BINARY\tassets/customer.data", result.stderr) + + self.write_bytes("assets/customer.data", b"A" * 8192 + b"\xffprivate") + result = self.run_checker(baseline) + self.assertIn("UNREVIEWED_BINARY\tassets/customer.data", result.stderr) + + def test_git_ref_baseline_detects_changed_opaque_file(self) -> None: + self.write("modules/example/fixture.bin", "baseline\x00bytes") + subprocess.run( + [ + "git", + "-C", + str(self.root), + "-c", + "user.name=Boundary Test", + "-c", + "user.email=boundary@example.invalid", + "commit", + "-q", + "-m", + "baseline", + ], + check=True, + ) + baseline_ref = subprocess.run( + ["git", "-C", str(self.root), "rev-parse", "HEAD"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + self.write("modules/example/fixture.bin", "changed\x00bytes") + result = self.run_checker(baseline_ref=baseline_ref) + self.assertIn("UNREVIEWED_BINARY\tmodules/example/fixture.bin", result.stderr) + + def test_cli_requires_an_opaque_file_baseline(self) -> None: + self.write("README.md") + result = subprocess.run( + ["python3", str(CHECKER), "--root", str(self.root), "--policy", str(POLICY)], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(2, result.returncode) + self.assertIn("one of the arguments --baseline-root --baseline-ref is required", result.stderr) + + def test_personal_path_and_real_ip_fail(self) -> None: + personal = "/" + "Users/alice/work" + address = "8" + ".8.8.8" + self.write("deploy/example.md", f"root={personal}\nhost={address}\n") + result = self.run_checker() + self.assertIn("PERSONAL_PATH", result.stderr) + self.assertIn("NON_DOCUMENTATION_IP", result.stderr) + + self.write("deploy/example.md", "root=/" + "Users/alice\n") + self.assertIn("PERSONAL_PATH", self.run_checker().stderr) + + def test_internal_host_fails_but_locale_key_passes(self) -> None: + host = "api.corp." + "internal" + self.write("docs/host.md", f"host={host}\n") + result = self.run_checker() + self.assertIn("INTERNAL_HOST\tdocs/host.md", result.stderr) + + locale_key = "hackathon.error." + "internal" + self.write("options/locale/test.ini", f"{locale_key} = Generic error.\n") + subprocess.run( + ["git", "-C", str(self.root), "rm", "-f", "docs/host.md"], + check=True, + stdout=subprocess.DEVNULL, + ) + self.assertEqual(0, self.run_checker().returncode) + + disguised = "prod.error." + "internal" + self.write("docs/host.md", f"url=https://{disguised}\n") + self.assertIn("INTERNAL_HOST\tdocs/host.md", self.run_checker().stderr) + + self.write("docs/host.md", f"host={disguised}\n") + self.assertIn("INTERNAL_HOST\tdocs/host.md", self.run_checker().stderr) + + self.write("README.md", f"{disguised}\n") + self.assertIn("INTERNAL_HOST\tREADME.md", self.run_checker().stderr) + + self.write("README.md", f"Connect to `{disguised}`\n") + self.assertIn("INTERNAL_HOST\tREADME.md", self.run_checker().stderr) + + self.write( + "routers/example.go", + f'ctx.Locale.TrString("{locale_key}")\n', + ) + subprocess.run( + [ + "git", + "-C", + str(self.root), + "rm", + "-f", + "docs/host.md", + "README.md", + ], + check=True, + stdout=subprocess.DEVNULL, + ) + self.assertEqual(0, self.run_checker().returncode) + + def test_single_label_internal_host_and_sensitive_path_parts_fail(self) -> None: + host = "db." + "internal" + self.write("README.md", f"host={host}\n") + self.assertIn("INTERNAL_HOST\tREADME.md", self.run_checker().stderr) + + subprocess.run( + ["git", "-C", str(self.root), "rm", "-f", "README.md"], + check=True, + stdout=subprocess.DEVNULL, + ) + self.write("prod.error." + "internal", "neutral\n") + result = self.run_checker() + self.assertIn("PRIVATE_FILE\tpath-redacted", result.stderr) + self.assertIn("PATH_INTERNAL_HOST\tpath-redacted", result.stderr) + + self.write("docs/operator.local/runbook.md", "neutral\n") + self.assertIn("LOCAL_FILE", self.run_checker().stderr) + + self.write("private-content/runbook.md", "neutral\n") + self.assertIn("PRIVATE_FILE", self.run_checker().stderr) + + def test_literal_credential_fails(self) -> None: + literal = "pass" + "word: hunter2" + self.write("README.md", literal) + result = self.run_checker() + self.assertIn("LITERAL_CREDENTIAL", result.stderr) + self.assertNotIn("hunter2", result.stderr) + + self.write("custom/conf/app.ini", literal) + self.assertIn("LITERAL_CREDENTIAL\tcustom/conf/app.ini", self.run_checker().stderr) + + def test_runtime_screenshot_fails(self) -> None: + self.write("docs/tests/e2e/reports/live.png") + self.assertIn("RUNTIME_EVIDENCE", self.run_checker().stderr) + + target = self.root / "custom/public/assets/img/customer.png" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(b"\x89PNG\r\n\x1a\n\x00private") + subprocess.run(["git", "-C", str(self.root), "add", "-f", str(target.relative_to(self.root))], check=True) + self.assertIn("UNREVIEWED_BINARY", self.run_checker().stderr) + + def test_symlink_target_is_scanned_without_dereferencing(self) -> None: + target = "/" + "Users/alice/private" + link = self.root / "docs/link" + link.parent.mkdir(parents=True, exist_ok=True) + os.symlink(target, link) + subprocess.run(["git", "-C", str(self.root), "add", "docs/link"], check=True) + result = self.run_checker() + self.assertIn("PERSONAL_PATH\tdocs/link", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/install-public-boundary-hook.sh b/scripts/install-public-boundary-hook.sh new file mode 100755 index 0000000000..be81834e04 --- /dev/null +++ b/scripts/install-public-boundary-hook.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Install the public-boundary hook from the trusted origin default commit, never +# from the mutable checkout that happens to invoke this installer. + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) + +# Ignore caller-provided repository/object/config views. SSH transport and +# credential-agent variables remain user-controlled; a user who controls the +# push process can already choose --no-verify and is outside this hook's threat +# boundary. +unset \ + GIT_DIR \ + GIT_WORK_TREE \ + GIT_IMPLICIT_WORK_TREE \ + GIT_COMMON_DIR \ + GIT_OBJECT_DIRECTORY \ + GIT_ALTERNATE_OBJECT_DIRECTORIES \ + GIT_INDEX_FILE \ + GIT_INDEX_VERSION \ + GIT_NAMESPACE \ + GIT_NO_REPLACE_OBJECTS \ + GIT_REPLACE_REF_BASE \ + GIT_CONFIG \ + GIT_CONFIG_PARAMETERS \ + GIT_CONFIG_COUNT \ + GIT_CONFIG_SYSTEM \ + GIT_CONFIG_GLOBAL \ + GIT_CONFIG_NOSYSTEM \ + GIT_EXEC_PATH \ + GIT_CEILING_DIRECTORIES \ + GIT_DISCOVERY_ACROSS_FILESYSTEM \ + GIT_PREFIX \ + GIT_INTERNAL_SUPER_PREFIX \ + GIT_GRAFT_FILE \ + GIT_SHALLOW_FILE +ROOT=$(git -C "$ROOT" rev-parse --path-format=absolute --show-toplevel) +COMMON_DIR=$(git -C "$ROOT" rev-parse --path-format=absolute --git-common-dir) +COMMON_DIR=$(cd "$COMMON_DIR" && pwd -P) +CANONICAL_HOOKS="$COMMON_DIR/hooks" +MANAGED_ROOT="$COMMON_DIR/hackforger-boundary-hooks" +REMOTE_NAME=origin + +fatal() { + echo "FATAL: $*" >&2 + exit 1 +} + +is_oid() { + [[ "$1" =~ ^[0-9a-f]{40}$ || "$1" =~ ^[0-9a-f]{64}$ ]] +} + +canonical_hooks_are_samples_only() { + local entry name + [ -e "$CANONICAL_HOOKS" ] || return 0 + [ -d "$CANONICAL_HOOKS" ] && [ ! -L "$CANONICAL_HOOKS" ] || return 1 + while IFS= read -r -d '' entry; do + name=${entry##*/} + [ -f "$entry" ] && [ ! -L "$entry" ] && [[ "$name" == *.sample ]] || return 1 + done < <(find "$CANONICAL_HOOKS" -mindepth 1 -maxdepth 1 -print0) +} + +managed_hook_path() { + local candidate=$1 suffix + [[ "$candidate" == "$MANAGED_ROOT/"* ]] || return 1 + suffix=${candidate#"$MANAGED_ROOT/"} + is_oid "$suffix" || return 1 + [ "$candidate" = "$MANAGED_ROOT/$suffix" ] +} + +verify_bundle() { + local directory=$1 expected=$2 count + [ -d "$directory" ] && [ ! -L "$directory" ] || return 1 + count=$(find "$directory" -mindepth 1 -maxdepth 1 -print | wc -l | tr -d '[:space:]') + [ "$count" = 5 ] || return 1 + for name in \ + pre-push \ + check_public_repository_boundary.py \ + boundary_guard_policy.py \ + private-content-markers.txt \ + source-commit + do + [ -f "$directory/$name" ] && [ ! -L "$directory/$name" ] || return 1 + cmp -s "$directory/$name" "$expected/$name" || return 1 + done + [ -x "$directory/pre-push" ] && [ -x "$directory/check_public_repository_boundary.py" ] +} + +restore_local_config() { + if [ "$PREVIOUS_LOCAL_SET" = 1 ]; then + git -C "$ROOT" config --local core.hooksPath "$PREVIOUS_LOCAL" + else + git -C "$ROOT" config --local --unset-all core.hooksPath 2>/dev/null || true + fi +} + +command -v git >/dev/null 2>&1 || fatal "git is required" +command -v python3 >/dev/null 2>&1 || fatal "python3 is required" +ORIGIN_URL=$(git -C "$ROOT" config --local --get-all remote.origin.url 2>/dev/null || true) +case "$ORIGIN_URL" in + git@github.com:HackForger/hackforger.git|https://github.com/HackForger/hackforger.git) + ;; + *) + fatal "origin must be the canonical HackForger/hackforger SSH or HTTPS URL" + ;; +esac + +# Disable local replace refs while identifying and extracting the trusted tree. +export GIT_NO_REPLACE_OBJECTS=1 +REMOTE_HEAD=$(git -C "$ROOT" ls-remote --symref "$REMOTE_NAME" HEAD) \ + || fatal "cannot read the trusted origin default ref" +DEFAULT_REF=$(printf '%s\n' "$REMOTE_HEAD" | awk '$1 == "ref:" && $3 == "HEAD" { print $2 }') +DEFAULT_OID=$(printf '%s\n' "$REMOTE_HEAD" | awk '$2 == "HEAD" && $1 ~ /^[0-9a-f]+$/ { print $1 }') +[[ "$DEFAULT_REF" =~ ^refs/heads/[A-Za-z0-9._/-]+$ ]] \ + || fatal "origin HEAD did not advertise exactly one safe default branch" +git -C "$ROOT" check-ref-format "$DEFAULT_REF" >/dev/null 2>&1 \ + || fatal "origin HEAD advertised an invalid default branch" +is_oid "$DEFAULT_OID" \ + || fatal "origin HEAD did not advertise exactly one valid commit object id" + +TMP_REF="refs/hackforger-boundary/install/$$-$RANDOM" +STAGING= +cleanup() { + git -C "$ROOT" update-ref -d "$TMP_REF" 2>/dev/null || true + if [ -n "${STAGING:-}" ] && [ -d "$STAGING" ]; then + chmod -R u+w "$STAGING" 2>/dev/null || true + rm -rf "$STAGING" + fi +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM HUP + +git -C "$ROOT" fetch --quiet --no-tags "$REMOTE_NAME" "+$DEFAULT_REF:$TMP_REF" \ + || fatal "cannot fetch the trusted origin default ref" +FETCHED_OID=$(git -C "$ROOT" rev-parse --verify "$TMP_REF^{commit}" 2>/dev/null || true) +[ "$FETCHED_OID" = "$DEFAULT_OID" ] \ + || fatal "origin default ref changed during installation; rerun the installer" + +umask 077 +mkdir -p "$MANAGED_ROOT" +[ -d "$MANAGED_ROOT" ] && [ ! -L "$MANAGED_ROOT" ] \ + || fatal "managed hook root is not a real directory" +chmod 0700 "$MANAGED_ROOT" +STAGING=$(mktemp -d "$MANAGED_ROOT/.install.XXXXXX") + +extract_blob() { + local source_path=$1 expected_mode=$2 destination=$3 entry metadata tracked_path mode type oid + entry=$(git -C "$ROOT" ls-tree "$DEFAULT_OID" -- ":(literal)$source_path") + [ -n "$entry" ] || fatal "trusted default commit is missing $source_path" + metadata=${entry%%$'\t'*} + tracked_path=${entry#*$'\t'} + [ "$tracked_path" = "$source_path" ] || fatal "unexpected trusted tree path for $source_path" + read -r mode type oid <<< "$metadata" + [ "$mode" = "$expected_mode" ] && [ "$type" = blob ] && is_oid "$oid" \ + || fatal "trusted default commit has an unsafe entry for $source_path" + git -C "$ROOT" cat-file blob "$oid" > "$destination" +} + +extract_blob scripts/pre-push-public-boundary.sh 100755 "$STAGING/pre-push" +extract_blob scripts/ci/check_public_repository_boundary.py 100755 \ + "$STAGING/check_public_repository_boundary.py" +extract_blob scripts/ci/boundary_guard_policy.py 100644 "$STAGING/boundary_guard_policy.py" +extract_blob scripts/ci/private-content-markers.txt 100644 "$STAGING/private-content-markers.txt" +printf '%s\n' "$DEFAULT_OID" > "$STAGING/source-commit" +chmod 0555 "$STAGING/pre-push" "$STAGING/check_public_repository_boundary.py" +chmod 0444 \ + "$STAGING/boundary_guard_policy.py" \ + "$STAGING/private-content-markers.txt" \ + "$STAGING/source-commit" + +INSTALL_DIR="$MANAGED_ROOT/$DEFAULT_OID" +if [ -e "$INSTALL_DIR" ] || [ -L "$INSTALL_DIR" ]; then + verify_bundle "$INSTALL_DIR" "$STAGING" \ + || fatal "existing immutable hook bundle does not match trusted commit $DEFAULT_OID" +else + chmod 0555 "$STAGING" + mv "$STAGING" "$INSTALL_DIR" + STAGING= +fi +chmod 0555 "$INSTALL_DIR" "$INSTALL_DIR/pre-push" "$INSTALL_DIR/check_public_repository_boundary.py" +chmod 0444 \ + "$INSTALL_DIR/boundary_guard_policy.py" \ + "$INSTALL_DIR/private-content-markers.txt" \ + "$INSTALL_DIR/source-commit" + +CURRENT=$(git -C "$ROOT" config --get-all core.hooksPath 2>/dev/null || true) +[[ "$CURRENT" != *$'\n'* ]] || fatal "multiple or malformed core.hooksPath values are configured" +if [ -z "$CURRENT" ] || [ "$CURRENT" = "$CANONICAL_HOOKS" ]; then + canonical_hooks_are_samples_only \ + || fatal "canonical hooks directory contains a real hook; refusing to replace it" +elif [ "$CURRENT" = "$INSTALL_DIR" ]; then + : +elif managed_hook_path "$CURRENT"; then + [ -f "$CURRENT/pre-push" ] && [ ! -L "$CURRENT/pre-push" ] && [ -x "$CURRENT/pre-push" ] \ + || fatal "configured managed hooks path is not a valid installed bundle" +else + fatal "core.hooksPath is already set to '$CURRENT'; refusing to replace it" +fi + +if PREVIOUS_LOCAL=$(git -C "$ROOT" config --local --get-all core.hooksPath 2>/dev/null); then + PREVIOUS_LOCAL_SET=1 +else + PREVIOUS_LOCAL_SET=0 + PREVIOUS_LOCAL= +fi +[[ "$PREVIOUS_LOCAL" != *$'\n'* ]] || fatal "multiple local core.hooksPath values are configured" + +# Keep the source commit reachable so the immutable hook can use it as its +# trusted opaque-file baseline even after remote-tracking refs move. +git -C "$ROOT" update-ref "refs/hackforger-boundary/installed/$DEFAULT_OID" "$DEFAULT_OID" +git -C "$ROOT" config --local core.hooksPath "$INSTALL_DIR" +EFFECTIVE=$(git -C "$ROOT" rev-parse --path-format=absolute --git-path hooks 2>/dev/null || true) +if [ "$EFFECTIVE" != "$INSTALL_DIR" ] \ + || [ ! -f "$EFFECTIVE/pre-push" ] \ + || [ -L "$EFFECTIVE/pre-push" ] \ + || [ ! -x "$EFFECTIVE/pre-push" ]; then + restore_local_config + fatal "effective pre-push hook is not the installed immutable executable" +fi + +echo "Installed HackForger public-boundary pre-push hook from $DEFAULT_OID" +echo "core.hooksPath=$INSTALL_DIR" diff --git a/scripts/pre-push-public-boundary.sh b/scripts/pre-push-public-boundary.sh new file mode 100755 index 0000000000..a624406488 --- /dev/null +++ b/scripts/pre-push-public-boundary.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This file is source material for scripts/install-public-boundary-hook.sh. The +# effective hook executes only from an immutable, commit-versioned bundle under +# the Git common directory; it never calls back into a mutable checkout. + +HOOK_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +CHECKER="$HOOK_DIR/check_public_repository_boundary.py" +POLICY="$HOOK_DIR/private-content-markers.txt" +GUARD_POLICY="$HOOK_DIR/boundary_guard_policy.py" +SOURCE_COMMIT_FILE="$HOOK_DIR/source-commit" +REMOTE_NAME=${1:-} +TRUSTED_REMOTE=origin +TMP_REF= + +fatal() { + echo "FATAL: $*" >&2 + exit 2 +} + +# Git itself exports repository-local variables to hooks. Drop them and any +# caller-provided object/index/config overlays, then rediscover the worktree +# from the cwd that Git sets for a non-bare pre-push hook. SSH transport and +# credential-agent variables remain user-controlled; a user controlling the +# push process can already use --no-verify and is outside this hook's boundary. +unset \ + GIT_DIR \ + GIT_WORK_TREE \ + GIT_IMPLICIT_WORK_TREE \ + GIT_COMMON_DIR \ + GIT_OBJECT_DIRECTORY \ + GIT_ALTERNATE_OBJECT_DIRECTORIES \ + GIT_INDEX_FILE \ + GIT_INDEX_VERSION \ + GIT_NAMESPACE \ + GIT_NO_REPLACE_OBJECTS \ + GIT_REPLACE_REF_BASE \ + GIT_CONFIG \ + GIT_CONFIG_PARAMETERS \ + GIT_CONFIG_COUNT \ + GIT_CONFIG_SYSTEM \ + GIT_CONFIG_GLOBAL \ + GIT_CONFIG_NOSYSTEM \ + GIT_EXEC_PATH \ + GIT_CEILING_DIRECTORIES \ + GIT_DISCOVERY_ACROSS_FILESYSTEM \ + GIT_PREFIX \ + GIT_INTERNAL_SUPER_PREFIX \ + GIT_GRAFT_FILE \ + GIT_SHALLOW_FILE + +for artifact in "$CHECKER" "$POLICY" "$GUARD_POLICY" "$SOURCE_COMMIT_FILE"; do + [ -f "$artifact" ] && [ ! -L "$artifact" ] \ + || fatal "immutable public-boundary hook bundle is incomplete" +done +[ -x "$CHECKER" ] || fatal "immutable boundary checker is not executable" + +ROOT=$(git rev-parse --path-format=absolute --show-toplevel 2>/dev/null) \ + || fatal "pre-push hook must run inside a non-bare worktree" +ROOT=$(cd "$ROOT" && pwd -P) +# Do not allow local replace refs to rewrite the commit graph or trusted trees +# inspected by this hook or its checker. +export GIT_NO_REPLACE_OBJECTS=1 + +reject_graph_overrides() { + local grafts shallow + shallow=$(git -C "$ROOT" rev-parse --is-shallow-repository 2>/dev/null) \ + || fatal "cannot determine whether the repository is shallow" + [ "$shallow" = false ] \ + || fatal "shallow repositories are not allowed while enforcing the public boundary; run git fetch --unshallow origin" + grafts=$(git -C "$ROOT" rev-parse --path-format=absolute --git-path info/grafts 2>/dev/null) \ + || fatal "cannot resolve the legacy Git grafts path" + if [ -e "$grafts" ] || [ -L "$grafts" ]; then + [ -f "$grafts" ] && [ ! -L "$grafts" ] && [ ! -s "$grafts" ] \ + || fatal "legacy Git grafts are not allowed while enforcing the public boundary" + fi +} + +[[ "$REMOTE_NAME" =~ ^[A-Za-z0-9._-]+$ ]] \ + || fatal "pre-push remote name has an unsafe form" + +SOURCE_COMMIT=$(cat "$SOURCE_COMMIT_FILE") +[[ "$SOURCE_COMMIT" =~ ^[0-9a-f]{40}$ || "$SOURCE_COMMIT" =~ ^[0-9a-f]{64}$ ]] \ + || fatal "immutable hook bundle has an invalid source commit" +[ "${HOOK_DIR##*/}" = "$SOURCE_COMMIT" ] \ + || fatal "immutable hook bundle path does not match its source commit" + +COMMITS=$(mktemp "${TMPDIR:-/tmp}/hackforger-boundary-commits.XXXXXX") +cleanup() { + rm -f "$COMMITS" + if [ -n "${TMP_REF:-}" ]; then + git -C "$ROOT" update-ref -d "$TMP_REF" 2>/dev/null || true + fi +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM HUP +REF_ARGS=() +HAS_UPDATE=0 +GRAPH_CHECKED=0 + +is_zero_oid() { + [[ "$1" =~ ^0{40}$ || "$1" =~ ^0{64}$ ]] +} + +while read -r local_ref local_oid remote_ref remote_oid; do + [ -n "${local_ref:-}" ] || continue + # A deletion removes public metadata. Do not scan or reject the name/type of + # the ref being deleted, and do not require a baseline for deletion-only pushes. + is_zero_oid "${local_oid:-}" && continue + HAS_UPDATE=1 + if [ "$GRAPH_CHECKED" = 0 ]; then + reject_graph_overrides + GRAPH_CHECKED=1 + fi + REF_ARGS+=(--ref-name "$local_ref" --ref-name "$remote_ref") + [[ "$local_oid" =~ ^[0-9a-f]{40}$ || "$local_oid" =~ ^[0-9a-f]{64}$ ]] \ + || fatal "pre-push local object id is invalid" + [ "$(git -C "$ROOT" cat-file -t "$local_oid" 2>/dev/null)" = commit ] \ + || fatal "annotated tags or non-commit refs require dedicated security review" + local_commit=$(git -C "$ROOT" rev-parse --verify "$local_oid^{commit}" 2>/dev/null) \ + || fatal "only commit-backed refs may be pushed" + git -C "$ROOT" merge-base --is-ancestor "$SOURCE_COMMIT" "$local_commit" 2>/dev/null \ + || fatal "pushed refs must contain the installed boundary source commit; rebase or merge the trusted default" + if is_zero_oid "${remote_oid:-}"; then + git -C "$ROOT" rev-list --reverse "$SOURCE_COMMIT..$local_commit" >> "$COMMITS" + else + [[ "$remote_oid" =~ ^[0-9a-f]{40}$ || "$remote_oid" =~ ^[0-9a-f]{64}$ ]] \ + || fatal "pre-push remote object id is invalid" + remote_commit=$(git -C "$ROOT" rev-parse --verify "$remote_oid^{commit}" 2>/dev/null) \ + || fatal "remote ref does not resolve to a local commit" + git -C "$ROOT" rev-list --reverse "$remote_commit..$local_commit" >> "$COMMITS" + fi +done + +if [ "$HAS_UPDATE" = 0 ]; then + echo "public repository boundary: PASS (ref deletions only)" + exit 0 +fi + +ORIGIN_URL=$(git -C "$ROOT" config --local --get-all remote.origin.url 2>/dev/null || true) +case "$ORIGIN_URL" in + git@github.com:HackForger/hackforger.git|https://github.com/HackForger/hackforger.git) + ;; + *) + fatal "origin must be the canonical HackForger/hackforger SSH or HTTPS URL" + ;; +esac + +verify_guard_tree() { + local commit=$1 source_path=$2 expected_mode=$3 installed=$4 + local entry metadata tracked_path mode type oid + entry=$(git -C "$ROOT" ls-tree "$commit" -- ":(literal)$source_path") + [ -n "$entry" ] || fatal "trusted default is missing boundary guard $source_path; reinstall is required" + metadata=${entry%%$'\t'*} + tracked_path=${entry#*$'\t'} + [ "$tracked_path" = "$source_path" ] \ + || fatal "trusted default returned an unexpected boundary guard path" + read -r mode type oid <<< "$metadata" + [ "$mode" = "$expected_mode" ] && [ "$type" = blob ] \ + && [[ "$oid" =~ ^[0-9a-f]{40}$ || "$oid" =~ ^[0-9a-f]{64}$ ]] \ + || fatal "trusted default has an unsafe boundary guard entry; reinstall is required" + git -C "$ROOT" cat-file blob "$oid" | cmp -s - "$installed" \ + || fatal "installed boundary guard is stale or modified; rerun scripts/install-public-boundary-hook.sh" +} + +verify_guard_bundle_at() { + local commit=$1 + verify_guard_tree "$commit" scripts/pre-push-public-boundary.sh 100755 "$HOOK_DIR/pre-push" + verify_guard_tree "$commit" scripts/ci/check_public_repository_boundary.py 100755 "$CHECKER" + verify_guard_tree "$commit" scripts/ci/boundary_guard_policy.py 100644 "$GUARD_POLICY" + verify_guard_tree "$commit" scripts/ci/private-content-markers.txt 100644 "$POLICY" +} + +# First prove that the non-checkout bundle still matches its pinned source. +verify_guard_bundle_at "$SOURCE_COMMIT" +REMOTE_HEAD=$(git -C "$ROOT" ls-remote --symref "$TRUSTED_REMOTE" HEAD) \ + || fatal "cannot verify the trusted origin default guard; push is blocked" +DEFAULT_REF=$(printf '%s\n' "$REMOTE_HEAD" | awk '$1 == "ref:" && $3 == "HEAD" { print $2 }') +LIVE_OID=$(printf '%s\n' "$REMOTE_HEAD" | awk '$2 == "HEAD" && $1 ~ /^[0-9a-f]+$/ { print $1 }') +[[ "$DEFAULT_REF" =~ ^refs/heads/[A-Za-z0-9._/-]+$ ]] \ + || fatal "origin HEAD did not advertise exactly one safe default branch" +git -C "$ROOT" check-ref-format "$DEFAULT_REF" >/dev/null 2>&1 \ + || fatal "origin HEAD advertised an invalid default branch" +[[ "$LIVE_OID" =~ ^[0-9a-f]{40}$ || "$LIVE_OID" =~ ^[0-9a-f]{64}$ ]] \ + || fatal "origin HEAD did not advertise exactly one valid commit object id" +if [ "$LIVE_OID" != "$SOURCE_COMMIT" ]; then + TMP_REF="refs/hackforger-boundary/pre-push/$$-$RANDOM" + git -C "$ROOT" fetch --quiet --no-tags "$TRUSTED_REMOTE" "+$DEFAULT_REF:$TMP_REF" \ + || fatal "cannot fetch the trusted origin default guard; push is blocked" + FETCHED_OID=$(git -C "$ROOT" rev-parse --verify "$TMP_REF^{commit}" 2>/dev/null || true) + [ "$FETCHED_OID" = "$LIVE_OID" ] \ + || fatal "origin default changed while validating the boundary guard; retry" + verify_guard_bundle_at "$LIVE_OID" +fi + +BASELINE_COMMIT=$(git -C "$ROOT" rev-parse --verify "$SOURCE_COMMIT^{commit}" 2>/dev/null) \ + || fatal "installed boundary source commit is unavailable; reinstall the hook" +# Recheck immediately before the checker starts; it also performs Git graph +# operations while scanning the immutable commit list. +reject_graph_overrides +LC_ALL=C sort -u "$COMMITS" -o "$COMMITS" +python3 "$CHECKER" \ + --root "$ROOT" \ + --policy "$POLICY" \ + --baseline-ref "$BASELINE_COMMIT" \ + --history-commit-list "$COMMITS" \ + --history-only \ + "${REF_ARGS[@]}" diff --git a/skills/hackforger-development/SKILL.md b/skills/hackforger-development/SKILL.md new file mode 100644 index 0000000000..05cc18d76c --- /dev/null +++ b/skills/hackforger-development/SKILL.md @@ -0,0 +1,102 @@ +--- +name: hackforger-development +description: Develop, review, test, document, or deploy HackForger while preserving Forgejo architecture and the public-repository boundary. Use for changes to HackForger Go code, templates, locales, custom assets, CI, deployment tooling, agent instructions, or repository documentation, especially when work may involve branded content, instance-specific configuration, production operations, or a separate private content repository. +--- + +# HackForger development + +Keep HackForger reusable and business-neutral while following Forgejo's codebase conventions. + +## Start safely + +1. Verify the repository, branch, worktree, and authoritative task source before editing. +2. Use an isolated worktree for non-trivial changes. Preserve unrelated user changes. +3. Read the nearest repository instructions and the files that own the behavior. +4. On a trusted clone, run `bash scripts/install-public-boundary-hook.sh`; it + refuses to replace an unrelated existing `core.hooksPath`. Re-run it after + an audited boundary-guard update when the installed snapshot reports stale. +5. Run `bash scripts/check-public-repository-boundary.sh` before and after the change. + +Run the boundary check before every push, not only before merge. A branch in a +public GitHub repository is already public when pushed; CI can reject it but +cannot undo that first disclosure. Use a private business repository for +business work and a fork for untrusted public contributions. + +The installed, commit-versioned pre-push hook scans every outgoing commit +version, including files added and later deleted, plus outgoing ref metadata. +The standalone boundary command additionally scans both staged Git blobs and +differing worktree bytes. `--no-verify` is forbidden for ordinary work. A +dedicated guard-update branch may use it only after explicit security-owner +approval, with the local and ruleset bypass recorded in the security review. + +## Enforce the repository boundary + +Keep these in the public HackForger repository: + +- reusable application code and tests; +- neutral templates, examples, fixtures, and documentation; +- generic deployment primitives driven by required configuration; +- CI and tooling that enforce this boundary. + +Put these in the appropriate access-controlled business repository: + +- branded landing pages, copy, media, campaign data, and business help content; +- real domains, hosts, addresses, accounts, filesystem layouts, and topology; +- instance-specific deployment configuration, runbooks, reports, and evidence; +- business-specific wrappers around the generic hydration or deployment tools. + +Keep passwords, tokens, private keys, and other credentials in a secret manager or ignored local environment file, not in either Git repository. + +When a request needs business-specific material: + +1. Resolve the business repository from the task handoff, + `HACKFORGER_PRIVATE_CONTENT_REPO`, or an operator-provided sibling-workspace + path. Do not guess a repository name. +2. Confirm the target repository is access-controlled (`visibility=private`) + before writing. +3. Add reusable capability to HackForger and concrete business content to the private repository. +4. If the private repository is unavailable or ambiguous, stop and ask; never place the content temporarily in HackForger. + +Do not solve a boundary failure with an allowlist unless the matched text is genuinely reusable and neutral. + +Boundary enforcement is self-protected. Changes to `CODEOWNERS`, any GitHub +workflow, the boundary wrapper, checker, tests, or marker policy require a +dedicated security-owner review and an explicit audited ruleset bypass; never +mix such a change with ordinary application or business-content work. + +Do not treat the advisory push workflow as prevention: a branch or tag is +already public when that workflow starts. Maintainers must keep the exact +`Public repository boundary` status and CODEOWNER review required on maintained +branches. If repository writers are outside the trust boundary, use an +independent status-writing GitHub App or an organization-enforced required +workflow; repository workflows share the GitHub Actions App identity. + +## Follow Forgejo architecture + +- Preserve the dependency direction `routers -> services -> models -> modules`. +- Pass `context.Context` first and return typed errors. +- Use `db.WithTx` for multi-table state changes. +- Publish HackForger feed events for state changes. +- Follow existing Forgejo API, template, locale, and test patterns. +- Use Vue 3 for frontend components and justify new Go dependencies. +- Use `gh` for the GitHub-hosted repository; do not add Forgejo Actions for GitHub CI. + +## Hydrate private content safely + +- Treat the private repository as the source of truth; do not copy hydrated output into the public Git index. +- Require a versioned manifest and checksum verification before publishing. +- Stage content outside the public worktree or in an ignored directory. +- Back up the live target before replacement. +- Prefer a validated staging directory and atomic directory swap. Do not use `rsync --delete` for private content. +- Verify the deployed manifest, key files, and public behavior after publication. + +## Verify before completion + +Run the narrowest relevant tests first, then the repository-required gate bundle. At minimum: + +```bash +bash scripts/check-public-repository-boundary.sh +git diff --check +``` + +For deployment tooling, also run syntax checks and a non-production fixture test. For user-facing changes, exercise the real web flow with browser evidence. Separate local proof from remote CI and production proof. diff --git a/skills/hackforger-development/agents/openai.yaml b/skills/hackforger-development/agents/openai.yaml new file mode 100644 index 0000000000..66b4699f33 --- /dev/null +++ b/skills/hackforger-development/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "HackForger Development" + short_description: "Develop HackForger within its public repository boundary" + default_prompt: "Use $hackforger-development to implement this HackForger change without adding business-specific content to the public repository."