diff --git a/.github/scripts/cloud-run-simulation-entry-smoke.sh b/.github/scripts/cloud-run-simulation-entry-smoke.sh index 84c0726da..ee36a32e3 100755 --- a/.github/scripts/cloud-run-simulation-entry-smoke.sh +++ b/.github/scripts/cloud-run-simulation-entry-smoke.sh @@ -25,6 +25,12 @@ previous_revision="${3:-}" # and model. # This is the identity the gateway reports from the routing registry; the # numerical release sentinel checks what the worker computes. +# REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION=true (candidate checks only): +# /ready must report the entrypoint's own database reachable and at the +# migration revision the image expects, which shows that the runtime secret +# reached the revision and that the migration job ran. Stable and rollback +# checks leave it unset, because an older image may face a newer schema. +require_database_revision="${REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION:-}" expected_identity="${EXPECTED_IDENTITY:-none}" expected_policyengine_version="${EXPECTED_POLICYENGINE_VERSION:-}" expected_app_name="${EXPECTED_APP_NAME:-}" @@ -39,6 +45,14 @@ case "${expected_identity}" in exit 2 ;; esac +case "${require_database_revision}" in + "" | true) ;; + *) + printf 'REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION must be true or empty, not %s\n' \ + "${require_database_revision}" >&2 + exit 2 + ;; +esac case "${expected_latest}" in "" | true) ;; *) @@ -131,8 +145,18 @@ while true; do attempt=$((attempt + 1)) done -curl --fail --silent --show-error "${base_url}/ready" | - jq -e '.status == "ready"' >/dev/null +ready_body="$(curl --fail --silent --show-error "${base_url}/ready")" +jq -e '.status == "ready"' <<<"${ready_body}" >/dev/null +if [ "${require_database_revision}" = "true" ] && + ! jq -e ' + .simulation_database.reachable == true + and (.simulation_database.revision | type == "string") + and .simulation_database.revision == .simulation_database.expected_revision + ' <<<"${ready_body}" >/dev/null; then + printf '%s: the simulation database is not at the expected revision: %s\n' \ + "${base_url}" "$(jq -c '.simulation_database' <<<"${ready_body}")" >&2 + exit 1 +fi curl --fail --silent --show-error "${base_url}/versions" --output "${versions_body}" jq -e 'type == "object"' "${versions_body}" >/dev/null diff --git a/.github/scripts/deploy-cloud-run-simulation-entry.sh b/.github/scripts/deploy-cloud-run-simulation-entry.sh index 4320514ff..d0c042930 100755 --- a/.github/scripts/deploy-cloud-run-simulation-entry.sh +++ b/.github/scripts/deploy-cloud-run-simulation-entry.sh @@ -77,6 +77,7 @@ required_variables=( OLD_GATEWAY_AUTH_AUDIENCE_VALUE OLD_GATEWAY_AUTH_CLIENT_ID_VALUE OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME + SIMULATION_DATABASE_URL_SECRET_NAME OBSERVABILITY_SERVICE_NAMESPACE OBSERVABILITY_TRACE_PROJECT_ID OTEL_EXPORTER_OTLP_ENDPOINT @@ -115,10 +116,15 @@ export SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE="${SIMULATION_ENTRYPOINT_ [[ "${SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE}" == "0" || "${SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE}" == "1" ]] \ || fail "SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE must be 0 or 1" +# The runtime role's URL for the entrypoint's own simulation schema. The +# migration job, which uses a separate migrator URL, runs before this deploy. runtime_secrets=( "OLD_GATEWAY_AUTH_CLIENT_SECRET=$( resolve_secret_reference "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}" )" + "SIMULATION_DATABASE_URL=$( + resolve_secret_reference "${SIMULATION_DATABASE_URL_SECRET_NAME}" + )" ) if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then diff --git a/.github/scripts/simulation-database-migrate.sh b/.github/scripts/simulation-database-migrate.sh new file mode 100755 index 000000000..e120e5de7 --- /dev/null +++ b/.github/scripts/simulation-database-migrate.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Apply the simulation schema's Alembic chain, then require the database to be +# at the chain's head. Run from libs/policyengine-simulation-persistence, whose +# locked environment (with the migrations extra) provides Alembic. +# migrations/env.py reads the migrator URL from SIMULATION_MIGRATION_DATABASE_URL; +# this script never prints it. + +set -euo pipefail +set +x + +if [[ -z "${SIMULATION_MIGRATION_DATABASE_URL:-}" ]]; then + printf 'SIMULATION_MIGRATION_DATABASE_URL must name the simulation database\n' >&2 + exit 1 +fi + +alembic_config="${SIMULATION_ALEMBIC_CONFIG:-../../migrations/alembic.ini}" + +uv run --frozen --extra migrations alembic -c "${alembic_config}" upgrade head +uv run --frozen --extra migrations alembic -c "${alembic_config}" current --check-heads diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c69cb49af..4cbb5246b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -16,7 +16,7 @@ jobs: # every job passed. See docs/engineering/skills/github-prs.md. name: CI complete if: ${{ always() }} - needs: [test, lint, quality-guards, docker-build, test-integration] + needs: [test, simulation-database, lint, quality-guards, docker-build, test-integration] runs-on: ubuntu-latest permissions: contents: read @@ -51,6 +51,7 @@ jobs: - projects/policyengine-simulation-gateway - libs/policyengine-simulation-contract - libs/policyengine-simulation-observability + - libs/policyengine-simulation-persistence - libs/policyengine-stage12-persistence - libs/policyengine-fastapi @@ -75,6 +76,53 @@ jobs: working-directory: ${{ matrix.project }} run: uv run pytest tests/ -v + simulation-database: + name: Simulation database + # Runs the simulation schema's migration chain (upgrade, alembic check, + # downgrade, upgrade) and the persistence tests on a fresh PostgreSQL. + # Full CI runs only on the lowest unmerged pull request in a stack. + if: ${{ !github.event.pull_request.stack || github.event.pull_request.stack.base.ref == github.event.pull_request.base.ref }} + runs-on: ubuntu-latest + services: + postgres: + image: postgres:17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: simulation_ci + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: >- + --health-cmd="pg_isready -U postgres" + --health-interval=5s + --health-timeout=5s + --health-retries=20 + env: + SIMULATION_TEST_DATABASE_URL: postgresql://postgres:simulation_ci@127.0.0.1:5432/postgres + # A missing server fails the tests instead of skipping them. + SIMULATION_REQUIRE_DATABASE_TESTS: "1" + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Install dependencies + working-directory: libs/policyengine-simulation-persistence + run: uv sync --extra test --frozen + + - name: Run the migration chain and persistence tests on PostgreSQL + working-directory: libs/policyengine-simulation-persistence + run: uv run pytest tests/ -v -m postgres + lint: name: Lint # Full CI runs only on the lowest unmerged pull request in a stack. diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index 70bc8f111..a48931da4 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -153,10 +153,37 @@ jobs: GATEWAY_AUTH_REQUIRED: ${{ vars.GATEWAY_AUTH_REQUIRED }} run: ../../.github/scripts/modal-sync-secrets.sh "${{ inputs.modal_environment }}" + migrate_simulation_database: + # Applies the simulation schema's Alembic chain before the entrypoint + # candidate deploys. The previous revision keeps serving meanwhile, so + # every migration must work with the release before it. + name: Migrate the simulation database schema + needs: prepare + runs-on: ubuntu-latest + environment: ${{ inputs.release_environment }} + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - uses: astral-sh/setup-uv@v8.1.0 + + - name: Install migration dependencies + working-directory: libs/policyengine-simulation-persistence + run: uv sync --frozen --extra migrations + + - name: Apply simulation schema migrations + working-directory: libs/policyengine-simulation-persistence + env: + SIMULATION_MIGRATION_DATABASE_URL: ${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }} + run: ../../.github/scripts/simulation-database-migrate.sh + deploy_entrypoint: name: Deploy and unit test Cloud Run entrypoint - if: ${{ always() && needs.prepare.result == 'success' && ((inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'success') || (!inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'skipped')) }} - needs: [prepare, configure_stage12_v2] + if: ${{ always() && needs.prepare.result == 'success' && needs.migrate_simulation_database.result == 'success' && ((inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'success') || (!inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'skipped')) }} + needs: [prepare, configure_stage12_v2, migrate_simulation_database] runs-on: ubuntu-latest environment: ${{ inputs.release_environment }} permissions: @@ -237,6 +264,7 @@ jobs: OLD_GATEWAY_AUTH_AUDIENCE_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_AUDIENCE }} OLD_GATEWAY_AUTH_CLIENT_ID_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_CLIENT_ID }} OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME: ${{ vars.OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME }} + SIMULATION_DATABASE_URL_SECRET_NAME: ${{ vars.SIMULATION_DATABASE_URL_SECRET_NAME }} STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }} STAGE12_ARTIFACT_BUCKET_VALUE: ${{ vars.STAGE12_ARTIFACT_BUCKET }} STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }} @@ -744,6 +772,8 @@ jobs: EXPECTED_UK_DATASET_SHA256: ${{ inputs.deploy_existing_stack && needs.prepare.outputs.uk_dataset_sha256 || '' }} # Latest must route to this release only when it should advance it. EXPECTED_LATEST: ${{ inputs.deploy_existing_stack && needs.update_routing.outputs.advances_latest == 'true' && 'true' || '' }} + # The candidate reaches its own database at the migrated revision. + REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION: "true" run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${{ needs.deploy_entrypoint.outputs.candidate_url }}" "${{ needs.deploy_entrypoint.outputs.revision }}" - name: Download the numerical release sentinel diff --git a/AGENTS.md b/AGENTS.md index d28639b40..337bd8967 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,12 +99,19 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. - `sim.stage12.deployment-scope`: Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources or runs the existing stack's complete integration suite; the Stage 12 worker can request a Google OIDC token. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) - `sim.stage12.deployment-uses-api-schema`: Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) -- `sim.stage12.no-migration-chain`: This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py`) - `sim.stage12.no-production-user-associations`: Stage 12 runtime modules never create production user associations (user_simulations or user_reports). (blocking: pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py`) - `sim.stage12.no-schema-ddl`: Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py`) - `sim.stage12.persistence-typed-dml-only`: The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations. (blocking: pytest `libs/policyengine-stage12-persistence/tests/test_source_boundaries.py`) - `sim.stage12.production-lock-has-db-runtime`: The production lock file includes the direct Stage 12 database runtime dependencies. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +### Simulation persistence + +- `sim.persistence.executor-independent`: The Modal executor and gateway never declare, lock or import policyengine-simulation-persistence; only the Simulation Entrypoint and the migration job install it. (blocking: pytest `libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py`) +- `sim.persistence.migrate-before-entrypoint`: Each deployment applies the simulation chain in its own job, the only one that receives SIMULATION_MIGRATION_DATABASE_URL, before the entrypoint candidate deploys; the entrypoint receives SIMULATION_DATABASE_URL as a pinned Secret Manager version; and the candidate smoke requires the database at the image's revision. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py`) +- `sim.persistence.migration-cycle`: On a fresh PostgreSQL, the simulation chain upgrades to its head, passes alembic check, downgrades to base and upgrades again, run by a non-superuser migrator, and the runtime role can read the revision but cannot change the schema. (blocking: pytest `libs/policyengine-simulation-persistence/tests/test_migration_cycle.py`) +- `sim.persistence.no-modal-database-credentials`: No Modal secret, Modal deployment or Modal application receives SIMULATION_DATABASE_URL or SIMULATION_MIGRATION_DATABASE_URL; the Stage 12 comparison's own database secret is the only database credential Modal holds. (blocking: pytest `libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py`) +- `sim.persistence.one-migration-chain`: The simulation schema changes only through the one Alembic chain at migrations/ (configured by migrations/alembic.ini, versioned in simulation.alembic_version); no other Alembic configuration, environment or revision directory exists, and runtime source code contains no schema DDL. (blocking: pytest `libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py`) + ### Deployment pipeline - `sim.deploy.container-context-minimal`: Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) diff --git a/Makefile b/Makefile index dd81d034b..cf6ab99a4 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,7 @@ test: projects/policyengine-simulation-gateway \ libs/policyengine-simulation-contract \ libs/policyengine-simulation-observability \ + libs/policyengine-simulation-persistence \ libs/policyengine-stage12-persistence \ libs/policyengine-fastapi; do \ echo "Testing $$proj..."; \ diff --git a/docs/generated/rules.md b/docs/generated/rules.md index f0ddb6085..f1edaf91a 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -8,11 +8,11 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 39 | 1 | 0 | 14 | 54 | +| mechanical | 43 | 1 | 0 | 14 | 58 | | metric | 0 | 0 | 0 | 0 | 0 | | judgment | 0 | 0 | 0 | 0 | 0 | -**Blocking:** 40 of 54 rules. +**Blocking:** 44 of 58 rules. ## Gaps @@ -54,10 +54,11 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf - `sim.deploy.smoke-tests-strict` - `sim.deploy.traffic-changes-exact` - `sim.deploy.workflow-shell-safety` +- `sim.persistence.migrate-before-entrypoint` +- `sim.persistence.migration-cycle` - `sim.process.rules-registry` - `sim.stage12.deployment-scope` - `sim.stage12.deployment-uses-api-schema` -- `sim.stage12.no-migration-chain` - `sim.stage12.no-production-user-associations` - `sim.stage12.persistence-typed-dml-only` - `sim.stage12.production-lock-has-db-runtime` @@ -92,7 +93,6 @@ flowchart LR | --- | --- | --- | --- | --- | --- | --- | --- | | `sim.stage12.deployment-scope` | Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources or runs the existing stack's complete integration suite; the Stage 12 worker can request a Google OIDC token. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.stage12.deployment-uses-api-schema` | Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | -| `sim.stage12.no-migration-chain` | This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | | `sim.stage12.no-production-user-associations` | Stage 12 runtime modules never create production user associations (user_simulations or user_reports). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | | `sim.stage12.no-schema-ddl` | Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | yes | | `sim.stage12.persistence-typed-dml-only` | The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations. | mechanical | enforced | yes | pytest `libs/policyengine-stage12-persistence/tests/test_source_boundaries.py` | `pr.yml:test` | **unproven** | @@ -104,9 +104,8 @@ flowchart LR e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] j_CI_pr_yml_test["CI pr.yml:test"] r_sim_stage12_deployment_uses_api_schema("sim.stage12.deployment-uses-api-schema (enforced)") - r_sim_stage12_no_migration_chain("sim.stage12.no-migration-chain (enforced)") - e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py["pytest projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py"] r_sim_stage12_no_production_user_associations("sim.stage12.no-production-user-associations (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py["pytest projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py"] r_sim_stage12_no_schema_ddl("sim.stage12.no-schema-ddl (enforced)") r_sim_stage12_persistence_typed_dml_only("sim.stage12.persistence-typed-dml-only (enforced)") e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py["pytest libs/policyengine-stage12-persistence/tests/test_source_boundaries.py"] @@ -114,15 +113,46 @@ flowchart LR r_sim_stage12_deployment_scope --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test r_sim_stage12_deployment_uses_api_schema --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py - r_sim_stage12_no_migration_chain --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py - e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py --> j_CI_pr_yml_test r_sim_stage12_no_production_user_associations --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py + e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py --> j_CI_pr_yml_test r_sim_stage12_no_schema_ddl --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py r_sim_stage12_persistence_typed_dml_only --> e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py --> j_CI_pr_yml_test r_sim_stage12_production_lock_has_db_runtime --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py ``` +## Simulation persistence + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.persistence.executor-independent` | The Modal executor and gateway never declare, lock or import policyengine-simulation-persistence; only the Simulation Entrypoint and the migration job install it. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py` | `pr.yml:test` | yes | +| `sim.persistence.migrate-before-entrypoint` | Each deployment applies the simulation chain in its own job, the only one that receives SIMULATION_MIGRATION_DATABASE_URL, before the entrypoint candidate deploys; the entrypoint receives SIMULATION_DATABASE_URL as a pinned Secret Manager version; and the candidate smoke requires the database at the image's revision. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py` | `pr.yml:test` | **unproven** | +| `sim.persistence.migration-cycle` | On a fresh PostgreSQL, the simulation chain upgrades to its head, passes alembic check, downgrades to base and upgrades again, run by a non-superuser migrator, and the runtime role can read the revision but cannot change the schema. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_migration_cycle.py` | `pr.yml:simulation-database` | **unproven** | +| `sim.persistence.no-modal-database-credentials` | No Modal secret, Modal deployment or Modal application receives SIMULATION_DATABASE_URL or SIMULATION_MIGRATION_DATABASE_URL; the Stage 12 comparison's own database secret is the only database credential Modal holds. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py` | `pr.yml:test` | yes | +| `sim.persistence.one-migration-chain` | The simulation schema changes only through the one Alembic chain at migrations/ (configured by migrations/alembic.ini, versioned in simulation.alembic_version); no other Alembic configuration, environment or revision directory exists, and runtime source code contains no schema DDL. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py` | `pr.yml:test` | yes | + +```mermaid +flowchart LR + r_sim_persistence_executor_independent("sim.persistence.executor-independent (enforced)") + e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py["pytest libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py"] + j_CI_pr_yml_test["CI pr.yml:test"] + r_sim_persistence_migrate_before_entrypoint("sim.persistence.migrate-before-entrypoint (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_simulation_database_deployment_py["pytest projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py"] + r_sim_persistence_migration_cycle("sim.persistence.migration-cycle (enforced)") + e_pytest_libs_policyengine_simulation_persistence_tests_test_migration_cycle_py["pytest libs/policyengine-simulation-persistence/tests/test_migration_cycle.py"] + j_CI_pr_yml_simulation_database["CI pr.yml:simulation-database"] + r_sim_persistence_no_modal_database_credentials("sim.persistence.no-modal-database-credentials (enforced)") + r_sim_persistence_one_migration_chain("sim.persistence.one-migration-chain (enforced)") + r_sim_persistence_executor_independent --> e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py + e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py --> j_CI_pr_yml_test + r_sim_persistence_migrate_before_entrypoint --> e_pytest_projects_policyengine_simulation_entry_tests_test_simulation_database_deployment_py + e_pytest_projects_policyengine_simulation_entry_tests_test_simulation_database_deployment_py --> j_CI_pr_yml_test + r_sim_persistence_migration_cycle --> e_pytest_libs_policyengine_simulation_persistence_tests_test_migration_cycle_py + e_pytest_libs_policyengine_simulation_persistence_tests_test_migration_cycle_py --> j_CI_pr_yml_simulation_database + r_sim_persistence_no_modal_database_credentials --> e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py + r_sim_persistence_one_migration_chain --> e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py +``` + ## Deployment pipeline | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md new file mode 100644 index 000000000..c9e927b8f --- /dev/null +++ b/libs/policyengine-simulation-persistence/README.md @@ -0,0 +1,82 @@ +# PolicyEngine simulation persistence + +SQLAlchemy Core tables and database connections for the Simulation +Entrypoint's own Postgres schema, `simulation`. The Simulation Entrypoint owns +every table in that schema; no other service creates, changes or writes them. + +## Ownership rules + +These rules are registered in `rules/registry.toml` and checked by +`tests/test_ownership_boundaries.py`: + +- **One migration chain** (`sim.persistence.one-migration-chain`). Schema + changes are made only by the Alembic chain in the repository's + `migrations/` directory, configured by `migrations/alembic.ini`. It keeps its + version table at `simulation.alembic_version`. No other Alembic + configuration, environment or revision directory exists in the repository, + and runtime source code contains no schema DDL. +- **The executor does not depend on persistence** + (`sim.persistence.executor-independent`). The Modal executor and gateway + never declare, lock or import this library. Only the Simulation Entrypoint + and the migration job install it. +- **No database credentials in Modal** + (`sim.persistence.no-modal-database-credentials`). No Modal secret or Modal + deployment receives `SIMULATION_DATABASE_URL` or + `SIMULATION_MIGRATION_DATABASE_URL`. The Stage 12 comparison's own database + secret is the only database credential Modal holds, until Stage 12 is + removed. + +The temporary Stage 12 comparison tables are not part of this schema: they are +owned and migrated by the Public API, and `libs/policyengine-stage12-persistence` +writes them with DML only. + +## Database roles + +| Role | Used by | Privileges | +| --- | --- | --- | +| `simulation_migrator` | The deploy workflow's migration job, through `SIMULATION_MIGRATION_DATABASE_URL` | Creates the `simulation` schema if it is missing, and owns every object in it | +| `simulation_runtime` | The Simulation Entrypoint, through `SIMULATION_DATABASE_URL` | `USAGE` on the schema, `SELECT` on the version table, and `SELECT`, `INSERT` and `UPDATE` on the tables the migrations create | + +The initial revision grants the runtime role its privileges, including default +privileges on tables the migrator creates later. Both roles must exist before +the first migration runs; an operator creates them. + +## Deployment + +On every deployment, the migration job (`migrate_simulation_database` in +`.github/workflows/simulation-deploy.reusable.yml`) runs +`alembic upgrade head` before the Simulation Entrypoint's candidate revision is +deployed. The Simulation Entrypoint's readiness check (`GET /ready`) reports +the database's revision next to `HEAD_REVISION`, the revision its image +expects. + +The previous revision keeps serving traffic while the migration runs, and a +rollback moves traffic back to an older image without downgrading the schema. +Every revision must therefore work with code from the release before it: +add columns and tables, and remove them only in a later release once nothing +reads them. + +## Adding a revision + +1. Change the tables in `src/policyengine_simulation_persistence/`. +2. Add a revision under `migrations/versions/` whose `down_revision` is the + current head, and set `HEAD_REVISION` in `schema.py` to its revision ID. +3. Run the tests against a disposable PostgreSQL server (below). They run the + chain's upgrade, `alembic check`, downgrade and upgrade cycle. + +Only one migration PR in this chain may be open at a time; a second one rebases +onto the new head after the first merges. + +## Tests + +```bash +uv sync --extra test +uv run pytest tests/ +``` + +Tests marked `postgres` need a disposable PostgreSQL server. Point +`SIMULATION_TEST_DATABASE_URL` at its superuser, for example +`postgresql://postgres:password@127.0.0.1:5432/postgres`; each test creates and +drops its own database. Without the variable those tests skip. CI's +`Simulation database` job runs them against `postgres:17` with +`SIMULATION_REQUIRE_DATABASE_TESTS=1`, which turns a skip into a failure. diff --git a/libs/policyengine-simulation-persistence/pyproject.toml b/libs/policyengine-simulation-persistence/pyproject.toml new file mode 100644 index 000000000..7c4a6f847 --- /dev/null +++ b/libs/policyengine-simulation-persistence/pyproject.toml @@ -0,0 +1,41 @@ +[project] +name = "policyengine-simulation-persistence" +version = "0.1.0" +readme = "README.md" +authors = [ + {name = "PolicyEngine", email = "hello@policyengine.org"}, +] +license = {file = "../../LICENSE"} +requires-python = ">=3.13,<3.14" +dependencies = [ + "psycopg[binary]>=3.2,<4", + "sqlalchemy>=2,<3", +] + +[project.optional-dependencies] +# The migration job installs Alembic; the Simulation Entrypoint runtime does not. +migrations = [ + "alembic>=1.16,<2", +] +test = [ + "alembic>=1.16,<2", + "pytest>=8.3.4", + "pytest-cov>=6.1.1", + "pyyaml>=6.0.2", +] + +[build-system] +requires = ["hatchling>=1.26"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +sources = ["src"] +only-include = ["src/policyengine_simulation_persistence"] +packages = ["src/policyengine_simulation_persistence"] + +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] +markers = [ + "postgres: needs a disposable PostgreSQL server named by SIMULATION_TEST_DATABASE_URL", +] diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py new file mode 100644 index 000000000..fb098c8ed --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -0,0 +1,25 @@ +"""Tables and connections for the Simulation Entrypoint's ``simulation`` schema.""" + +from policyengine_simulation_persistence.engine import ( + create_simulation_engine, + normalized_database_url, + read_schema_revision, +) +from policyengine_simulation_persistence.schema import ( + ALEMBIC_VERSION_TABLE, + HEAD_REVISION, + RUNTIME_ROLE, + SIMULATION_SCHEMA, + simulation_metadata, +) + +__all__ = [ + "ALEMBIC_VERSION_TABLE", + "HEAD_REVISION", + "RUNTIME_ROLE", + "SIMULATION_SCHEMA", + "create_simulation_engine", + "normalized_database_url", + "read_schema_revision", + "simulation_metadata", +] diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py new file mode 100644 index 000000000..85619a5be --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py @@ -0,0 +1,74 @@ +"""Database connections for the Simulation Entrypoint's own schema.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import create_engine, inspect, select +from sqlalchemy.engine import URL, make_url + +from policyengine_simulation_persistence.schema import ( + ALEMBIC_VERSION_TABLE, + SIMULATION_SCHEMA, + alembic_version, +) + +if TYPE_CHECKING: + from sqlalchemy import Connection, Engine + +CONNECT_TIMEOUT_SECONDS = 5 +# Each Cloud Run instance serves up to 80 concurrent requests, but only the +# simulation job routes use the database, and each holds a connection for one +# short transaction. +POOL_SIZE = 2 +MAX_OVERFLOW = 3 + + +def normalized_database_url(database_url: str) -> URL: + """Return ``database_url`` with the psycopg 3 driver selected. + + The stored secrets use the canonical ``postgresql://`` scheme. Raise + ValueError, without echoing the value, when the URL is missing, malformed + or incomplete. + """ + if not database_url or database_url != database_url.strip(): + msg = "Simulation database URL is missing or malformed" + raise ValueError(msg) + try: + parsed = make_url(database_url) + except Exception as error: + msg = "Simulation database URL is malformed" + raise ValueError(msg) from error + if parsed.drivername != "postgresql": + msg = "Simulation database URL must use postgresql://" + raise ValueError(msg) + if not parsed.username or not parsed.host or not parsed.database: + msg = "Simulation database URL is incomplete" + raise ValueError(msg) + return parsed.set(drivername="postgresql+psycopg") + + +def create_simulation_engine(database_url: str) -> Engine: + """Build the engine the Simulation Entrypoint uses; it creates no objects.""" + return create_engine( + normalized_database_url(database_url), + pool_size=POOL_SIZE, + max_overflow=MAX_OVERFLOW, + pool_pre_ping=True, + connect_args={ + "connect_timeout": CONNECT_TIMEOUT_SECONDS, + # Never use server-side prepared statements, so the same engine + # works through Supabase's session and transaction poolers. + "prepare_threshold": None, + }, + ) + + +def read_schema_revision(connection: Connection) -> str | None: + """Return the applied Alembic revision, or None before the first migration.""" + if not inspect(connection).has_table( + ALEMBIC_VERSION_TABLE, + schema=SIMULATION_SCHEMA, + ): + return None + return connection.execute(select(alembic_version.c.version_num)).scalar() diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/schema.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/schema.py new file mode 100644 index 000000000..8cf367b96 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/schema.py @@ -0,0 +1,43 @@ +"""Names and metadata of the Simulation Entrypoint's own Postgres schema. + +The Simulation Entrypoint owns every table in the ``simulation`` schema. One +Alembic chain, at the repository's ``migrations/`` directory, creates and +changes them, and records its revision in ``simulation.alembic_version``. +""" + +from __future__ import annotations + +from sqlalchemy import Column, MetaData, String, Table + +SIMULATION_SCHEMA = "simulation" +ALEMBIC_VERSION_TABLE = "alembic_version" +# The login role the Simulation Entrypoint runtime connects as. Migrations grant +# it schema usage and table DML; it never owns or changes schema objects. +RUNTIME_ROLE = "simulation_runtime" +# The newest revision in migrations/versions. Each migration PR updates it, and +# tests/test_schema.py fails when it differs from the chain's head. The +# readiness check reports it next to the database's revision. +HEAD_REVISION = "0001" + +NAMING_CONVENTION = { + "ix": "ix_%(table_name)s_%(column_0_N_name)s", + "uq": "uq_%(table_name)s_%(column_0_N_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + +simulation_metadata = MetaData( + schema=SIMULATION_SCHEMA, + naming_convention=NAMING_CONVENTION, +) + +# Alembic creates and owns this table; it is described here, outside +# simulation_metadata, only so the runtime can read the applied revision +# without depending on Alembic. +_alembic_metadata = MetaData(schema=SIMULATION_SCHEMA) +alembic_version = Table( + ALEMBIC_VERSION_TABLE, + _alembic_metadata, + Column("version_num", String(32), primary_key=True), +) diff --git a/libs/policyengine-simulation-persistence/tests/conftest.py b/libs/policyengine-simulation-persistence/tests/conftest.py new file mode 100644 index 000000000..c31819dba --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/conftest.py @@ -0,0 +1,114 @@ +"""Disposable PostgreSQL databases for the migration and store tests. + +Tests marked ``postgres`` need a PostgreSQL server whose superuser URL is in +``SIMULATION_TEST_DATABASE_URL``. Each test gets a new database owned by a +non-superuser migrator role, so migrations run with the privileges they have in +production. Without the variable those tests skip, unless +``SIMULATION_REQUIRE_DATABASE_TESTS=1`` (set in CI), where they fail instead. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import TYPE_CHECKING +from uuid import uuid4 + +import pytest +from migration_support import ( + MIGRATION_DATABASE_URL_VARIABLE, + MIGRATOR_ROLE, + TEST_ROLE_PASSWORD, +) +from policyengine_simulation_persistence.schema import RUNTIME_ROLE +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL, make_url + +if TYPE_CHECKING: + from collections.abc import Iterator + +DATABASE_URL_VARIABLE = "SIMULATION_TEST_DATABASE_URL" +REQUIRE_DATABASE_VARIABLE = "SIMULATION_REQUIRE_DATABASE_TESTS" + + +@dataclass(frozen=True) +class DisposableDatabase: + """One empty database and the URLs of the roles that use it.""" + + name: str + migrator_url: str + runtime_url: str + superuser_url: str + + +def _canonical(url: URL) -> str: + return url.set(drivername="postgresql").render_as_string(hide_password=False) + + +def _ensure_login_role(connection: object, role_name: str) -> None: + connection.execute( # type: ignore[attr-defined] + text( + "DO $$ BEGIN " + f"IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '{role_name}') " + f"THEN CREATE ROLE {role_name} LOGIN PASSWORD '{TEST_ROLE_PASSWORD}'; " + "END IF; END $$" + ) + ) + + +@pytest.fixture(scope="session") +def postgres_server_url() -> URL: + """Return the superuser URL of the test server, or skip without one.""" + configured = os.environ.get(DATABASE_URL_VARIABLE, "") + if not configured: + message = f"{DATABASE_URL_VARIABLE} names no PostgreSQL server" + if os.environ.get(REQUIRE_DATABASE_VARIABLE) == "1": + pytest.fail(message) + pytest.skip(message) + return make_url(configured).set(drivername="postgresql+psycopg") + + +@pytest.fixture +def disposable_database( + postgres_server_url: URL, + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[DisposableDatabase]: + """Create an empty database owned by the migrator, and drop it afterwards. + + ``SIMULATION_MIGRATION_DATABASE_URL`` points at it, as in the deploy job. + """ + database_name = f"simulation_test_{uuid4().hex[:12]}" + server_engine = create_engine(postgres_server_url, isolation_level="AUTOCOMMIT") + try: + with server_engine.connect() as connection: + _ensure_login_role(connection, MIGRATOR_ROLE) + _ensure_login_role(connection, RUNTIME_ROLE) + connection.execute( + text(f'CREATE DATABASE "{database_name}" OWNER {MIGRATOR_ROLE}') + ) + database = DisposableDatabase( + name=database_name, + migrator_url=_canonical( + postgres_server_url.set( + username=MIGRATOR_ROLE, + password=TEST_ROLE_PASSWORD, + database=database_name, + ) + ), + runtime_url=_canonical( + postgres_server_url.set( + username=RUNTIME_ROLE, + password=TEST_ROLE_PASSWORD, + database=database_name, + ) + ), + superuser_url=_canonical(postgres_server_url.set(database=database_name)), + ) + monkeypatch.setenv(MIGRATION_DATABASE_URL_VARIABLE, database.migrator_url) + yield database + finally: + with server_engine.connect() as connection: + connection.execute( + text(f'DROP DATABASE IF EXISTS "{database_name}" WITH (FORCE)') + ) + server_engine.dispose() diff --git a/libs/policyengine-simulation-persistence/tests/migration_support.py b/libs/policyengine-simulation-persistence/tests/migration_support.py new file mode 100644 index 000000000..fde328bed --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/migration_support.py @@ -0,0 +1,18 @@ +"""Paths and names shared by the migration and store tests.""" + +from __future__ import annotations + +from pathlib import Path + +from alembic.config import Config + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +ALEMBIC_CONFIG_PATH = REPOSITORY_ROOT / "migrations" / "alembic.ini" +MIGRATION_DATABASE_URL_VARIABLE = "SIMULATION_MIGRATION_DATABASE_URL" +MIGRATOR_ROLE = "simulation_migrator" +TEST_ROLE_PASSWORD = "simulation-test-password" + + +def alembic_config() -> Config: + """Return the repository's Alembic configuration.""" + return Config(str(ALEMBIC_CONFIG_PATH)) diff --git a/libs/policyengine-simulation-persistence/tests/ownership_checks.py b/libs/policyengine-simulation-persistence/tests/ownership_checks.py new file mode 100644 index 000000000..47902f6b0 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/ownership_checks.py @@ -0,0 +1,217 @@ +"""Checks behind the simulation schema ownership rules in rules/registry.toml. + +Each check takes the files it inspects, so the rule tests run it on the +repository and the proof tests run it on the fixtures under +``rules/fixtures//``. +""" + +from __future__ import annotations + +import ast +import configparser +import re +from pathlib import Path +from typing import TYPE_CHECKING + +import tomllib +import yaml + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + +CHAIN_DIRECTORY = Path("migrations") +CHAIN_CONFIG = CHAIN_DIRECTORY / "alembic.ini" +CHAIN_REVISIONS = CHAIN_DIRECTORY / "versions" +SIMULATION_SCHEMA = "simulation" +IGNORED_DIRECTORY_NAMES = frozenset( + {".git", ".venv", "node_modules", "__pycache__", ".pytest_cache", ".ruff_cache"} +) +# Proof fixtures deliberately contain bad layouts. +IGNORED_RELATIVE_DIRECTORIES = (Path("rules") / "fixtures",) +PERSISTENCE_DISTRIBUTION = "policyengine-simulation-persistence" +PERSISTENCE_PACKAGE = "policyengine_simulation_persistence" +SIMULATION_DATABASE_CREDENTIAL = re.compile(r"SIMULATION_(?:MIGRATION_)?DATABASE_URL") +MODAL_SECRET_COMMAND = re.compile(r"\bmodal\s+secret\s+create\b") +MODAL_DEPLOY_COMMAND = re.compile(r"\bmodal\s+deploy\b") +SCHEMA_CHANGE = re.compile( + r"\b(?:CREATE|ALTER|DROP|TRUNCATE)\s+(?:TABLE|TYPE|SCHEMA|INDEX)\b" + r"|\b(?:create_all|drop_all)\s*\(", + flags=re.IGNORECASE, +) + + +def _repository_files(root: Path, pattern: str) -> Iterator[Path]: + for path in sorted(root.rglob(pattern)): + relative = path.relative_to(root) + if IGNORED_DIRECTORY_NAMES.intersection(relative.parts): + continue + if any( + relative.is_relative_to(ignored) for ignored in IGNORED_RELATIVE_DIRECTORIES + ): + continue + yield path + + +def _configures_simulation_version_table(environment_source: str) -> bool: + for node in ast.walk(ast.parse(environment_source)): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "configure" + ): + continue + for keyword in node.keywords: + if keyword.arg != "version_table_schema": + continue + value = keyword.value + if isinstance(value, ast.Name) and value.id == "SIMULATION_SCHEMA": + return True + if isinstance(value, ast.Constant) and value.value == SIMULATION_SCHEMA: + return True + return False + + +def migration_chain_violations(root: Path) -> list[str]: + """Return how ``root`` departs from one simulation chain at ``migrations/``.""" + root = root.resolve() + violations: list[str] = [] + configurations = [ + path.relative_to(root) for path in _repository_files(root, "alembic*.ini") + ] + if configurations != [CHAIN_CONFIG]: + violations.append( + f"expected only {CHAIN_CONFIG} as an Alembic configuration, found " + f"{[str(path) for path in configurations]}" + ) + for versions in _repository_files(root, "versions"): + relative = versions.relative_to(root) + if versions.is_dir() and relative != CHAIN_REVISIONS: + if any(versions.glob("*.py")): + violations.append(f"revision directory outside the chain: {relative}") + for environment in _repository_files(root, "env.py"): + relative = environment.relative_to(root) + if relative != CHAIN_DIRECTORY / "env.py" and "alembic" in ( + environment.read_text(encoding="utf-8") + ): + violations.append(f"Alembic environment outside the chain: {relative}") + + config_path = root / CHAIN_CONFIG + if not config_path.is_file(): + return violations + parser = configparser.ConfigParser(interpolation=None) + parser.read(config_path, encoding="utf-8") + script_location = parser.get("alembic", "script_location", fallback="") + resolved = Path(script_location.replace("%(here)s", str(config_path.parent))) + if not resolved.is_absolute(): + resolved = root / resolved + if resolved.resolve() != (root / CHAIN_DIRECTORY).resolve(): + violations.append(f"script_location {script_location!r} is not migrations/") + environment_path = root / CHAIN_DIRECTORY / "env.py" + if not environment_path.is_file() or not _configures_simulation_version_table( + environment_path.read_text(encoding="utf-8") + ): + violations.append( + "migrations/env.py must keep its version table in the simulation schema" + ) + return violations + + +def schema_change_violations(source_roots: Iterable[Path]) -> list[str]: + """Return runtime source files that create or change schema objects.""" + return [ + str(path) + for source_root in source_roots + for path in sorted(source_root.rglob("*.py")) + if SCHEMA_CHANGE.search(path.read_text(encoding="utf-8")) + ] + + +def declares_persistence_dependency(pyproject_text: str) -> bool: + """Return whether a pyproject declares the persistence library anywhere.""" + project = tomllib.loads(pyproject_text).get("project", {}) + requirements = [ + *project.get("dependencies", []), + *( + requirement + for group in project.get("optional-dependencies", {}).values() + for requirement in group + ), + ] + return any( + re.split(r"[\s\[<>=!~;]", requirement, maxsplit=1)[0] + == PERSISTENCE_DISTRIBUTION + for requirement in requirements + ) + + +def persistence_dependency_violations(project_root: Path) -> list[str]: + """Return where a project depends on or imports the persistence library.""" + violations: list[str] = [] + pyproject = project_root / "pyproject.toml" + if declares_persistence_dependency(pyproject.read_text(encoding="utf-8")): + violations.append(f"{pyproject} declares {PERSISTENCE_DISTRIBUTION}") + lock = project_root / "uv.lock" + if lock.is_file() and f'name = "{PERSISTENCE_DISTRIBUTION}"' in lock.read_text( + encoding="utf-8" + ): + violations.append(f"{lock} locks {PERSISTENCE_DISTRIBUTION}") + violations.extend( + f"{path} imports {PERSISTENCE_PACKAGE}" + for path in sorted((project_root / "src").rglob("*.py")) + if PERSISTENCE_PACKAGE in path.read_text(encoding="utf-8") + ) + return violations + + +def _step_text(step: dict) -> str: + return "\n".join( + str(value) + for value in (step.get("run", ""), step.get("uses", ""), step.get("with", "")) + ) + + +def modal_credential_violations( + workflow_texts: dict[str, str], + script_texts: dict[str, str], +) -> list[str]: + """Return Modal secret or deploy steps that carry simulation database URLs. + + ``workflow_texts`` and ``script_texts`` map a file name to its contents. A + step counts as giving Modal a value when it creates a Modal secret or + deploys a Modal application, directly or through a script that does. + """ + modal_scripts = { + name + for name, text in script_texts.items() + if MODAL_SECRET_COMMAND.search(text) or MODAL_DEPLOY_COMMAND.search(text) + } + violations = [ + f"{name} gives Modal a simulation database credential" + for name in sorted(modal_scripts) + if SIMULATION_DATABASE_CREDENTIAL.search(script_texts[name]) + ] + for workflow_name, workflow_text in sorted(workflow_texts.items()): + workflow = yaml.safe_load(workflow_text) or {} + for job_name, job in (workflow.get("jobs") or {}).items(): + for step in job.get("steps") or []: + text = _step_text(step) + reaches_modal = ( + MODAL_SECRET_COMMAND.search(text) + or MODAL_DEPLOY_COMMAND.search(text) + or any(name in text for name in modal_scripts) + ) + if not reaches_modal: + continue + environment = step.get("env") or {} + carried = [ + f"{key}={value}" + for key, value in environment.items() + if SIMULATION_DATABASE_CREDENTIAL.search(f"{key}={value}") + ] + if carried or SIMULATION_DATABASE_CREDENTIAL.search(text): + violations.append( + f"{workflow_name} job {job_name} step " + f"{step.get('name', '')!r} gives Modal a " + "simulation database credential" + ) + return violations diff --git a/libs/policyengine-simulation-persistence/tests/test_migration_cycle.py b/libs/policyengine-simulation-persistence/tests/test_migration_cycle.py new file mode 100644 index 000000000..9a3f9931b --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_migration_cycle.py @@ -0,0 +1,131 @@ +"""Run the simulation migration chain against a fresh PostgreSQL database.""" + +from __future__ import annotations + +import pytest +from alembic import command +from migration_support import MIGRATOR_ROLE, alembic_config +from policyengine_simulation_persistence import ( + HEAD_REVISION, + SIMULATION_SCHEMA, + create_simulation_engine, + read_schema_revision, +) +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.exc import ProgrammingError + +pytestmark = pytest.mark.postgres + + +def _revision(database_url: str) -> str | None: + engine = create_simulation_engine(database_url) + try: + with engine.connect() as connection: + return read_schema_revision(connection) + finally: + engine.dispose() + + +def test_upgrade_check_downgrade_upgrade_cycle(disposable_database) -> None: + config = alembic_config() + + command.upgrade(config, "head") + assert _revision(disposable_database.migrator_url) == HEAD_REVISION + command.check(config) + + command.downgrade(config, "base") + assert _revision(disposable_database.migrator_url) is None + + command.upgrade(config, "head") + assert _revision(disposable_database.migrator_url) == HEAD_REVISION + command.check(config) + + +def test_the_version_table_lives_only_in_the_simulation_schema( + disposable_database, +) -> None: + command.upgrade(alembic_config(), "head") + + engine = create_engine( + disposable_database.superuser_url.replace( + "postgresql://", "postgresql+psycopg://", 1 + ) + ) + try: + with engine.connect() as connection: + inspector = inspect(connection) + assert inspector.has_table("alembic_version", schema=SIMULATION_SCHEMA) + assert not inspector.has_table("alembic_version", schema="public") + finally: + engine.dispose() + + +def test_an_unmigrated_database_reports_no_revision(disposable_database) -> None: + assert _revision(disposable_database.migrator_url) is None + + +def _precreate_schema_without_database_create(database) -> None: + """Set up the operator alternative to granting the migrator CREATE. + + The superuser takes over the database, so the migrator keeps only the + default CONNECT and TEMPORARY privileges, and creates the simulation + schema owned by the migrator. + """ + engine = create_engine( + database.superuser_url.replace("postgresql://", "postgresql+psycopg://", 1), + isolation_level="AUTOCOMMIT", + ) + try: + with engine.connect() as connection: + connection.execute( + text(f'ALTER DATABASE "{database.name}" OWNER TO CURRENT_USER') + ) + connection.execute( + text(f"CREATE SCHEMA {SIMULATION_SCHEMA} AUTHORIZATION {MIGRATOR_ROLE}") + ) + migrator_may_create = connection.execute( + text( + "SELECT has_database_privilege(:role, current_database(), 'CREATE')" + ), + {"role": MIGRATOR_ROLE}, + ).scalar() + finally: + engine.dispose() + assert migrator_may_create is False + + +def test_a_precreated_schema_needs_no_database_create_privilege( + disposable_database, +) -> None: + _precreate_schema_without_database_create(disposable_database) + + command.upgrade(alembic_config(), "head") + + assert _revision(disposable_database.migrator_url) == HEAD_REVISION + assert _revision(disposable_database.runtime_url) == HEAD_REVISION + + +def test_the_runtime_role_reads_the_revision_but_cannot_change_the_schema( + disposable_database, +) -> None: + command.upgrade(alembic_config(), "head") + + assert _revision(disposable_database.runtime_url) == HEAD_REVISION + engine = create_simulation_engine(disposable_database.runtime_url) + try: + with ( + pytest.raises(ProgrammingError, match="permission denied"), + engine.begin() as connection, + ): + connection.execute( + text("CREATE TABLE simulation.unowned (identifier integer)") + ) + with ( + pytest.raises(ProgrammingError, match="permission denied"), + engine.begin() as connection, + ): + connection.execute( + text("UPDATE simulation.alembic_version SET version_num = ''") + ) + finally: + engine.dispose() diff --git a/libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py b/libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py new file mode 100644 index 000000000..ecd0c923f --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py @@ -0,0 +1,96 @@ +"""Simulation schema ownership: one chain, no executor dependency, none in Modal.""" + +from __future__ import annotations + +from migration_support import REPOSITORY_ROOT +from ownership_checks import ( + declares_persistence_dependency, + migration_chain_violations, + modal_credential_violations, + persistence_dependency_violations, + schema_change_violations, +) + +FIXTURES = REPOSITORY_ROOT / "rules" / "fixtures" +WORKFLOWS = REPOSITORY_ROOT / ".github" / "workflows" +WORKFLOW_SCRIPTS = REPOSITORY_ROOT / ".github" / "scripts" +MODAL_PROJECTS = ( + REPOSITORY_ROOT / "projects" / "policyengine-simulation-executor", + REPOSITORY_ROOT / "projects" / "policyengine-simulation-gateway", +) + + +def _texts(directory, pattern): + return { + path.name: path.read_text(encoding="utf-8") + for path in sorted(directory.glob(pattern)) + } + + +def test_repository_has_one_simulation_migration_chain() -> None: + assert migration_chain_violations(REPOSITORY_ROOT) == [] + + +def test_runtime_sources_change_the_schema_only_through_the_chain() -> None: + source_roots = ( + REPOSITORY_ROOT / "libs" / "policyengine-simulation-persistence" / "src", + REPOSITORY_ROOT / "projects" / "policyengine-simulation-entry" / "src", + ) + + assert schema_change_violations(source_roots) == [] + + +def test_rule_proof_one_migration_chain() -> None: + """Proof for rule sim.persistence.one-migration-chain (rules/registry.toml).""" + fixtures = FIXTURES / "sim.persistence.one-migration-chain" + + assert migration_chain_violations(fixtures / "bad_repository") != [] + assert migration_chain_violations(fixtures / "good_repository") == [] + + +def test_modal_projects_never_depend_on_the_persistence_library() -> None: + for project_root in MODAL_PROJECTS: + assert persistence_dependency_violations(project_root) == [] + + +def test_rule_proof_executor_independent() -> None: + """Proof for rule sim.persistence.executor-independent (rules/registry.toml).""" + fixtures = FIXTURES / "sim.persistence.executor-independent" + + assert declares_persistence_dependency( + (fixtures / "bad_pyproject.toml").read_text(encoding="utf-8") + ) + assert not declares_persistence_dependency( + (fixtures / "good_pyproject.toml").read_text(encoding="utf-8") + ) + + +def test_modal_never_receives_simulation_database_credentials() -> None: + assert ( + modal_credential_violations( + _texts(WORKFLOWS, "*.yml"), + _texts(WORKFLOW_SCRIPTS, "*.sh"), + ) + == [] + ) + for project_root in MODAL_PROJECTS: + for path in sorted((project_root / "src").rglob("*.py")): + assert "SIMULATION_DATABASE_URL" not in path.read_text(encoding="utf-8") + assert "SIMULATION_MIGRATION_DATABASE_URL" not in path.read_text( + encoding="utf-8" + ) + + +def test_rule_proof_no_modal_database_credentials() -> None: + """Proof for rule sim.persistence.no-modal-database-credentials.""" + fixtures = FIXTURES / "sim.persistence.no-modal-database-credentials" + scripts = {} + + assert modal_credential_violations( + {"bad.yml": (fixtures / "bad_workflow.yml").read_text(encoding="utf-8")}, + scripts, + ) + assert not modal_credential_violations( + {"good.yml": (fixtures / "good_workflow.yml").read_text(encoding="utf-8")}, + scripts, + ) diff --git a/libs/policyengine-simulation-persistence/tests/test_schema.py b/libs/policyengine-simulation-persistence/tests/test_schema.py new file mode 100644 index 000000000..393512dcd --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_schema.py @@ -0,0 +1,95 @@ +"""Schema names, connection settings and the chain head, without a database.""" + +from __future__ import annotations + +import pytest +from alembic import command +from alembic.script import ScriptDirectory +from migration_support import MIGRATION_DATABASE_URL_VARIABLE, alembic_config +from policyengine_simulation_persistence import ( + HEAD_REVISION, + SIMULATION_SCHEMA, + create_simulation_engine, + normalized_database_url, + simulation_metadata, +) + + +def test_head_revision_names_the_single_head_of_the_chain() -> None: + script = ScriptDirectory.from_config(alembic_config()) + + assert script.get_heads() == [HEAD_REVISION] + + +def test_every_table_is_in_the_simulation_schema() -> None: + assert simulation_metadata.schema == SIMULATION_SCHEMA + for table in simulation_metadata.tables.values(): + assert table.schema == SIMULATION_SCHEMA, table.name + + +def test_database_url_selects_psycopg_without_changing_the_secret_format() -> None: + url = normalized_database_url( + "postgresql://simulation_runtime:secret@db.example:5432/postgres" + ) + + assert url.drivername == "postgresql+psycopg" + assert url.username == "simulation_runtime" + assert url.database == "postgres" + + +@pytest.mark.parametrize( + "database_url", + [ + "", + " postgresql://user:secret@db.example/postgres", + "postgres://user:secret@db.example/postgres", + "postgresql+psycopg://user:secret@db.example/postgres", + "postgresql://db.example/postgres", + "not a url", + ], +) +def test_malformed_database_urls_are_rejected_without_echoing_them( + database_url, +) -> None: + with pytest.raises(ValueError, match="Simulation database URL") as error: + normalized_database_url(database_url) + + assert "secret" not in str(error.value) + + +def test_runtime_engine_never_prepares_server_side_statements(monkeypatch) -> None: + captured_options = {} + + def capture_engine(url, **options): + captured_options.update(options, url=url) + return object() + + monkeypatch.setattr( + "policyengine_simulation_persistence.engine.create_engine", + capture_engine, + ) + + create_simulation_engine("postgresql://runtime:secret@db.example/postgres") + + assert captured_options["connect_args"]["prepare_threshold"] is None + assert captured_options["connect_args"]["connect_timeout"] > 0 + assert captured_options["pool_pre_ping"] is True + + +def test_migrations_refuse_to_run_without_the_migration_database_url( + monkeypatch, +) -> None: + monkeypatch.delenv(MIGRATION_DATABASE_URL_VARIABLE, raising=False) + + with pytest.raises(RuntimeError, match=MIGRATION_DATABASE_URL_VARIABLE): + command.upgrade(alembic_config(), "head") + + +def test_migrations_refuse_offline_sql_generation(monkeypatch) -> None: + monkeypatch.setenv( + MIGRATION_DATABASE_URL_VARIABLE, + "postgresql://simulation_migrator:secret@db.example/postgres", + ) + + with pytest.raises(RuntimeError, match="online"): + command.upgrade(alembic_config(), "head", sql=True) diff --git a/libs/policyengine-simulation-persistence/uv.lock b/libs/policyengine-simulation-persistence/uv.lock new file mode 100644 index 000000000..a0aff9e60 --- /dev/null +++ b/libs/policyengine-simulation-persistence/uv.lock @@ -0,0 +1,281 @@ +version = 1 +revision = 5 +requires-python = "==3.13.*" + +[[package]] +name = "alembic" +version = "1.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/aa/02910bdb8e2f1444f6654d5b296cd827d126f82209050ee7b1000f92ac4b/alembic-1.20.0.tar.gz", hash = "sha256:db505480647bc60386c5369402f4a57a506b7539c9e9ef5e270d45cbbe4939bf", size = 2093272, upload-time = "2026-09-11T19:09:11.126Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/27/78a89b55b0904d222183164e079b4ca56208e94eff1d35ad1f1ad5be9b06/alembic-1.20.0-py3-none-any.whl", hash = "sha256:77eb101048d95f982c0353e9233404889dcd7a6fc244c107836c0e2fc9cf7d9d", size = 268719, upload-time = "2026-09-11T19:09:12.88Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/55/d1eaf3e73781174340a00dc1ba2aee8a65f82fadb18e2797b192b6b3925b/coverage-7.16.2.tar.gz", hash = "sha256:ca64d9f1f384f151b9511bec01126072acd2f313439f8ed015a22d8790aab6fa", size = 971999, upload-time = "2026-09-27T12:29:01.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/f6/8eb4f220ef24f84fb27d852d4f9bf83e0c73ec1a4a08dd9a87e3f4529739/coverage-7.16.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1a37c6e478cf687e1aa30a593d19c92c02fad9d122b51ab73f51b8dc7a0c0fc9", size = 223993, upload-time = "2026-09-27T12:26:40.164Z" }, + { url = "https://files.pythonhosted.org/packages/40/23/d4bbaf0c154e0b0c2b5264890dbf6ef098dcb50ec8f2469be9490d191660/coverage-7.16.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0993d0e90858c03943d3cb152e068a20dd4707924deec84dd2230261baae3b1b", size = 224368, upload-time = "2026-09-27T12:26:41.762Z" }, + { url = "https://files.pythonhosted.org/packages/7f/48/fc1e88fd571ec5cb38150b7f89f7696ca1bdf9920e01432febb69774cc85/coverage-7.16.2-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bb2fc905bbf4e6b7f40806ea79e31515abf6349594cdf0adf27c4215f0463204", size = 255358, upload-time = "2026-09-27T12:26:43.442Z" }, + { url = "https://files.pythonhosted.org/packages/1d/56/6785397d07c29c8e70fbb9a07e97d062b43c21ffc5f12385917847f09f63/coverage-7.16.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4358b9c8c0125b460407f3017c6cce8156e904b32772c5630d27112f52bdbfe5", size = 257955, upload-time = "2026-09-27T12:26:45.725Z" }, + { url = "https://files.pythonhosted.org/packages/27/3b/c8cdd07721e5f99abd81cea970d971997f99bf158c0b85f51bd284179c8b/coverage-7.16.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f15254427c9b33eedac4f198eaf9e356eb4f6214551afb43da6194a2c088ad7", size = 259183, upload-time = "2026-09-27T12:26:47.208Z" }, + { url = "https://files.pythonhosted.org/packages/9b/11/606b192fe43d32574ec6238549d48de588fdcc18485682a5ec0a8ac357f2/coverage-7.16.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9a75a4704ff640e46170042eec1f984385a121227c505d5a16ad8e495f452541", size = 261325, upload-time = "2026-09-27T12:26:49.084Z" }, + { url = "https://files.pythonhosted.org/packages/67/90/eea481f8b0305ceeb33f081a5f47e298391dbd1b589de0c4b3b3aa50d3f2/coverage-7.16.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:14253fc7bb15749b849795a06f5d3b6d8bc3fb8a4b5ddc341faf7a89dce205fc", size = 255531, upload-time = "2026-09-27T12:26:50.509Z" }, + { url = "https://files.pythonhosted.org/packages/6b/be/dedbf9aea1457b120c27ac10b8fc2a357f37fa2b54c3e7286d42980a0a2a/coverage-7.16.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:921415102a90637fcc2e3f169f61dad7699ecf690e8639fc21b813acbedc0967", size = 257329, upload-time = "2026-09-27T12:26:52.005Z" }, + { url = "https://files.pythonhosted.org/packages/fa/cb/b25c19d5bb2bd0f2e4e27fe8e2ffcae80c7a91ae181c0dc749ed60e9b1a4/coverage-7.16.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cce2bc991293f15cc4084ca116827b5900c5f34e1a54dfe83f10ab5c43162eb7", size = 255294, upload-time = "2026-09-27T12:26:53.634Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a2/892c5c5f4ad44b7b2ca009aee705191f3f268f15052244f2f9e3539b2e35/coverage-7.16.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:e1fa594c887365b69745f25a416806e61085dd07b94c9eae68a6e20730629b23", size = 259444, upload-time = "2026-09-27T12:26:55.243Z" }, + { url = "https://files.pythonhosted.org/packages/ed/99/a562537deba0a3e370182ae71c149be796c39d8087365f17a09188f27145/coverage-7.16.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:11e597173af1dc33d5f8a7332ada544199269a223af1ee1770ddd5e245ad0fe8", size = 255111, upload-time = "2026-09-27T12:26:56.851Z" }, + { url = "https://files.pythonhosted.org/packages/2d/20/854ec68641a9b3362ff068a32dfa41637299761617ef253791dbade6fc76/coverage-7.16.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3e7f99698ba3a7d13988bdd984b7ebf13af4dbe2166dc8502eef90d77603b0a4", size = 256882, upload-time = "2026-09-27T12:26:58.41Z" }, + { url = "https://files.pythonhosted.org/packages/db/0d/748e4518b0ac0f9ff2687c248a6e5f8c0737306e709372632a2556f84443/coverage-7.16.2-cp313-cp313-win32.whl", hash = "sha256:f80bd9f9633eafc73d0a913ba2645c96ba58bba1befc30590f7c0fbfde59d865", size = 226045, upload-time = "2026-09-27T12:26:59.983Z" }, + { url = "https://files.pythonhosted.org/packages/31/fa/6e46edba66a183fe4d99d4bb52c173287e9b8dddabe0888d24cb8210e580/coverage-7.16.2-cp313-cp313-win_amd64.whl", hash = "sha256:8be099e979fc42559328a21828281b4578304191ae46ed4e80a407048a82eee6", size = 226581, upload-time = "2026-09-27T12:27:01.494Z" }, + { url = "https://files.pythonhosted.org/packages/1b/d9/9ef6845367600b336ff75d000444a0d32497d6972c833141bd39356abf68/coverage-7.16.2-cp313-cp313-win_arm64.whl", hash = "sha256:28ff850182a67d117990fa2ce5ea1032836d8c9630dae867e8bdd3bff4533b79", size = 226409, upload-time = "2026-09-27T12:27:03.116Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0c/7a64e1ac90541a8edf50daef0914848011fb057a5bf55284a4811e21939a/coverage-7.16.2-py3-none-any.whl", hash = "sha256:11d28e9123a9156cb405d8d27b44256c9a58fb5decc2073a8f17862057e3aa0f", size = 215754, upload-time = "2026-09-27T12:28:59.075Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "mako" +version = "1.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/09/e07c4b5579a79f4b16f8d4f29f6c54514ac787c4ad506b8c4f28a0e6b0bf/mako-1.4.3.tar.gz", hash = "sha256:cd6537fe88d5fec315c55c2f8529bc4ce7a9a352ad7db3eeaa6a66e2dd4ec37a", size = 412799, upload-time = "2026-09-22T20:54:31.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/a0/053d6af3e8f871e0073b4a36732d9e65be77a72e5434c31b94f6af78a6bb/mako-1.4.3-py3-none-any.whl", hash = "sha256:723296007c870bfd6b3f0c3230dba7198096e5269297ebf5e4eff9e7ffa39d4f", size = 80164, upload-time = "2026-09-22T20:54:33.128Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz", hash = "sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6", size = 153777, upload-time = "2026-10-02T23:07:22.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/18/4bc5ba32499e87bb2b0ef5b3a9bb9c00a131fa961ddf0be548cb550f548b/markupsafe-3.0.4-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1", size = 12786, upload-time = "2026-10-02T23:05:08.709Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6f/17f0c099bf25f3e31e63cc19244d9f6af861a9a4ab778c203997903cfdd0/markupsafe-3.0.4-cp313-cp313-android_24_x86_64.whl", hash = "sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1", size = 12510, upload-time = "2026-10-02T23:05:09.93Z" }, + { url = "https://files.pythonhosted.org/packages/11/af/1a141081b905036ee904ec4bd945e1f70b4e1b32d33c4e59e8cf1d58b247/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96", size = 11583, upload-time = "2026-10-02T23:05:10.884Z" }, + { url = "https://files.pythonhosted.org/packages/e7/0a/a89385ae590232622a03e091805cff12f24fabe6c11e0e8bae096cece81c/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148", size = 11801, upload-time = "2026-10-02T23:05:11.913Z" }, + { url = "https://files.pythonhosted.org/packages/ed/85/ea548dc013962eb73653124bc595635fbf9e0fa41d1f181a967ccb784dfb/markupsafe-3.0.4-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e", size = 11443, upload-time = "2026-10-02T23:05:12.887Z" }, + { url = "https://files.pythonhosted.org/packages/cc/72/15f2e5ec9cf2eb00d5cdfe968d94e4156a7bd7303832c3f3b2c403a36839/markupsafe-3.0.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248", size = 11665, upload-time = "2026-10-02T23:05:13.829Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e0/4030bea613677e333c8a2c901fd405055f657f9d06acba5b7357984b6ef7/markupsafe-3.0.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72", size = 12061, upload-time = "2026-10-02T23:05:14.807Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a5/28b76a7449eb702966b88bef599e2360b411fbb3afeee8fe560939be06ec/markupsafe-3.0.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2", size = 24395, upload-time = "2026-10-02T23:05:15.909Z" }, + { url = "https://files.pythonhosted.org/packages/07/6c/21232811afc3a063b5e934b1ae2efda52f46154ec382f585149c020e61fe/markupsafe-3.0.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85", size = 27183, upload-time = "2026-10-02T23:05:16.976Z" }, + { url = "https://files.pythonhosted.org/packages/14/38/6ccdfa5b59049cb36fb80cbc80aee9cf1fc9bb77d1335ad435f2070b08cf/markupsafe-3.0.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde", size = 24772, upload-time = "2026-10-02T23:05:18.209Z" }, + { url = "https://files.pythonhosted.org/packages/63/e0/cec6865dfe88cb48fedd4b20aed6af5158e41092adcbf3e028bcc6ec2108/markupsafe-3.0.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6", size = 23017, upload-time = "2026-10-02T23:05:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/ee/76/6ed4940bb7648a9aac457c14f870cfdd5105f139a0fb1f29cd61fafa47d1/markupsafe-3.0.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f", size = 22036, upload-time = "2026-10-02T23:05:20.352Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4f/ed476226d4fe46a09090a36025bf319296810028df55eb12f1253b540f3a/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39", size = 23786, upload-time = "2026-10-02T23:05:21.576Z" }, + { url = "https://files.pythonhosted.org/packages/9a/35/66ff30450e35ef5fba9ebc930c9411747e537fd9447b65e44f5007e2b84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee", size = 22672, upload-time = "2026-10-02T23:05:22.922Z" }, + { url = "https://files.pythonhosted.org/packages/32/0b/72f45ce4b4efcbca4b80cf1b06703eff0be8d37e82abb78f66c85a7ead1e/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2", size = 24107, upload-time = "2026-10-02T23:05:24.175Z" }, + { url = "https://files.pythonhosted.org/packages/d2/03/71776e5fdcba04614b384cc102e8a4198208579d896fd1394cb7cb9aa900/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46", size = 21615, upload-time = "2026-10-02T23:05:25.215Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/801ce02a02e7aee0f784b1ec7843026178f6adeb9c93ac67eb1992a9a84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17", size = 23076, upload-time = "2026-10-02T23:05:26.423Z" }, + { url = "https://files.pythonhosted.org/packages/4a/85/c43776625428f3bb4a61e8633940400e3efe6409e3c6f5bff26de5e45618/markupsafe-3.0.4-cp313-cp313-win32.whl", hash = "sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0", size = 14118, upload-time = "2026-10-02T23:05:27.716Z" }, + { url = "https://files.pythonhosted.org/packages/6f/36/163da64de88a13db79214ef75fa041be7fa13bdb42261cf5b7484de14bfb/markupsafe-3.0.4-cp313-cp313-win_amd64.whl", hash = "sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5", size = 14313, upload-time = "2026-10-02T23:05:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a8/9b662783ffaa1149221432a923cee562f78b9cbbb8baa3df9b3753e63e1e/markupsafe-3.0.4-cp313-cp313-win_arm64.whl", hash = "sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc", size = 14189, upload-time = "2026-10-02T23:05:29.917Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "policyengine-simulation-persistence" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] + +[package.optional-dependencies] +migrations = [ + { name = "alembic" }, +] +test = [ + { name = "alembic" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "pyyaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", marker = "extra == 'migrations'", specifier = ">=1.16,<2" }, + { name = "alembic", marker = "extra == 'test'", specifier = ">=1.16,<2" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, + { name = "pyyaml", marker = "extra == 'test'", specifier = ">=6.0.2" }, + { name = "sqlalchemy", specifier = ">=2,<3" }, +] +provides-extras = ["migrations", "test"] + +[[package]] +name = "psycopg" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/26/3ea4ca5eaea1c0debcdf7ee7c1613fbe721dc27a03c461c0817ffd8a0601/psycopg-3.3.6.tar.gz", hash = "sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2", size = 168171, upload-time = "2026-09-18T13:22:55.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/de/748bd7609c71cae5d737f0ba9192f19329f70180ecda8fff3cac02c5abe3/psycopg-3.3.6-py3-none-any.whl", hash = "sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631", size = 215490, upload-time = "2026-09-18T13:15:29.374Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/c3/c072584b69ad44a747b448cfc9766fecb8aae56e372a017e2ef668790057/psycopg_binary-3.3.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ad8f35e67cc16d1fad1fa8c88972dc9b3a3141ea67897399904edab96a301b6", size = 4712284, upload-time = "2026-09-18T13:19:13.451Z" }, + { url = "https://files.pythonhosted.org/packages/0a/b9/4283b785339e8e2318d03048994b093d650ea6289fabaa806b765dc0d449/psycopg_binary-3.3.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:373704aea331d3f3e3402c125a1543f5875e2986ebb54f97d1647942161f803f", size = 4772031, upload-time = "2026-09-18T13:19:18.524Z" }, + { url = "https://files.pythonhosted.org/packages/6f/72/7a1321d359246769fff1affffbd0132785a28f7f63c18524c15a502398f4/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b82491019b884d62318b5f30706c3d7e6d4e5a6cb7eabcb3edc0c1b0fdaceae9", size = 5556392, upload-time = "2026-09-18T13:19:24.418Z" }, + { url = "https://files.pythonhosted.org/packages/de/b0/c6f8a0585a5dacbea74e130bcfc66629390e8f5bbc79d2a8e806e8952150/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cec5ea900390897d0b46130f60bc2883bf19c314f9044235217c8be88b0ef269", size = 5237855, upload-time = "2026-09-18T13:19:31.257Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fc/c3a7a8bbef7e945ec584ac61d460a612363ea398511cd0e220242b1d69f1/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98c02090d88f2ebc0ec1e8da538f77d225ce0fffecf372aa39262e62a1b054ef", size = 6833856, upload-time = "2026-09-18T13:19:43.622Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f2/8e80b921db728ebb68fc105bd7c4277f908210ad755bd6481d5ea7add740/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ee2c4728c691245e24501fcd7a97b5b381236b9985bc445bba88cdce7d1b5784", size = 5070730, upload-time = "2026-09-18T13:19:49.968Z" }, + { url = "https://files.pythonhosted.org/packages/54/6a/5b313e0c5348244f0e973aff3258bf86766656256d5ece8d541a53e35b4a/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f19cc87343eaa55255e76b31259a570072ac95d6ae82c92dd34b97691f5e49dc", size = 4598089, upload-time = "2026-09-18T13:19:56.426Z" }, + { url = "https://files.pythonhosted.org/packages/32/e9/db7f76ec24bf6699e92bf604e5c4bae10664a681a8999ef42aa0faf0f2c6/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdccb3a0e184b03e9baa673b15a809cf36c339c85dbda0ebc25a698846dfbee8", size = 4278481, upload-time = "2026-09-18T13:20:04.681Z" }, + { url = "https://files.pythonhosted.org/packages/61/83/72c67013656f4d6b547caabffb193e91d57e63f90eefdcc6d045c400e97d/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9892188bb15e5803beb51afe8a25add6b56be391a53058e8bca03b74e1e6bf22", size = 4009229, upload-time = "2026-09-18T13:20:11.905Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/5e4500df2c999eb0faed8b184e6958b834172128274f06167a5deef4c19c/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3af90f92769d8cc10f94515ee7a0aef36ea85ca733a0ce22858f6e0953f41138", size = 4321467, upload-time = "2026-09-18T13:20:17.949Z" }, + { url = "https://files.pythonhosted.org/packages/55/7f/e350e1cf498ba2565c3f87b12f429d2012eb86b76c2b3845a19ee5fbb4d6/psycopg_binary-3.3.6-cp313-cp313-win_amd64.whl", hash = "sha256:0ebfad5d131de9f892ae9e70cc7616207768b6714b66a52d4612b8ceaf78b372", size = 3658179, upload-time = "2026-09-18T13:20:22.691Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/bb/28e3106ee586e5b2f5ecf211eec1348055f0f926314b13fe23b9cffa9039/sqlalchemy-2.1.5.tar.gz", hash = "sha256:0df76cf1ca680ad745bd378a6299a058611a8a784eeaeb4ed78ad4a0b7ce2003", size = 10569593, upload-time = "2026-10-16T20:52:20.57Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/aa/f10fc142d2db255249107ee8b21bfbabc10b90f7fadf13c3cf898acc6179/sqlalchemy-2.1.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dc1c86db97372e6bba18a7bcf0f6150dbdd808237f0f52802ef0bec676a94642", size = 2465792, upload-time = "2026-10-16T22:42:39.038Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0d/d395e310e36b5bd39a121cd455504aa1e9111f419efaa88be6b814b4e54d/sqlalchemy-2.1.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c229afac414d691d2f69a6f4e294ac5b5fd334c49da1f4fb86ab7f2a466271d", size = 4599703, upload-time = "2026-10-16T22:55:25.351Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6d/04f4ffc5038571e712c4f88f20d7022091bafab082e85ea5b37a7e8270f1/sqlalchemy-2.1.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1cf433bcb77696cc915552d7f1e8d9117a5dae5aff9fceb06a69d74b3b3c2bd1", size = 4652645, upload-time = "2026-10-16T22:59:23.273Z" }, + { url = "https://files.pythonhosted.org/packages/11/85/08bc336576755daf087e0888e44657f4f72b6676a99522e7a612f0c2481e/sqlalchemy-2.1.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:399d3e212cb557d5bd1ca6f79981e8c3f45c0f6be60b63a9ded790396a81af08", size = 4316833, upload-time = "2026-10-16T23:15:10.011Z" }, + { url = "https://files.pythonhosted.org/packages/a7/5c/476441255cc7afa25e8c75e5a61d3c5c3af7cf61c951ebed2ee9873ec4b1/sqlalchemy-2.1.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a51c56afdb880e9f83736a2e81c02dd002fe1c5ad68346a35d97a2e135b1568", size = 4523134, upload-time = "2026-10-16T22:55:27.374Z" }, + { url = "https://files.pythonhosted.org/packages/63/ca/064e5e3ca7c8f256c8f9b5206eafd410a39226c2ab6fe7e8c1e70725e432/sqlalchemy-2.1.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:17f8c68bd35bb76b2d9ef1097941fb783ef9ef6604c1d2a72c2e276430aeffca", size = 4318058, upload-time = "2026-10-16T23:15:12.139Z" }, + { url = "https://files.pythonhosted.org/packages/90/8c/e38956834896e21038a67193f91b189b9a61555b92fac42368027d5a2524/sqlalchemy-2.1.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:59dbd7412531da0124df8667a244b5155773632e9bd79a5c2ead393b11b59de5", size = 4613916, upload-time = "2026-10-16T22:59:25.19Z" }, + { url = "https://files.pythonhosted.org/packages/14/b3/5531eb9d10d4bd38bd7e6c505776ac73be259dc117c9f07a8c46db4e9a2b/sqlalchemy-2.1.5-cp313-cp313-win32.whl", hash = "sha256:3f566787cb3a6a25ad8c5170e90167d4e8ea103d412aa4ea95021a6df0b95964", size = 2379136, upload-time = "2026-10-16T22:48:37.111Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f6/c48924aafae616d59a2eb0375feeaf4bc4356327a442bda3b9205861b543/sqlalchemy-2.1.5-cp313-cp313-win_amd64.whl", hash = "sha256:fb8dcd5e1cc12fdd4d490819ec7a56e8c02e86ca9087a472e28b422ff932bae0", size = 2429688, upload-time = "2026-10-16T22:48:39.038Z" }, + { url = "https://files.pythonhosted.org/packages/51/84/0855a7d42b700b091af6f817c171c2ff97478b0d5143fe61963adbff9201/sqlalchemy-2.1.5-cp313-cp313-win_arm64.whl", hash = "sha256:65eece9d9494e46de1eb3d49eb438acfcecf059e7a92a2f3daf72ec26d371757", size = 2390320, upload-time = "2026-10-16T22:31:43.871Z" }, + { url = "https://files.pythonhosted.org/packages/de/e9/61176e4753f74ab5f6e72a56314ee654d25c31e6fc98c99060442998cccb/sqlalchemy-2.1.5-py3-none-any.whl", hash = "sha256:2f77b77b716f005b2dc3cc059d56df6a919bfb5b439e4149890a9df6c36be9e3", size = 2060214, upload-time = "2026-10-16T22:43:30.801Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, +] diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 000000000..fea6f84b2 --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,44 @@ +# The Simulation Entrypoint's only Alembic chain: the `simulation` schema. +# Run it from the persistence library's environment, for example: +# uv run --project libs/policyengine-simulation-persistence --extra migrations \ +# alembic -c migrations/alembic.ini upgrade head +# The database URL comes only from SIMULATION_MIGRATION_DATABASE_URL. +# Ownership rules: libs/policyengine-simulation-persistence/README.md + +[alembic] +script_location = %(here)s +path_separator = os +file_template = %%(rev)s_%%(slug)s + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 000000000..a8b00eab5 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,104 @@ +"""Alembic environment for the Simulation Entrypoint's ``simulation`` schema.""" + +from __future__ import annotations + +import os +from logging.config import fileConfig +from typing import TYPE_CHECKING + +from alembic import context +from policyengine_simulation_persistence import ( + ALEMBIC_VERSION_TABLE, + SIMULATION_SCHEMA, + normalized_database_url, + simulation_metadata, +) +from sqlalchemy import create_engine, inspect, pool +from sqlalchemy.schema import CreateSchema + +if TYPE_CHECKING: + from sqlalchemy import Connection + +DATABASE_URL_VARIABLE = "SIMULATION_MIGRATION_DATABASE_URL" + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name, disable_existing_loggers=False) + + +def _include_name(name: str | None, type_: str, _parent_names: object) -> bool: + """Compare only the simulation schema, never other services' schemas.""" + if type_ == "schema": + return name == SIMULATION_SCHEMA + return True + + +def _include_object( + _object: object, + name: str | None, + type_: str, + _reflected: bool, # noqa: FBT001 # Alembic passes these positionally + _compare_to: object, +) -> bool: + """Leave Alembic's own version table out of the schema comparison.""" + return not (type_ == "table" and name == ALEMBIC_VERSION_TABLE) + + +def _ensure_simulation_schema(connection: Connection) -> None: + """Create the simulation schema, owned by the migrator, if it is missing. + + PostgreSQL checks CREATE on the database before it checks whether a schema + exists, even for CREATE SCHEMA IF NOT EXISTS. Creating the schema only when + it is missing lets an operator create it in advance, owned by the migrator, + instead of granting the migrator CREATE on the database. + """ + if not inspect(connection).has_schema(SIMULATION_SCHEMA): + connection.execute(CreateSchema(SIMULATION_SCHEMA)) + + +def _run(connection: Connection) -> None: + # The version table lives in the simulation schema, so the schema must + # exist before Alembic looks for it. + _ensure_simulation_schema(connection) + context.configure( + connection=connection, + target_metadata=simulation_metadata, + version_table=ALEMBIC_VERSION_TABLE, + version_table_schema=SIMULATION_SCHEMA, + include_schemas=True, + include_name=_include_name, + include_object=_include_object, + compare_type=True, + compare_server_default=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Apply migrations to the database named by the migration URL.""" + database_url = os.environ.get(DATABASE_URL_VARIABLE, "") + if not database_url: + msg = f"{DATABASE_URL_VARIABLE} must name the simulation database" + raise RuntimeError(msg) + engine = create_engine( + normalized_database_url(database_url), + poolclass=pool.NullPool, + connect_args={"connect_timeout": 10, "prepare_threshold": None}, + ) + try: + # One transaction covers the schema creation and every revision, so a + # failed migration leaves nothing half-applied. + with engine.begin() as connection: + _run(connection) + finally: + engine.dispose() + + +if context.is_offline_mode(): + msg = ( + "Simulation migrations run only online, against the database they " + "change; offline SQL generation is not supported" + ) + raise RuntimeError(msg) +run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 000000000..a56f67cc9 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +${imports if imports else ""} +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + """Apply this revision.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Revert this revision.""" + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/0001_initial_simulation_schema.py b/migrations/versions/0001_initial_simulation_schema.py new file mode 100644 index 000000000..ec0704669 --- /dev/null +++ b/migrations/versions/0001_initial_simulation_schema.py @@ -0,0 +1,52 @@ +"""Grant the Simulation Entrypoint runtime role access to the simulation schema. + +The environment creates the ``simulation`` schema, owned by the migrator, and +Alembic creates ``simulation.alembic_version`` before this revision runs. This +revision lets the runtime role use the schema, read the applied revision for +the readiness check, and read and write every table the migrator creates in +later revisions. The runtime role never owns or changes schema objects. + +Revision ID: 0001 +Revises: +Create Date: 2026-10-20 +""" + +from __future__ import annotations + +from alembic import op + +revision = "0001" +down_revision = None +branch_labels = None +depends_on = None + +# The statements name the schema and role literally: a migration is a fixed +# historical record, so later renames must not change what it did. +UPGRADE_STATEMENTS = ( + "GRANT USAGE ON SCHEMA simulation TO simulation_runtime", + "GRANT SELECT ON simulation.alembic_version TO simulation_runtime", + "ALTER DEFAULT PRIVILEGES IN SCHEMA simulation " + "GRANT SELECT, INSERT, UPDATE ON TABLES TO simulation_runtime", + "ALTER DEFAULT PRIVILEGES IN SCHEMA simulation " + "GRANT USAGE, SELECT ON SEQUENCES TO simulation_runtime", +) +DOWNGRADE_STATEMENTS = ( + "ALTER DEFAULT PRIVILEGES IN SCHEMA simulation " + "REVOKE USAGE, SELECT ON SEQUENCES FROM simulation_runtime", + "ALTER DEFAULT PRIVILEGES IN SCHEMA simulation " + "REVOKE SELECT, INSERT, UPDATE ON TABLES FROM simulation_runtime", + "REVOKE SELECT ON simulation.alembic_version FROM simulation_runtime", + "REVOKE USAGE ON SCHEMA simulation FROM simulation_runtime", +) + + +def upgrade() -> None: + """Grant schema usage, revision reads and default table privileges.""" + for statement in UPGRADE_STATEMENTS: + op.execute(statement) + + +def downgrade() -> None: + """Revoke everything ``upgrade`` granted.""" + for statement in DOWNGRADE_STATEMENTS: + op.execute(statement) diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index 9783ef1cc..d22fab8ca 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -22,6 +22,30 @@ Credential values belong in the cloud secret store. They must not be committed to the repository or stored as plain GitHub variables. The deployment workflow receives only the configuration needed to reference those protected resources. +## Simulation database + +The Simulation Entrypoint owns the `simulation` schema in the v2 Postgres +database. Before the entrypoint candidate deploys in each environment, the +`migrate_simulation_database` job applies the schema's Alembic chain +(`migrations/`) with the migrator role. It is the only job that receives +`SIMULATION_MIGRATION_DATABASE_URL`, a GitHub environment secret in `beta` and +`prod`. If the migration fails, the entrypoint is not deployed. + +The entrypoint itself connects as the runtime role. The deploy step resolves +the Secret Manager secret named by the `SIMULATION_DATABASE_URL_SECRET_NAME` +environment variable to its newest enabled version and mounts that version as +`SIMULATION_DATABASE_URL`. Only the entrypoint's runtime service account may +read it. Modal never receives either URL. + +`GET /ready` reports the database's applied revision next to the revision the +image expects, under `simulation_database`. The database does not change the +readiness status, because no old-gateway or Stage 12 route uses it. The +candidate smoke check requires the two revisions to match, which shows that the +runtime secret reached the revision and the migration ran; the stable and +rollback checks do not, because an older image may run against a newer schema. +The ownership rules and roles are described in +`libs/policyengine-simulation-persistence/README.md`. + ## Candidate qualification For each eligible revision, the workflow first deploys `beta` diff --git a/projects/policyengine-simulation-entry/Dockerfile.dockerignore b/projects/policyengine-simulation-entry/Dockerfile.dockerignore index 9379db178..a54f9060f 100644 --- a/projects/policyengine-simulation-entry/Dockerfile.dockerignore +++ b/projects/policyengine-simulation-entry/Dockerfile.dockerignore @@ -13,6 +13,8 @@ libs/* !libs/policyengine-simulation-contract/** !libs/policyengine-simulation-observability/ !libs/policyengine-simulation-observability/** +!libs/policyengine-simulation-persistence/ +!libs/policyengine-simulation-persistence/** !libs/policyengine-stage12-persistence/ !libs/policyengine-stage12-persistence/** libs/**/tests diff --git a/projects/policyengine-simulation-entry/pyproject.toml b/projects/policyengine-simulation-entry/pyproject.toml index 9351f4908..e4793eb6a 100644 --- a/projects/policyengine-simulation-entry/pyproject.toml +++ b/projects/policyengine-simulation-entry/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "policyengine-observability[fastapi,google,httpx,otlp-grpc]>=3.0.2,<4", "policyengine-simulation-contract", "policyengine-simulation-observability", + "policyengine-simulation-persistence", "policyengine-stage12-persistence", "pyjwt>=2.10.1,<3", "uvicorn>=0.35.0,<1", @@ -30,6 +31,7 @@ packages = ["src/policyengine_simulation_entry"] [tool.uv.sources] policyengine-simulation-contract = { path = "../../libs/policyengine-simulation-contract", editable = true } policyengine-simulation-observability = { path = "../../libs/policyengine-simulation-observability", editable = true } +policyengine-simulation-persistence = { path = "../../libs/policyengine-simulation-persistence", editable = true } policyengine-stage12-persistence = { path = "../../libs/policyengine-stage12-persistence", editable = true } [project.optional-dependencies] diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index 86fbe3efe..95e3039f2 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -22,7 +22,6 @@ JobSubmitResponse, PingRequest, PingResponse, - ReadinessResponse, SimulationErrorResponse, SimulationRequest, VersionMap, @@ -42,6 +41,7 @@ from policyengine_simulation_observability.observability import ( init_simulation_observability, ) +from policyengine_simulation_persistence import HEAD_REVISION from pydantic import BaseModel, TypeAdapter, ValidationError from starlette.datastructures import MutableHeaders @@ -62,9 +62,12 @@ BackendTelemetryAttributes, BackendTelemetryPayload, RequestIdentifiers, + SimulationDatabaseReadiness, + SimulationEntrypointReadinessResponse, TemporaryStage12ReportResponse, TemporaryStage12SubmissionResponse, ) +from policyengine_simulation_entry.simulation_database import SimulationDatabase from policyengine_simulation_entry.stage12_backend import ( Stage12ComparisonBackend, TemporaryStage12DispatchFailed, @@ -83,6 +86,22 @@ _json_object_adapter = TypeAdapter(JsonObject) type ResponseIdentifier = Literal["job_id", "batch_job_id"] STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS = 5.0 +SIMULATION_DATABASE_READINESS_TIMEOUT_SECONDS = 3.0 + + +class SimulationDatabaseConnection(Protocol): + """The Simulation Entrypoint's own database, as the routes use it. + + The real implementation is ``SimulationDatabase``; tests inject fakes. + """ + + async def read_revision(self) -> str | None: + """Return the applied migration revision, or None before the first one.""" + ... + + def dispose(self) -> None: + """Close every pooled connection.""" + ... class ComparisonBackend(Protocol): @@ -172,9 +191,15 @@ def create_app( backend: SimulationBackend | None = None, auth_dependency: AuthenticationDependency | None = None, comparison_backend: ComparisonBackend | None = None, + simulation_database: SimulationDatabaseConnection | None = None, ) -> FastAPI: """Build the app with injectable auth/backend seams for hermetic tests.""" runtime_settings = settings or Settings.from_env() + runtime_database = simulation_database + if runtime_database is None and runtime_settings.simulation_database_url: + runtime_database = SimulationDatabase.from_url( + runtime_settings.simulation_database_url + ) @asynccontextmanager async def lifespan(_: FastAPI) -> AsyncIterator[None]: @@ -184,6 +209,8 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: yield finally: await runtime_backend.close() + if runtime_database is not None: + runtime_database.dispose() runtime.shutdown() app = FastAPI( @@ -767,13 +794,46 @@ async def get_country_versions(kind: str, request: Request) -> Response: async def health() -> HealthResponse: return HealthResponse() - @app.get("/ready", response_model=ReadinessResponse) + async def simulation_database_readiness() -> SimulationDatabaseReadiness | None: + if runtime_database is None: + return None + try: + revision = await asyncio.wait_for( + runtime_database.read_revision(), + timeout=SIMULATION_DATABASE_READINESS_TIMEOUT_SECONDS, + ) + # Any failure to reach or read the database is reported, never raised: + # the old-gateway routes this check covers do not use it. + except Exception as error: # noqa: BLE001 # reported as unreachable + logger.warning( + "simulation_database_unreachable", + extra={"error_type": type(error).__name__}, + ) + return SimulationDatabaseReadiness( + reachable=False, + revision=None, + expected_revision=HEAD_REVISION, + ) + return SimulationDatabaseReadiness( + reachable=True, + revision=revision, + expected_revision=HEAD_REVISION, + ) + + @app.get("/ready", response_model=SimulationEntrypointReadinessResponse) async def ready() -> Response: - if await runtime_backend.ready(): - return JSONResponse(ReadinessResponse(status="ready").model_dump()) + backend_ready = await runtime_backend.ready() + readiness = SimulationEntrypointReadinessResponse( + status="ready" if backend_ready else "not_ready", + simulation_database=await simulation_database_readiness(), + ) + # Without a configured database the response keeps its old shape. return JSONResponse( - ReadinessResponse(status="not_ready").model_dump(), - status_code=503, + readiness.model_dump( + mode="json", + exclude={"simulation_database"} if runtime_database is None else None, + ), + status_code=200 if backend_ready else 503, ) @app.post( diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py index 5757485d9..a88531302 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py @@ -90,6 +90,10 @@ class Settings: stage12_database_url: str = "" stage12_artifact_bucket: str = "" stage12_retention_days: int = 30 + # The runtime role's URL for the Simulation Entrypoint's own schema. Empty + # outside deployed environments; nothing on the old-gateway or Stage 12 + # paths reads it. + simulation_database_url: str = "" @classmethod def from_env(cls) -> Settings: @@ -140,6 +144,7 @@ def from_env(cls) -> Settings: stage12_database_url=os.getenv("STAGE12_DATABASE_URL", ""), stage12_artifact_bucket=os.getenv("STAGE12_ARTIFACT_BUCKET", ""), stage12_retention_days=int(os.getenv("STAGE12_RETENTION_DAYS", "30")), + simulation_database_url=os.getenv("SIMULATION_DATABASE_URL", ""), ) @property @@ -252,3 +257,8 @@ def validate(self) -> None: ): msg = "STAGE12_DATABASE_URL must use the canonical postgresql:// scheme." raise ConfigurationError(msg) + if self.simulation_database_url and not self.simulation_database_url.startswith( + "postgresql://" + ): + msg = "SIMULATION_DATABASE_URL must use the canonical postgresql:// scheme." + raise ConfigurationError(msg) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py index d387526df..74aa0f74e 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py @@ -115,3 +115,28 @@ class TemporaryStage12ReportResponse(BaseModel): report: ComparisonReportRecord simulations: tuple[ComparisonSimulationRecord, ...] + + +class SimulationDatabaseReadiness(BaseModel): + """The Simulation Entrypoint's own database, as the readiness check found it. + + ``revision`` is the migration revision the database has applied, or null + when it was unreachable or has no migrations yet; ``expected_revision`` is + the newest revision this image knows. Neither changes ``status``: no + old-gateway or Stage 12 route uses this database. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + reachable: bool + revision: str | None + expected_revision: str + + +class SimulationEntrypointReadinessResponse(BaseModel): + """Old-gateway readiness, plus the database revision when one is configured.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + status: Literal["ready", "not_ready"] + simulation_database: SimulationDatabaseReadiness | None = None diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py new file mode 100644 index 000000000..e0ca4f1bd --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py @@ -0,0 +1,42 @@ +"""The Simulation Entrypoint's connection to its own ``simulation`` schema.""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +from policyengine_simulation_persistence import ( + create_simulation_engine, + read_schema_revision, +) + +if TYPE_CHECKING: + from sqlalchemy import Engine + + +class SimulationDatabase: + """Run short, blocking database operations off the event loop.""" + + def __init__(self, engine: Engine) -> None: + """Use ``engine``, which connects as the runtime role.""" + self.engine = engine + + @classmethod + def from_url(cls, database_url: str) -> SimulationDatabase: + """Build the runtime engine; no connection opens until first use. + + Raise ValueError, without echoing the URL, when it is malformed. + """ + return cls(create_simulation_engine(database_url)) + + def _read_revision(self) -> str | None: + with self.engine.connect() as connection: + return read_schema_revision(connection) + + async def read_revision(self) -> str | None: + """Return the applied migration revision, or None before the first one.""" + return await asyncio.to_thread(self._read_revision) + + def dispose(self) -> None: + """Close every pooled connection.""" + self.engine.dispose() diff --git a/projects/policyengine-simulation-entry/tests/test_config.py b/projects/policyengine-simulation-entry/tests/test_config.py index e8bcfa4ad..3f28a1d80 100644 --- a/projects/policyengine-simulation-entry/tests/test_config.py +++ b/projects/policyengine-simulation-entry/tests/test_config.py @@ -162,3 +162,42 @@ def test_stage12_database_url_requires_the_canonical_scheme(database_url): def test_stage12_retention_is_fixed_at_thirty_days(): with pytest.raises(ConfigurationError, match="exactly 30"): make_settings(stage12_retention_days=31).validate() + + +def test_the_simulation_database_is_optional(monkeypatch): + from policyengine_simulation_entry.config import Settings + + monkeypatch.delenv("SIMULATION_DATABASE_URL", raising=False) + settings = Settings.from_env() + + assert settings.simulation_database_url == "" + settings_with_gateway = make_settings(simulation_database_url="") + settings_with_gateway.validate() + + +def test_the_simulation_database_url_is_read_from_the_environment(monkeypatch): + from policyengine_simulation_entry.config import Settings + + monkeypatch.setenv( + "SIMULATION_DATABASE_URL", + "postgresql://simulation_runtime:secret@db.example:5432/postgres", + ) + + assert Settings.from_env().simulation_database_url == ( + "postgresql://simulation_runtime:secret@db.example:5432/postgres" + ) + + +@pytest.mark.parametrize( + "database_url", + [ + "postgres://simulation_runtime:secret@db.example/postgres", + "postgresql+psycopg://simulation_runtime:secret@db.example/postgres", + "mysql://simulation_runtime:secret@db.example/postgres", + ], +) +def test_the_simulation_database_url_must_be_canonical(database_url): + with pytest.raises(ConfigurationError, match="SIMULATION_DATABASE_URL") as error: + make_settings(simulation_database_url=database_url).validate() + + assert "secret" not in str(error.value) diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index ce809ea84..259c732f8 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -132,7 +132,8 @@ def _write_fake_smoke_curl(tmp_path: Path, revisions: list[str]) -> tuple[Path, printf '{"status":"healthy"}' > "${body_file}" ;; */ready) - printf '{"status":"ready"}' + ready_body='{"status":"ready"}' + printf '%s' "${FAKE_CURL_READY:-${ready_body}}" ;; */versions) versions='{}' @@ -348,8 +349,9 @@ def test_deployment_uses_gcloud_workflow_without_terraform(): assert "SIMULATION_ENTRYPOINT_PUBLIC_URL" not in reusable_workflow assert "staging.simulation.api.policyengine.org" not in deploy_workflow assert ( - # Sentinel, restore and final routing acknowledgement use secrets too. - reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 15 + # Sentinel, restore, final routing acknowledgement and the simulation + # database migration use secrets too. + reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 16 ) assert "APP_ENVIRONMENT: ${{ inputs.deployment_environment }}" in reusable_workflow assert "STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }}" in reusable_workflow @@ -487,6 +489,9 @@ def _cloud_run_deploy_environment(tmp_path: Path) -> dict[str, str]: "STAGE12_DATABASE_URL_SECRET_NAME": "stage12-staging-database-url", "MODAL_TOKEN_ID_SECRET_NAME": "modal-token-id", "MODAL_TOKEN_SECRET_SECRET_NAME": "modal-token-secret", + "SIMULATION_DATABASE_URL_SECRET_NAME": ( + "simulation-staging-runtime-database-url" + ), } @@ -525,6 +530,7 @@ def test_cloud_run_deployment_escapes_environment_and_pins_secret_versions(tmp_p secrets_argument = arguments[arguments.index("--set-secrets") + 1] assert secrets_argument == ( "OLD_GATEWAY_AUTH_CLIENT_SECRET=old-client-secret:3," + "SIMULATION_DATABASE_URL=simulation-staging-runtime-database-url:3," "MODAL_TOKEN_ID=modal-token-id:3," "MODAL_TOKEN_SECRET=modal-token-secret:3," "STAGE12_DATABASE_URL=stage12-staging-database-url:3" diff --git a/projects/policyengine-simulation-entry/tests/test_openapi.py b/projects/policyengine-simulation-entry/tests/test_openapi.py index e293b7e0c..d3cade670 100644 --- a/projects/policyengine-simulation-entry/tests/test_openapi.py +++ b/projects/policyengine-simulation-entry/tests/test_openapi.py @@ -106,6 +106,22 @@ def test_canonical_spm_extensions_are_public(): assert "400" in spec["paths"][path]["get"]["responses"] +def test_readiness_documents_the_simulation_database_revision(): + spec = create_app().openapi() + readiness = spec["paths"]["/ready"]["get"]["responses"]["200"] + schema = readiness["content"]["application/json"]["schema"] + database = spec["components"]["schemas"]["SimulationDatabaseReadiness"] + + assert schema == { + "$ref": "#/components/schemas/SimulationEntrypointReadinessResponse" + } + assert set(database["properties"]) == { + "reachable", + "revision", + "expected_revision", + } + + def test_route_table_is_frozen(): assert methods(cast("OpenAPIDocument", create_app().openapi())) == EXPECTED_ROUTES @@ -145,7 +161,9 @@ def test_normalized_contract_matches_old_gateway(): cloud_run_spec ) == normalized_compatibility_paths(gateway_spec) cloud_run_schemas = deepcopy(cloud_run_spec["components"]["schemas"]) - cloud_run_schemas.pop("ReadinessResponse") + # The entrypoint's readiness response adds its own database's revision. + cloud_run_schemas.pop("SimulationEntrypointReadinessResponse") + cloud_run_schemas.pop("SimulationDatabaseReadiness") cloud_run_schemas.pop("SimulationErrorResponse") assert cloud_run_schemas == gateway_spec["components"]["schemas"] gateway_operation_ids = operation_ids(gateway_spec) diff --git a/projects/policyengine-simulation-entry/tests/test_simulation_database.py b/projects/policyengine-simulation-entry/tests/test_simulation_database.py new file mode 100644 index 000000000..95b1defae --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_simulation_database.py @@ -0,0 +1,184 @@ +"""The Simulation Entrypoint's own database: readiness reporting and lifetime.""" + +from __future__ import annotations + +import asyncio + +import pytest +from conftest import FakeBackend, make_settings +from fastapi.testclient import TestClient +from policyengine_simulation_entry import app as app_module +from policyengine_simulation_entry import simulation_database as database_module +from policyengine_simulation_entry.app import create_app +from policyengine_simulation_entry.simulation_database import SimulationDatabase +from policyengine_simulation_persistence import HEAD_REVISION +from sqlalchemy.exc import OperationalError + +DATABASE_URL = "postgresql://simulation_runtime:secret@db.example:5432/postgres" + + +class FakeSimulationDatabase: + """Report a fixed revision, or fail the way an unreachable database does.""" + + def __init__(self, *, revision=HEAD_REVISION, failure=None, delay_seconds=0): + self.revision = revision + self.failure = failure + self.delay_seconds = delay_seconds + self.disposed = False + + async def read_revision(self): + if self.delay_seconds: + await asyncio.sleep(self.delay_seconds) + if self.failure is not None: + raise self.failure + return self.revision + + def dispose(self): + self.disposed = True + + +def _client(backend, database): + return TestClient( + create_app( + settings=make_settings(simulation_database_url=DATABASE_URL), + backend=backend, + auth_dependency=lambda: None, + simulation_database=database, + ) + ) + + +def test_readiness_reports_the_database_revision_and_the_expected_one(): + backend = FakeBackend() + with _client(backend, FakeSimulationDatabase(revision="0001")) as client: + result = client.get("/ready") + + assert result.status_code == 200 + assert result.json() == { + "status": "ready", + "simulation_database": { + "reachable": True, + "revision": "0001", + "expected_revision": HEAD_REVISION, + }, + } + + +def test_readiness_reports_a_database_that_has_no_migrations_yet(): + with _client(FakeBackend(), FakeSimulationDatabase(revision=None)) as client: + database = client.get("/ready").json()["simulation_database"] + + assert database == { + "reachable": True, + "revision": None, + "expected_revision": HEAD_REVISION, + } + + +def test_an_unreachable_database_does_not_change_old_gateway_readiness(caplog): + failure = OperationalError("SELECT 1", {}, Exception("password=secret")) + with _client(FakeBackend(), FakeSimulationDatabase(failure=failure)) as client: + result = client.get("/ready") + + assert result.status_code == 200 + assert result.json() == { + "status": "ready", + "simulation_database": { + "reachable": False, + "revision": None, + "expected_revision": HEAD_REVISION, + }, + } + assert "secret" not in caplog.text + + +def test_readiness_waits_only_briefly_for_the_database(monkeypatch): + monkeypatch.setattr( + app_module, "SIMULATION_DATABASE_READINESS_TIMEOUT_SECONDS", 0.01 + ) + slow_database = FakeSimulationDatabase(delay_seconds=1) + with _client(FakeBackend(), slow_database) as client: + database = client.get("/ready").json()["simulation_database"] + + assert database["reachable"] is False + + +def test_readiness_still_follows_the_old_gateway(): + backend = FakeBackend() + backend.is_ready = False + with _client(backend, FakeSimulationDatabase()) as client: + result = client.get("/ready") + + assert result.status_code == 503 + assert result.json()["status"] == "not_ready" + assert result.json()["simulation_database"]["reachable"] is True + + +def test_the_database_is_released_at_shutdown(): + database = FakeSimulationDatabase() + with _client(FakeBackend(), database): + assert database.disposed is False + + assert database.disposed is True + + +def test_settings_build_the_database_only_when_it_is_configured(monkeypatch): + built_urls = [] + + def build(database_url): + built_urls.append(database_url) + return FakeSimulationDatabase() + + monkeypatch.setattr(SimulationDatabase, "from_url", staticmethod(build)) + + create_app(settings=make_settings(), backend=FakeBackend()) + assert built_urls == [] + + create_app( + settings=make_settings(simulation_database_url=DATABASE_URL), + backend=FakeBackend(), + ) + assert built_urls == [DATABASE_URL] + + +def test_the_database_reads_the_revision_without_blocking_the_event_loop( + monkeypatch, +): + connections = [] + + class FakeConnection: + def __enter__(self): + connections.append(self) + return self + + def __exit__(self, *_): + return False + + class FakeEngine: + disposed = False + + def connect(self): + return FakeConnection() + + def dispose(self): + self.disposed = True + + monkeypatch.setattr( + database_module, + "read_schema_revision", + lambda connection: "0001" if connection in connections else None, + ) + engine = FakeEngine() + database = SimulationDatabase(engine) + + assert asyncio.run(database.read_revision()) == "0001" + database.dispose() + assert engine.disposed is True + + +@pytest.mark.parametrize("database_url", ["", "postgres://user:secret@db/postgres"]) +def test_a_malformed_database_url_never_builds_an_engine(database_url): + with pytest.raises(ValueError, match="Simulation database URL") as error: + SimulationDatabase.from_url(database_url) + + assert "secret" not in str(error.value) diff --git a/projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py b/projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py new file mode 100644 index 000000000..88c6b63e4 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py @@ -0,0 +1,291 @@ +"""Deployment wiring for the Simulation Entrypoint's own database. + +The migration job applies the chain before the entrypoint candidate deploys, +with the migrator URL in that job alone; the entrypoint receives the runtime +URL as a pinned Cloud Run secret; and the candidate smoke requires the +database to be at the image's revision. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest +from test_deployment_assets import ( + CLOUD_RUN_DEPLOY_SCRIPT, + REUSABLE_DEPLOY_WORKFLOW, + SMOKE_SCRIPT, + _cloud_run_deploy_environment, + _gh_eval, + _smoke_test_env, + _workflow_jobs, + _write_fake_smoke_curl, +) +from workflow_script_support import WORKFLOWS_DIR, FakeCommands, run_script + +MIGRATION_SCRIPT = "simulation-database-migrate.sh" +MIGRATION_JOB = "migrate_simulation_database" +MIGRATION_SECRET = "SIMULATION_MIGRATION_DATABASE_URL" +FAKE_MIGRATION_URL = "postgresql://simulation_migrator:secret@db.example/postgres" + + +@pytest.fixture +def fakes(tmp_path: Path) -> FakeCommands: + return FakeCommands.create(tmp_path) + + +# --- The migration job ---------------------------------------------------- + + +def test_the_migration_job_runs_before_the_entrypoint_deploys(): + jobs = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) + migration = jobs[MIGRATION_JOB] + deploy_entrypoint = jobs["deploy_entrypoint"] + + assert migration["needs"] == "prepare" + assert migration["environment"] == "${{ inputs.release_environment }}" + assert MIGRATION_JOB in deploy_entrypoint["needs"] + context = { + "inputs": {"deploy_stage12_v2": True}, + "needs": { + "prepare": {"result": "success"}, + "configure_stage12_v2": {"result": "success"}, + MIGRATION_JOB: {"result": "success"}, + }, + } + assert _gh_eval(deploy_entrypoint["if"], context, job_if=True) + for failed_result in ("failure", "skipped", "cancelled"): + context["needs"][MIGRATION_JOB]["result"] = failed_result + assert not _gh_eval(deploy_entrypoint["if"], context, job_if=True) + + +def test_the_migration_job_applies_the_chain_with_the_migrator_url_alone(): + migration = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)[MIGRATION_JOB] + steps = {step.get("name"): step for step in migration["steps"]} + install = steps["Install migration dependencies"] + apply_step = steps["Apply simulation schema migrations"] + + assert install["working-directory"] == "libs/policyengine-simulation-persistence" + assert install["run"] == "uv sync --frozen --extra migrations" + assert apply_step["working-directory"] == "libs/policyengine-simulation-persistence" + assert apply_step["run"] == f"../../.github/scripts/{MIGRATION_SCRIPT}" + assert apply_step["env"] == { + MIGRATION_SECRET: "${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }}" + } + assert "env" not in migration + assert "permissions" not in migration + + +def test_only_the_migration_step_receives_the_migrator_url(): + carriers = [] + for workflow in sorted(WORKFLOWS_DIR.glob("*.yml")): + for job_name, job in _workflow_jobs(workflow).items(): + if MIGRATION_SECRET in json.dumps(job.get("env", {})): + carriers.append((workflow.name, job_name, "job env")) + for step in job.get("steps", []): + if MIGRATION_SECRET in json.dumps(step): + carriers.append((workflow.name, job_name, step.get("name"))) + + assert carriers == [ + ( + "simulation-deploy.reusable.yml", + MIGRATION_JOB, + "Apply simulation schema migrations", + ) + ] + + +def test_the_migration_script_upgrades_and_then_requires_the_head(fakes): + fakes.add("uv") + + result = run_script( + MIGRATION_SCRIPT, + env=fakes.env(SIMULATION_MIGRATION_DATABASE_URL=FAKE_MIGRATION_URL), + ) + + assert result.returncode == 0, result.stderr + config = "../../migrations/alembic.ini" + assert fakes.calls("uv") == [ + ["uv", "run", "--frozen", "--extra", "migrations", "alembic", "-c", config, + "upgrade", "head"], + ["uv", "run", "--frozen", "--extra", "migrations", "alembic", "-c", config, + "current", "--check-heads"], + ] # fmt: skip + assert "secret" not in result.stdout + result.stderr + + +def test_the_migration_script_refuses_to_run_without_the_migrator_url(fakes): + fakes.add("uv") + + result = run_script( + MIGRATION_SCRIPT, + env=fakes.env(SIMULATION_MIGRATION_DATABASE_URL=""), + ) + + assert result.returncode != 0 + assert MIGRATION_SECRET in result.stderr + assert fakes.calls("uv") == [] + + +def test_a_failed_upgrade_fails_the_job_before_the_head_check(fakes): + fakes.add("uv", 'if [[ " $* " == *" upgrade "* ]]; then exit 3; fi\n') + + result = run_script( + MIGRATION_SCRIPT, + env=fakes.env(SIMULATION_MIGRATION_DATABASE_URL=FAKE_MIGRATION_URL), + ) + + assert result.returncode == 3 + assert len(fakes.calls("uv")) == 1 + + +# --- The entrypoint's runtime URL ----------------------------------------- + + +def _run_deploy(tmp_path, **overrides): + environment = _cloud_run_deploy_environment(tmp_path) | overrides + result = subprocess.run( + [CLOUD_RUN_DEPLOY_SCRIPT], + check=False, + capture_output=True, + text=True, + env=environment, + ) + return result, environment + + +def _secrets_argument(environment) -> str: + arguments = ( + Path(environment["GCLOUD_ARGUMENTS_CAPTURE"]) + .read_text(encoding="utf-8") + .splitlines() + ) + return arguments[arguments.index("--set-secrets") + 1] + + +@pytest.mark.parametrize("deploy_stage12", ["true", "false"]) +def test_the_entrypoint_receives_the_runtime_url_as_a_pinned_secret( + tmp_path, deploy_stage12 +): + result, environment = _run_deploy(tmp_path, DEPLOY_STAGE12_V2=deploy_stage12) + + assert result.returncode == 0, result.stderr + assert ( + "SIMULATION_DATABASE_URL=simulation-staging-runtime-database-url:3" + in _secrets_argument(environment).split(",") + ) + runtime_environment = json.loads( + Path(environment["GCLOUD_ENV_FILE_CAPTURE"]).read_text(encoding="utf-8") + ) + assert "SIMULATION_DATABASE_URL" not in runtime_environment + assert MIGRATION_SECRET not in runtime_environment + + +@pytest.mark.parametrize( + "secret_name", + ["", "not a secret name", "projects/x/secrets/name/versions/1"], +) +def test_the_entrypoint_deploy_requires_a_valid_runtime_url_secret( + tmp_path, secret_name +): + result, _ = _run_deploy(tmp_path, SIMULATION_DATABASE_URL_SECRET_NAME=secret_name) + + assert result.returncode != 0 + assert "SIMULATION_DATABASE_URL_SECRET_NAME" in result.stderr or ( + "Secret Manager reference" in result.stderr + ) + + +def test_the_deploy_step_passes_the_runtime_secret_name(): + deploy_entrypoint = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["deploy_entrypoint"] + deploy_step = next( + step + for step in deploy_entrypoint["steps"] + if step.get("name") == "Deploy Cloud Run entrypoint candidate" + ) + + assert deploy_step["env"]["SIMULATION_DATABASE_URL_SECRET_NAME"] == ( + "${{ vars.SIMULATION_DATABASE_URL_SECRET_NAME }}" + ) + + +# --- The candidate smoke -------------------------------------------------- + + +def test_the_candidate_smoke_requires_the_current_database_revision(): + integration = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["integration"] + smoke = next( + step + for step in integration["steps"] + if step.get("name") == "Verify the complete deployed request path" + ) + + assert smoke["env"]["REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION"] == "true" + + +def _ready_body(revision, expected_revision="0001"): + return json.dumps( + { + "status": "ready", + "simulation_database": { + "reachable": revision is not None, + "revision": revision, + "expected_revision": expected_revision, + }, + } + ) + + +def _run_smoke(tmp_path, ready_body, *, require): + state_file, revisions_file = _write_fake_smoke_curl(tmp_path, ["revision-1"]) + environment = _smoke_test_env(tmp_path, state_file, revisions_file) | { + "FAKE_CURL_READY": ready_body, + } + if require is not None: + environment["REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION"] = require + return subprocess.run( + ["bash", SMOKE_SCRIPT, "https://candidate.example.test", "revision-1"], + check=False, + capture_output=True, + text=True, + env=environment, + ) + + +def test_the_smoke_accepts_a_database_at_the_expected_revision(tmp_path): + result = _run_smoke(tmp_path, _ready_body("0001"), require="true") + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + "ready_body", + [ + _ready_body(None), + _ready_body("0001", expected_revision="0002"), + json.dumps({"status": "ready"}), + ], + ids=["unreachable", "behind", "unconfigured"], +) +def test_the_smoke_rejects_a_database_not_at_the_expected_revision( + tmp_path, ready_body +): + result = _run_smoke(tmp_path, ready_body, require="true") + + assert result.returncode == 1 + assert "simulation database" in result.stderr + + +def test_stable_and_rollback_smokes_ignore_the_database_revision(tmp_path): + result = _run_smoke(tmp_path, _ready_body(None), require=None) + + assert result.returncode == 0, result.stderr + + +def test_the_smoke_refuses_an_unknown_database_requirement(tmp_path): + result = _run_smoke(tmp_path, _ready_body("0001"), require="yes") + + assert result.returncode == 2 + assert "REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION" in result.stderr diff --git a/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py b/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py index b3e84d470..8fc5872a4 100644 --- a/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py +++ b/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py @@ -1,4 +1,8 @@ -"""Keep Stage 12 database schema ownership outside simulation runtimes.""" +"""Keep Stage 12 database schema ownership outside simulation runtimes. + +The Simulation Entrypoint's own schema has its own ownership rules, checked in +libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py. +""" from __future__ import annotations @@ -40,12 +44,6 @@ def test_rule_proof_stage12_no_schema_ddl() -> None: assert DDL_PATTERN.search(good) is None -def test_simulation_repository_has_no_schema_migration_chain() -> None: - assert not list(REPOSITORY_ROOT.glob("alembic*.ini")) - assert not list(REPOSITORY_ROOT.rglob("alembic/versions")) - assert not list(REPOSITORY_ROOT.rglob("migrations/versions")) - - def test_stage12_runtime_cannot_create_production_user_associations() -> None: forbidden = { "user_simulation", diff --git a/projects/policyengine-simulation-entry/uv.lock b/projects/policyengine-simulation-entry/uv.lock index 435ca0f30..a5971d22f 100644 --- a/projects/policyengine-simulation-entry/uv.lock +++ b/projects/policyengine-simulation-entry/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = "==3.13.*" [[package]] @@ -901,6 +901,7 @@ dependencies = [ { name = "policyengine-observability", extra = ["fastapi", "google", "httpx", "otlp-grpc"] }, { name = "policyengine-simulation-contract" }, { name = "policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence" }, { name = "pyjwt" }, { name = "uvicorn" }, @@ -928,6 +929,7 @@ requires-dist = [ { name = "policyengine-observability", extras = ["fastapi", "google", "httpx", "otlp-grpc"], specifier = ">=3.0.2,<4" }, { name = "policyengine-simulation-contract", editable = "../../libs/policyengine-simulation-contract" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence", editable = "../../libs/policyengine-stage12-persistence" }, { name = "pyjwt", specifier = ">=2.10.1,<3" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, @@ -963,6 +965,27 @@ requires-dist = [ ] provides-extras = ["test", "build"] +[[package]] +name = "policyengine-simulation-persistence" +version = "0.1.0" +source = { editable = "../../libs/policyengine-simulation-persistence" } +dependencies = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", marker = "extra == 'migrations'", specifier = ">=1.16,<2" }, + { name = "alembic", marker = "extra == 'test'", specifier = ">=1.16,<2" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, + { name = "pyyaml", marker = "extra == 'test'", specifier = ">=6.0.2" }, + { name = "sqlalchemy", specifier = ">=2,<3" }, +] +provides-extras = ["migrations", "test"] + [[package]] name = "policyengine-stage12-persistence" version = "0.1.0" diff --git a/ruff.toml b/ruff.toml index 1b9aa1535..6965d2202 100644 --- a/ruff.toml +++ b/ruff.toml @@ -7,7 +7,8 @@ per-file-target-version = { "scripts/**" = "py311" } # Directories that are not regular packages on purpose, so they have no # __init__.py: standalone scripts, the implicit namespace package that holds -# scripts.rules, and proof fixtures. Ruff resolves modules as if each listed +# scripts.rules, proof fixtures, and the Alembic chain, whose environment and +# revisions Alembic loads by path. Ruff resolves modules as if each listed # directory, and every directory under it, had an __init__.py. This is a # top-level setting: ruff rejects it in [lint]. The executor's src/modal/utils # scripts are not listed, because ruff would then treat them as part of a @@ -15,6 +16,7 @@ per-file-target-version = { "scripts/**" = "py311" } # an inline INP001 suppression instead. namespace-packages = [ ".github/scripts", + "migrations", "projects/policyengine-simulation-executor/scripts", "rules/fixtures", "scripts", diff --git a/rules/fixtures/sim.persistence.executor-independent/bad_pyproject.toml b/rules/fixtures/sim.persistence.executor-independent/bad_pyproject.toml new file mode 100644 index 000000000..cb1ae0f5a --- /dev/null +++ b/rules/fixtures/sim.persistence.executor-independent/bad_pyproject.toml @@ -0,0 +1,7 @@ +# Proof fixture: a Modal project that depends on the persistence library. +[project] +name = "policyengine-simulation-executor" +dependencies = [ + "policyengine-simulation-contract", + "policyengine-simulation-persistence", +] diff --git a/rules/fixtures/sim.persistence.executor-independent/good_pyproject.toml b/rules/fixtures/sim.persistence.executor-independent/good_pyproject.toml new file mode 100644 index 000000000..b75b2ff09 --- /dev/null +++ b/rules/fixtures/sim.persistence.executor-independent/good_pyproject.toml @@ -0,0 +1,6 @@ +# Proof fixture: a Modal project that depends only on the contract library. +[project] +name = "policyengine-simulation-executor" +dependencies = [ + "policyengine-simulation-contract", +] diff --git a/rules/fixtures/sim.persistence.no-modal-database-credentials/bad_workflow.yml b/rules/fixtures/sim.persistence.no-modal-database-credentials/bad_workflow.yml new file mode 100644 index 000000000..739dbc28b --- /dev/null +++ b/rules/fixtures/sim.persistence.no-modal-database-credentials/bad_workflow.yml @@ -0,0 +1,11 @@ +# Proof fixture: a step that copies the simulation database URL into a Modal secret. +name: Bad deployment +on: workflow_dispatch +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Synchronize Modal runtime secrets + env: + SIMULATION_DATABASE_URL: ${{ secrets.SIMULATION_DATABASE_URL }} + run: uv run modal secret create simulation-database "SIMULATION_DATABASE_URL=${SIMULATION_DATABASE_URL}" --force diff --git a/rules/fixtures/sim.persistence.no-modal-database-credentials/good_workflow.yml b/rules/fixtures/sim.persistence.no-modal-database-credentials/good_workflow.yml new file mode 100644 index 000000000..9a2318ce2 --- /dev/null +++ b/rules/fixtures/sim.persistence.no-modal-database-credentials/good_workflow.yml @@ -0,0 +1,16 @@ +# Proof fixture: Modal receives only the Stage 12 secret name; the simulation +# migration URL reaches the migration step alone. +name: Good deployment +on: workflow_dispatch +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Synchronize the separate Stage 12 runtime secret + env: + STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }} + run: uv run modal secret create stage12-runtime "STAGE12_DATABASE_URL_SECRET_NAME=${STAGE12_DATABASE_URL_SECRET_NAME}" --force + - name: Apply simulation schema migrations + env: + SIMULATION_MIGRATION_DATABASE_URL: ${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }} + run: .github/scripts/simulation-database-migrate.sh diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/alembic.ini b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/alembic.ini new file mode 100644 index 000000000..6cb09e697 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/alembic.ini @@ -0,0 +1,2 @@ +[alembic] +script_location = %(here)s diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/env.py b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/env.py new file mode 100644 index 000000000..ab2608882 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/env.py @@ -0,0 +1,7 @@ +"""Proof fixture: an Alembic environment that versions the simulation schema.""" + +from alembic import context + +SIMULATION_SCHEMA = "simulation" + +context.configure(version_table_schema=SIMULATION_SCHEMA) diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/versions/0001_initial.py b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/versions/0001_initial.py new file mode 100644 index 000000000..5d6301f03 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/versions/0001_initial.py @@ -0,0 +1,4 @@ +"""Proof fixture: the chain's first revision.""" + +revision = "0001" +down_revision = None diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/alembic.ini b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/alembic.ini new file mode 100644 index 000000000..6f4ed3585 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/alembic.ini @@ -0,0 +1,2 @@ +[alembic] +script_location = %(here)s/migrations diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/migrations/versions/0001_service.py b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/migrations/versions/0001_service.py new file mode 100644 index 000000000..3a3f03b05 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/migrations/versions/0001_service.py @@ -0,0 +1,4 @@ +"""Proof fixture: a revision in a second, forbidden chain.""" + +revision = "0001" +down_revision = None diff --git a/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/alembic.ini b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/alembic.ini new file mode 100644 index 000000000..6cb09e697 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/alembic.ini @@ -0,0 +1,2 @@ +[alembic] +script_location = %(here)s diff --git a/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/env.py b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/env.py new file mode 100644 index 000000000..ab2608882 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/env.py @@ -0,0 +1,7 @@ +"""Proof fixture: an Alembic environment that versions the simulation schema.""" + +from alembic import context + +SIMULATION_SCHEMA = "simulation" + +context.configure(version_table_schema=SIMULATION_SCHEMA) diff --git a/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/versions/0001_initial.py b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/versions/0001_initial.py new file mode 100644 index 000000000..5d6301f03 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/versions/0001_initial.py @@ -0,0 +1,4 @@ +"""Proof fixture: the chain's first revision.""" + +revision = "0001" +down_revision = None diff --git a/rules/registry.toml b/rules/registry.toml index 3e249ca79..b64ca0287 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -19,6 +19,10 @@ title = "Service and package boundaries" id = "stage12" title = "Stage 12 persistence" +[[category]] +id = "persistence" +title = "Simulation persistence" + [[category]] id = "deployment" title = "Deployment pipeline" @@ -95,48 +99,149 @@ autofix = false [rule.enforcer] kind = "none" -# --- Stage 12 persistence -------------------------------------------------- +# --- Simulation persistence ----------------------------------------------- [[rule]] -id = "sim.stage12.no-schema-ddl" -category = "stage12" -statement = "Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema." -rationale = "libs/policyengine-stage12-persistence/README.md" -scope = ["libs/policyengine-stage12-persistence/src/**/*.py", "projects/policyengine-simulation-entry/src/**/*.py", "projects/policyengine-simulation-executor/src/**/*.py"] +id = "sim.persistence.one-migration-chain" +category = "persistence" +statement = "The simulation schema changes only through the one Alembic chain at migrations/ (configured by migrations/alembic.ini, versioned in simulation.alembic_version); no other Alembic configuration, environment or revision directory exists, and runtime source code contains no schema DDL." +rationale = "libs/policyengine-simulation-persistence/README.md#ownership-rules" +scope = ["migrations/env.py", "migrations/alembic.ini", "migrations/versions/*.py", "libs/policyengine-simulation-persistence/src/**/*.py", "projects/policyengine-simulation-entry/src/**/*.py"] tier = "mechanical" status = "enforced" autofix = false [rule.enforcer] kind = "pytest" -project = "projects/policyengine-simulation-entry" -nodes = ["tests/test_stage12_schema_authority.py::test_simulation_runtimes_contain_no_schema_ddl"] +project = "libs/policyengine-simulation-persistence" +nodes = [ + "tests/test_ownership_boundaries.py::test_repository_has_one_simulation_migration_chain", + "tests/test_ownership_boundaries.py::test_runtime_sources_change_the_schema_only_through_the_chain", +] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" invocation = "uv run pytest tests/" [rule.proof] -bad_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py" -good_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py" +bad_fixture = "rules/fixtures/sim.persistence.one-migration-chain/bad_repository" +good_fixture = "rules/fixtures/sim.persistence.one-migration-chain/good_repository" +project = "libs/policyengine-simulation-persistence" +test = "tests/test_ownership_boundaries.py::test_rule_proof_one_migration_chain" + +[[rule]] +id = "sim.persistence.migration-cycle" +category = "persistence" +statement = "On a fresh PostgreSQL, the simulation chain upgrades to its head, passes alembic check, downgrades to base and upgrades again, run by a non-superuser migrator, and the runtime role can read the revision but cannot change the schema." +rationale = "libs/policyengine-simulation-persistence/README.md#tests" +scope = ["migrations/versions/*.py", "libs/policyengine-simulation-persistence/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-simulation-persistence" +nodes = [ + "tests/test_migration_cycle.py::test_upgrade_check_downgrade_upgrade_cycle", + "tests/test_migration_cycle.py::test_the_version_table_lives_only_in_the_simulation_schema", + "tests/test_migration_cycle.py::test_the_runtime_role_reads_the_revision_but_cannot_change_the_schema", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "simulation-database" +invocation = "uv run pytest tests/ -v -m postgres" + +[[rule]] +id = "sim.persistence.executor-independent" +category = "persistence" +statement = "The Modal executor and gateway never declare, lock or import policyengine-simulation-persistence; only the Simulation Entrypoint and the migration job install it." +rationale = "libs/policyengine-simulation-persistence/README.md#ownership-rules" +scope = ["projects/policyengine-simulation-executor/pyproject.toml", "projects/policyengine-simulation-gateway/pyproject.toml", "projects/policyengine-simulation-executor/src/**/*.py", "projects/policyengine-simulation-gateway/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-simulation-persistence" +nodes = ["tests/test_ownership_boundaries.py::test_modal_projects_never_depend_on_the_persistence_library"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.persistence.executor-independent/bad_pyproject.toml" +good_fixture = "rules/fixtures/sim.persistence.executor-independent/good_pyproject.toml" +project = "libs/policyengine-simulation-persistence" +test = "tests/test_ownership_boundaries.py::test_rule_proof_executor_independent" + +[[rule]] +id = "sim.persistence.no-modal-database-credentials" +category = "persistence" +statement = "No Modal secret, Modal deployment or Modal application receives SIMULATION_DATABASE_URL or SIMULATION_MIGRATION_DATABASE_URL; the Stage 12 comparison's own database secret is the only database credential Modal holds." +rationale = "libs/policyengine-simulation-persistence/README.md#ownership-rules" +scope = [".github/workflows/*.yml", ".github/scripts/*.sh", "projects/policyengine-simulation-executor/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-simulation-persistence" +nodes = ["tests/test_ownership_boundaries.py::test_modal_never_receives_simulation_database_credentials"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.persistence.no-modal-database-credentials/bad_workflow.yml" +good_fixture = "rules/fixtures/sim.persistence.no-modal-database-credentials/good_workflow.yml" +project = "libs/policyengine-simulation-persistence" +test = "tests/test_ownership_boundaries.py::test_rule_proof_no_modal_database_credentials" + +[[rule]] +id = "sim.persistence.migrate-before-entrypoint" +category = "persistence" +statement = "Each deployment applies the simulation chain in its own job, the only one that receives SIMULATION_MIGRATION_DATABASE_URL, before the entrypoint candidate deploys; the entrypoint receives SIMULATION_DATABASE_URL as a pinned Secret Manager version; and the candidate smoke requires the database at the image's revision." +rationale = "libs/policyengine-simulation-persistence/README.md#deployment" +scope = [".github/workflows/simulation-deploy.reusable.yml", ".github/scripts/simulation-database-migrate.sh", ".github/scripts/deploy-cloud-run-simulation-entry.sh", ".github/scripts/cloud-run-simulation-entry-smoke.sh"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" project = "projects/policyengine-simulation-entry" -test = "tests/test_stage12_schema_authority.py::test_rule_proof_stage12_no_schema_ddl" +nodes = [ + "tests/test_simulation_database_deployment.py::test_the_migration_job_runs_before_the_entrypoint_deploys", + "tests/test_simulation_database_deployment.py::test_only_the_migration_step_receives_the_migrator_url", + "tests/test_simulation_database_deployment.py::test_the_entrypoint_receives_the_runtime_url_as_a_pinned_secret", + "tests/test_simulation_database_deployment.py::test_the_candidate_smoke_requires_the_current_database_revision", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +# --- Stage 12 persistence -------------------------------------------------- [[rule]] -id = "sim.stage12.no-migration-chain" +id = "sim.stage12.no-schema-ddl" category = "stage12" -statement = "This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api." +statement = "Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema." rationale = "libs/policyengine-stage12-persistence/README.md" -scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py"] +scope = ["libs/policyengine-stage12-persistence/src/**/*.py", "projects/policyengine-simulation-entry/src/**/*.py", "projects/policyengine-simulation-executor/src/**/*.py"] tier = "mechanical" status = "enforced" autofix = false [rule.enforcer] kind = "pytest" project = "projects/policyengine-simulation-entry" -nodes = ["tests/test_stage12_schema_authority.py::test_simulation_repository_has_no_schema_migration_chain"] +nodes = ["tests/test_stage12_schema_authority.py::test_simulation_runtimes_contain_no_schema_ddl"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" invocation = "uv run pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py" +good_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py" +project = "projects/policyengine-simulation-entry" +test = "tests/test_stage12_schema_authority.py::test_rule_proof_stage12_no_schema_ddl" [[rule]] id = "sim.stage12.no-production-user-associations"