diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index f156d4e..74b0ab4 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -1,10 +1,8 @@ name: Quality on: + workflow_call: pull_request: - push: - branches: - - main concurrency: group: quality-${{ github.workflow }}-${{ github.ref }} @@ -19,21 +17,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 - services: - postgres: - image: postgres:17 - env: - POSTGRES_DB: postgres - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U postgres -d postgres" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - name: Checkout repository uses: actions/checkout@v6 @@ -84,39 +67,15 @@ jobs: - name: Test Rust workspace run: just test rust - env: - GRASS_TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres - - - name: Test PostgreSQL deployment lifecycle regressions - run: | - cargo test -p grass-control-api domain::delivery::tests::postgres_ -- --ignored - cargo test -p grass-control-api domain::node_deletions::tests::postgres_ -- --ignored - env: - GRASS_TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres - name: Test Console workspace run: just test console - - name: Apply migrations to the disposable CI database - run: cargo run -p grass-control-api -- --config "$RUNNER_TEMP/grass-migration.toml" migrate - env: - GWAPI_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres - - - name: Verify certificate database columns and constraints - run: >- - cargo test -p grass-control-api - infra::database::migration::m20260910_000032_managed_certificates::tests::managed_certificate_schema_matches_signed_lifecycle_and_ack_protocol - -- --ignored --exact - env: - GRASS_TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres + - name: Verify embedded Console asset rebuilds + run: just assets-check - - name: Verify regional ingress migration upgrade and rollback - run: >- - cargo test -p grass-control-api - infra::database::migrate::tests::postgres_regional_ingress_schema_matches_domain_and_is_reversible - -- --ignored --exact - env: - GRASS_TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres + - name: Verify release metadata policy + run: just release-check - name: Check project license metadata run: just license-check @@ -163,31 +122,86 @@ jobs: GRASS_NODE_SMOKE_IMAGE: grass-build:smoke GRASS_NODE_SMOKE_SOCKET: unix:///var/run/docker.sock - release-images: - name: Release images - runs-on: ubuntu-latest - timeout-minutes: 45 + msrv: + name: Rust MSRV (${{ matrix.runner }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + runner: [ubuntu-latest, macos-latest] + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-msrv + uses: taiki-e/install-action@v2 + with: + tool: cargo-msrv@0.18.4 + + - name: Cache minimum-version build output + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-msrv-${{ hashFiles('Cargo.toml', 'Cargo.lock') }} + + - name: Verify the workspace minimum Rust version + run: cargo msrv verify --manifest-path apps/control-api/Cargo.toml --no-log -- cargo check --workspace --all-targets --locked + service-regressions: + name: PostgreSQL and Redis regressions + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + postgres: + image: postgres:17 + env: + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - name: Checkout repository uses: actions/checkout@v6 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable - - name: Build release runtime variants - uses: docker/bake-action@v6 + - name: Install Just + uses: extractions/setup-just@v4 + + - name: Cache service regression build output + uses: actions/cache@v5 with: - files: docker-bake.hcl - load: true + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-services-${{ hashFiles('Cargo.lock') }} + + - name: Run complete PostgreSQL and Redis regressions + run: just test-services env: - TAGS_DEBIAN: grass-worker:ci-debian - TAGS_SLIM: grass-worker:ci-slim - TAGS_ALPINE: grass-worker:ci-alpine - - - name: Check packaged binaries - run: | - for variant in debian slim alpine; do - docker run --rm --network none "grass-worker:ci-${variant}" grass-control-api --version - docker run --rm --network none "grass-worker:ci-${variant}" grass-node --version - done + GRASS_TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres + GRASS_TEST_REDIS_URL: redis://localhost:6379/0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6115696..14f2a91 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,8 +20,31 @@ concurrency: cancel-in-progress: false jobs: + validation: + name: Validate release commit + uses: ./.github/workflows/quality.yml + permissions: + contents: read + + metadata: + name: Validate release metadata + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + docker_tags: ${{ steps.policy.outputs.docker_tags }} + prerelease: ${{ steps.policy.outputs.prerelease }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Derive branch or version policy + id: policy + run: python3 scripts/release-metadata.py + docker: name: Docker image + needs: [validation, metadata] runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -43,35 +66,28 @@ jobs: uses: docker/metadata-action@v5 with: images: ${{ env.RUNTIME_IMAGE }} - tags: | - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha + flavor: latest=false + tags: ${{ needs.metadata.outputs.docker_tags }} - name: Derive Slim image metadata id: meta-slim uses: docker/metadata-action@v5 with: images: ${{ env.RUNTIME_IMAGE }} - flavor: suffix=-slim - tags: | - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha + flavor: | + latest=false + suffix=-slim + tags: ${{ needs.metadata.outputs.docker_tags }} - name: Derive Alpine image metadata id: meta-alpine uses: docker/metadata-action@v5 with: images: ${{ env.RUNTIME_IMAGE }} - flavor: suffix=-alpine - tags: | - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha + flavor: | + latest=false + suffix=-alpine + tags: ${{ needs.metadata.outputs.docker_tags }} - name: Prepare Bake labels id: bake-labels @@ -84,7 +100,34 @@ jobs: echo EOF } >> "$GITHUB_OUTPUT" - - name: Build and push runtime variants + - name: Build release runtime variants for verification + uses: docker/bake-action@v6 + with: + files: docker-bake.hcl + load: true + env: + IMAGE: ${{ env.RUNTIME_IMAGE }} + TAGS_DEBIAN: ${{ steps.meta.outputs.tags }} + TAGS_SLIM: ${{ steps.meta-slim.outputs.tags }} + TAGS_ALPINE: ${{ steps.meta-alpine.outputs.tags }} + LABELS_JSON: ${{ steps.bake-labels.outputs.labels_json }} + + - name: Check packaged binaries in every runtime variant + shell: bash + env: + TAGS_DEBIAN: ${{ steps.meta.outputs.tags }} + TAGS_SLIM: ${{ steps.meta-slim.outputs.tags }} + TAGS_ALPINE: ${{ steps.meta-alpine.outputs.tags }} + run: | + for tags in "$TAGS_DEBIAN" "$TAGS_SLIM" "$TAGS_ALPINE"; do + image_ref="${tags%%$'\n'*}" + test -n "$image_ref" + docker run --rm --pull=never --network none "$image_ref" grass-control-api --version + docker run --rm --pull=never --network none "$image_ref" grass-node --version + done + + # Reuse the verified build cache and retain BuildKit publication metadata. + - name: Publish verified runtime variants uses: docker/bake-action@v6 with: files: docker-bake.hcl @@ -98,6 +141,7 @@ jobs: binaries: name: Release binaries + needs: [validation, metadata] if: startsWith(github.ref, 'refs/tags/v') runs-on: ${{ matrix.runner }} timeout-minutes: 60 @@ -141,7 +185,7 @@ jobs: run: vp build - name: Build release binaries - run: cargo build --release --target ${{ matrix.target }} -p grass-control-api -p grass-node + run: cargo build --release --locked --target ${{ matrix.target }} -p grass-control-api -p grass-node - name: Package artifacts run: | @@ -155,3 +199,5 @@ jobs: with: files: dist/*.tar.gz generate_release_notes: true + prerelease: ${{ needs.metadata.outputs.prerelease == 'true' }} + make_latest: ${{ needs.metadata.outputs.prerelease == 'true' && 'false' || 'true' }} diff --git a/Cargo.lock b/Cargo.lock index 3452885..0346ab2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3898,9 +3898,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", @@ -3972,9 +3972,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", diff --git a/Cargo.toml b/Cargo.toml index cdd84cf..de573de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ resolver = "3" edition = "2024" license = "BSD-3-Clause" repository = "https://github.com/Grass-Development-Team/grass-worker" -rust-version = "1.85" +rust-version = "1.88" version = "0.1.0" [workspace.dependencies] diff --git a/Justfile b/Justfile index cafdd67..3797bdb 100644 --- a/Justfile +++ b/Justfile @@ -23,7 +23,7 @@ test target="all": check target="all": {{ if target == "rust" { "cargo check --workspace" } else if target == "console" { "cd " + console + " && vp check" } else if target == "all" { "cargo check --workspace && cd " + console + " && vp check" } else { error("unknown check target: " + target) } }} -quality: fmt clippy test check build license-check +quality: fmt clippy test check build assets-check release-check license-check license-check: test -f LICENSE @@ -48,3 +48,22 @@ preview target="console": migrate: cargo run -p grass-control-api -- migrate + +# Verify the locked workspace with its declared minimum supported Rust version. +msrv: + cargo msrv verify --manifest-path apps/control-api/Cargo.toml --no-log -- cargo check --workspace --all-targets --locked + +# Build distributable binaries with the production Console embedded. +release: + cd {{ console }} && vp build + cargo build --release --locked -p grass-control-api -p grass-node + +assets-check: + python3 scripts/check-embedded-assets.py + +# This suite creates and removes test schemas in the configured disposable services. +test-services: + python3 scripts/test-services.py + +release-check: + python3 -m unittest discover -s scripts -p "test_release_metadata.py" diff --git a/apps/console/bun.lock b/apps/console/bun.lock index 0e3f940..f7cde28 100644 --- a/apps/console/bun.lock +++ b/apps/console/bun.lock @@ -41,6 +41,9 @@ "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", + "@types/node": "^24", + "@types/react": "^19", + "@types/react-dom": "^19", "jsdom": "^29.1.1", "typescript": "^6.0.3", "vite": "npm:@voidzero-dev/vite-plus-core@0.2.4", @@ -353,6 +356,12 @@ "@types/estree": ["@types/estree@1.0.9", "", {}, "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg=="], + "@types/node": ["@types/node@24.13.4", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-YJ7EqCstVTzIr0fMr7qul/977en+pQHrfmuKIo6Zr9i75Be21dr3MovcfvGtyvi2HAUrRerWps5sMO9I7WaxDw=="], + + "@types/react": ["@types/react@19.3.0", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg=="], + + "@types/react-dom": ["@types/react-dom@19.3.0", "", { "peerDependencies": { "@types/react": "^19.3.0" } }, "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q=="], + "@vitejs/plugin-react": ["@vitejs/plugin-react@6.0.3", "", { "dependencies": { "@rolldown/pluginutils": "^1.0.1" }, "peerDependencies": { "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", "babel-plugin-react-compiler": "^1.0.0", "vite": "^8.0.0" }, "optionalPeers": ["@rolldown/plugin-babel", "babel-plugin-react-compiler"] }, "sha512-vmFvco5/QuC2f9Oj+wTk0+9XeDFkHxSamwZKYc7MxYwKICfvUvlMhqKI0VuICPltGqh1neqBKDvO4kes1ya8vg=="], "@vitest/browser": ["@vitest/browser@4.1.11", "", { "dependencies": { "@blazediff/core": "1.9.1", "@vitest/mocker": "4.1.11", "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pngjs": "^7.0.0", "sirv": "^3.0.2", "tinyrainbow": "^3.1.0", "ws": "^8.19.0" }, "peerDependencies": { "vitest": "4.1.11" } }, "sha512-bwMovvAeuTFOK5kIFevw4VEf+1gVEICv4SYK4k3knJOxl6b1zEWud8mYKD73e1B0odAn174h1MofURy2TPWf3w=="], @@ -419,6 +428,8 @@ "css.escape": ["css.escape@1.5.1", "", {}, "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg=="], + "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], + "data-urls": ["data-urls@7.0.0", "", { "dependencies": { "whatwg-mimetype": "^5.0.0", "whatwg-url": "^16.0.0" } }, "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA=="], "decimal.js": ["decimal.js@10.6.0", "", {}, "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg=="], @@ -599,6 +610,8 @@ "undici": ["undici@7.29.0", "", {}, "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw=="], + "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], + "use-callback-ref": ["use-callback-ref@1.3.3", "", { "dependencies": { "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg=="], "use-sidecar": ["use-sidecar@1.1.3", "", { "dependencies": { "detect-node-es": "^1.1.0", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ=="], diff --git a/apps/console/package.json b/apps/console/package.json index 7e357ec..905a6b8 100644 --- a/apps/console/package.json +++ b/apps/console/package.json @@ -48,6 +48,9 @@ "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", + "@types/node": "^24", + "@types/react": "^19", + "@types/react-dom": "^19", "jsdom": "^29.1.1", "typescript": "^6.0.3", "vite": "npm:@voidzero-dev/vite-plus-core@0.2.4", diff --git a/apps/console/src/components/ui/alert-dialog.tsx b/apps/console/src/components/ui/alert-dialog.tsx index a9fa2e7..0c33597 100644 --- a/apps/console/src/components/ui/alert-dialog.tsx +++ b/apps/console/src/components/ui/alert-dialog.tsx @@ -2,7 +2,7 @@ import * as React from "react"; import * as AlertDialogPrimitive from "@radix-ui/react-alert-dialog"; import { cn } from "@/lib/utils"; -import { buttonVariants } from "@/components/ui/button"; +import { Button, buttonVariants } from "@/components/ui/button"; function AlertDialog({ ...props }: React.ComponentProps) { return ; @@ -101,9 +101,17 @@ function AlertDialogDescription({ function AlertDialogAction({ className, + variant, + size, ...props -}: React.ComponentProps) { - return ; +}: React.ComponentProps & + Pick, "variant" | "size">) { + return ( + + ); } function AlertDialogCancel({ diff --git a/apps/console/src/components/ui/button.tsx b/apps/console/src/components/ui/button.tsx index fb6bc19..e97a985 100644 --- a/apps/console/src/components/ui/button.tsx +++ b/apps/console/src/components/ui/button.tsx @@ -23,6 +23,7 @@ const buttonVariants = cva( sm: "h-8 rounded-md gap-1.5 px-3 has-[>svg]:px-2.5", lg: "h-10 rounded-md px-6 has-[>svg]:px-4", icon: "size-9", + "icon-sm": "size-8", }, }, defaultVariants: { diff --git a/apps/console/src/features/account/profile-route.test.tsx b/apps/console/src/features/account/profile-route.test.tsx index a5c78de..e5e794b 100644 --- a/apps/console/src/features/account/profile-route.test.tsx +++ b/apps/console/src/features/account/profile-route.test.tsx @@ -13,14 +13,24 @@ beforeEach(() => { vi.clearAllMocks(); updateProfile.mockResolvedValue(undefined); vi.mocked(useAuth).mockReturnValue({ + isLoading: false, + login: vi.fn(), + register: vi.fn(), + completeMfa: vi.fn(), + verifyEmail: vi.fn(), + uploadAvatar: vi.fn(), + removeAvatar: vi.fn(), + logout: vi.fn(), user: { + avatar_url: null, + email_verified: true, id: "user-1", email: "user@example.com", display_name: "Old Name", platform_role: "user", }, updateProfile, - } as ReturnType); + }); }); it("updates the display name while keeping the email read-only", async () => { diff --git a/apps/console/src/features/admin/cleanup.api.ts b/apps/console/src/features/admin/cleanup.api.ts index d6c0fb7..988db48 100644 --- a/apps/console/src/features/admin/cleanup.api.ts +++ b/apps/console/src/features/admin/cleanup.api.ts @@ -42,7 +42,7 @@ function query(filters: Record) { export const cleanupApi = { previewAudit: (filters: AuditCleanupFilters) => - request(`/api/v1/admin/cleanup/audit-events${query(filters)}`), + request(`/api/v1/admin/cleanup/audit-events${query({ ...filters })}`), deleteAudit: (filters: AuditCleanupFilters) => request("/api/v1/admin/cleanup/audit-events", { @@ -51,7 +51,7 @@ export const cleanupApi = { }), previewBuildLogs: (filters: BuildLogCleanupFilters) => - request(`/api/v1/admin/cleanup/build-logs${query(filters)}`), + request(`/api/v1/admin/cleanup/build-logs${query({ ...filters })}`), deleteBuildLogs: (filters: BuildLogCleanupFilters) => request("/api/v1/admin/cleanup/build-logs", { diff --git a/apps/console/src/features/admin/components/announcements-panel.test.tsx b/apps/console/src/features/admin/components/announcements-panel.test.tsx index be23108..d9e1abc 100644 --- a/apps/console/src/features/admin/components/announcements-panel.test.tsx +++ b/apps/console/src/features/admin/components/announcements-panel.test.tsx @@ -56,7 +56,7 @@ it("publishes a new announcement to active users", async () => { await user.click(screen.getByRole("button", { name: "Publish" })); await waitFor(() => expect(adminApi.publishAnnouncement).toHaveBeenCalled()); - expect(adminApi.publishAnnouncement.mock.calls[0]?.[0]).toEqual({ + expect(vi.mocked(adminApi.publishAnnouncement).mock.calls[0]?.[0]).toEqual({ title: "Planned maintenance", content: "The service will restart.", auto_popup: false, @@ -89,6 +89,6 @@ it("deletes an announcement from the history", async () => { await user.click(screen.getByRole("button", { name: "Delete announcement" })); await waitFor(() => - expect(adminApi.removeAnnouncement.mock.calls[0]?.[0]).toBe("announcement-1"), + expect(vi.mocked(adminApi.removeAnnouncement).mock.calls[0]?.[0]).toBe("announcement-1"), ); }); diff --git a/apps/console/src/features/admin/components/authentication-panel.test.tsx b/apps/console/src/features/admin/components/authentication-panel.test.tsx index b2142bf..465a69a 100644 --- a/apps/console/src/features/admin/components/authentication-panel.test.tsx +++ b/apps/console/src/features/admin/components/authentication-panel.test.tsx @@ -19,8 +19,31 @@ vi.mock("../admin.api", async (importOriginal) => { }; }); -const settings = { - mail: { mode: "smtp" }, +const settings: AdminSettings = { + site: { + name: "Old Name", + logo_url: "/assets/old-logo.svg", + url: "https://console.example.com", + public_base_url: "https://apps.example.com", + }, + signup: { policy: "open" }, + review: { production: "manual", preview: "auto" }, + domain_review: { default: "auto" }, + server: { host: "127.0.0.1", port: 7817 }, + database: { url_configured: true }, + redis: { backend: "redis", url_configured: true }, + secrets: { secret_key_configured: true, git_credentials_configured: false }, + mail: { + mode: "smtp", + from_address: "noreply@example.com", + from_name: "Grass Worker", + sendmail_command: "/usr/sbin/sendmail", + smtp_host: "smtp.example.com", + smtp_port: 587, + smtp_security: "starttls", + smtp_username: "mailer", + smtp_password_configured: true, + }, authentication: { password_policy: { min_length: 8, @@ -39,7 +62,18 @@ const settings = { required_factors: [], }, }, -} as AdminSettings; + session: { cookie_secure: true, idle_ttl_seconds: 900, session_ttl_seconds: 2_592_000 }, + audit: { retention_days: 90 }, + node_manager: { + auto_start_local_node: false, + local_node_binary: "grass-node", + local_node_config: "./node.toml", + restart_on_exit: true, + }, + migration: { auto_migrate: false }, + log: { level: "info", format: "pretty" }, + restart_required_sections: ["server", "redis", "node_manager", "migration", "log"], +}; beforeEach(() => { vi.clearAllMocks(); diff --git a/apps/console/src/features/admin/components/cleanup-panel.test.tsx b/apps/console/src/features/admin/components/cleanup-panel.test.tsx index 38ea3a8..a192ac0 100644 --- a/apps/console/src/features/admin/components/cleanup-panel.test.tsx +++ b/apps/console/src/features/admin/components/cleanup-panel.test.tsx @@ -91,7 +91,7 @@ it("previews and deletes the filtered audit events", async () => { await user.click(screen.getByRole("button", { name: "Delete Audit Events" })); expect(screen.getByRole("heading", { name: "Delete audit events?" })).toBeInTheDocument(); - await user.click(screen.getByRole("button", { name: "Delete", exact: true })); + await user.click(screen.getByRole("button", { name: "Delete" })); await waitFor(() => expect(cleanupApi.deleteAudit).toHaveBeenCalledWith( @@ -113,7 +113,7 @@ it("keeps build-log cleanup as a separate protected action", async () => { expect(screen.getByText("1 protected")).toBeInTheDocument(); await user.click(screen.getByRole("button", { name: "Delete Build Logs" })); - await user.click(screen.getByRole("button", { name: "Delete", exact: true })); + await user.click(screen.getByRole("button", { name: "Delete" })); await waitFor(() => expect(cleanupApi.deleteBuildLogs).toHaveBeenCalledWith({})); expect(await screen.findByText(/Deleted 3 records/)).toBeInTheDocument(); diff --git a/apps/console/src/features/admin/components/project-governance-page.test.tsx b/apps/console/src/features/admin/components/project-governance-page.test.tsx index ef86678..416f603 100644 --- a/apps/console/src/features/admin/components/project-governance-page.test.tsx +++ b/apps/console/src/features/admin/components/project-governance-page.test.tsx @@ -27,7 +27,7 @@ vi.mock("../admin.api", async (importOriginal) => { }; }); -const project = { +const project: Awaited>["project"] = { id: "project-1", uuid: "project-1", slug: "demo-site", @@ -41,6 +41,8 @@ const project = { source_config: {}, build_config: {}, archived_at: null, + status: "active", + deleted_at: null, created_at: "2026-07-30T00:00:00Z", updated_at: "2026-07-30T00:00:00Z", }; diff --git a/apps/console/src/features/admin/components/settings-panel.test.tsx b/apps/console/src/features/admin/components/settings-panel.test.tsx index c3990a1..50b06ea 100644 --- a/apps/console/src/features/admin/components/settings-panel.test.tsx +++ b/apps/console/src/features/admin/components/settings-panel.test.tsx @@ -163,7 +163,9 @@ it("shows every non-secret Control API setting and only secret configuration sta expect(screen.getByLabelText("Log filter")).toHaveValue("info"); expect(screen.getByLabelText("Log format")).toHaveTextContent("Pretty"); - const sensitive = screen.getByText("Sensitive configuration").closest("[data-slot='card']"); + const sensitive = screen + .getByText("Sensitive configuration") + .closest("[data-slot='card']"); expect(sensitive).not.toBeNull(); expect(within(sensitive!).getByText("Database URL")).toBeInTheDocument(); expect(within(sensitive!).getByText("Redis URL")).toBeInTheDocument(); diff --git a/apps/console/src/features/admin/components/team-groups-panel.tsx b/apps/console/src/features/admin/components/team-groups-panel.tsx index 7fe3a0b..a307ec1 100644 --- a/apps/console/src/features/admin/components/team-groups-panel.tsx +++ b/apps/console/src/features/admin/components/team-groups-panel.tsx @@ -36,6 +36,12 @@ import { adminApi, type AdminTeamGroup } from "../admin.api"; const INHERIT_NONE = "__none__"; const INHERIT_REVIEW = "inherit"; +function reviewPolicy(value: string): "auto" | "manual" | null { + if (value === INHERIT_REVIEW) return null; + if (value === "auto" || value === "manual") return value; + throw new Error("Unknown review policy"); +} + const reviewModeLabel = (mode: "auto" | "manual" | null) => mode ? `${mode.charAt(0).toUpperCase()}${mode.slice(1)}` : "Inherit"; @@ -207,9 +213,9 @@ function GroupFormDialog({ description, quota_plan_id: planId === INHERIT_NONE ? null : planId, review_policy: { - production: reviewProduction === INHERIT_REVIEW ? null : reviewProduction, - preview: reviewPreview === INHERIT_REVIEW ? null : reviewPreview, - domain: reviewDomain === INHERIT_REVIEW ? null : reviewDomain, + production: reviewPolicy(reviewProduction), + preview: reviewPolicy(reviewPreview), + domain: reviewPolicy(reviewDomain), }, }) : adminApi.createTeamGroup({ @@ -218,9 +224,9 @@ function GroupFormDialog({ description: description || undefined, quota_plan_id: planId === INHERIT_NONE ? undefined : planId, review_policy: { - production: reviewProduction === INHERIT_REVIEW ? null : reviewProduction, - preview: reviewPreview === INHERIT_REVIEW ? null : reviewPreview, - domain: reviewDomain === INHERIT_REVIEW ? null : reviewDomain, + production: reviewPolicy(reviewProduction), + preview: reviewPolicy(reviewPreview), + domain: reviewPolicy(reviewDomain), }, }), onSuccess: onSaved, diff --git a/apps/console/src/features/auth/auth-context.test.tsx b/apps/console/src/features/auth/auth-context.test.tsx index 0a5cde0..6f93394 100644 --- a/apps/console/src/features/auth/auth-context.test.tsx +++ b/apps/console/src/features/auth/auth-context.test.tsx @@ -31,6 +31,8 @@ it("updates the current user after saving the profile", async () => { email: "leo@example.com", display_name: "Leo", platform_role: "user", + avatar_url: null, + email_verified: true, }, }); vi.mocked(authApi.updateMe).mockResolvedValue({ @@ -39,6 +41,8 @@ it("updates the current user after saving the profile", async () => { email: "leo@example.com", display_name: "Leonard", platform_role: "user", + avatar_url: null, + email_verified: true, }, }); const wrapper = ({ children }: { children: ReactNode }) => ( @@ -60,6 +64,8 @@ it("clears local authentication when an API request reports an expired session", email: "leo@example.com", display_name: "Leo", platform_role: "user", + avatar_url: null, + email_verified: true, }, }); setCsrfToken("csrf-token"); @@ -87,6 +93,7 @@ it("keeps a newer login when the initial session restore fails late", async () = email: "new-session@example.com", display_name: "New session", platform_role: "user", + avatar_url: null, email_verified: true, }, csrf_token: "new-csrf-token", diff --git a/apps/console/src/features/dashboard/dashboard-route.tsx b/apps/console/src/features/dashboard/dashboard-route.tsx index d3e87f9..e25fd33 100644 --- a/apps/console/src/features/dashboard/dashboard-route.tsx +++ b/apps/console/src/features/dashboard/dashboard-route.tsx @@ -211,7 +211,11 @@ export function DashboardRoute() { )} !open && setSelectedAnnouncement(null)} /> diff --git a/apps/console/src/features/deployments/deployment-detail-route.test.tsx b/apps/console/src/features/deployments/deployment-detail-route.test.tsx index 064ee23..c4a588f 100644 --- a/apps/console/src/features/deployments/deployment-detail-route.test.tsx +++ b/apps/console/src/features/deployments/deployment-detail-route.test.tsx @@ -36,6 +36,11 @@ function detailFixture(overrides: Partial = {}): build_status: "ready", serve_status: "syncing", release_status: "draft", + release_pending: false, + pending_release_reason: null, + pending_release_requested_at: null, + screenshot_status: "pending", + screenshot_url: null, serve_resources: { cpu_millicores: 200, memory_mb: 256, disk_mb: 512 }, overcommitted: false, build_stage: null, diff --git a/apps/console/src/features/deployments/deployments-tab.test.tsx b/apps/console/src/features/deployments/deployments-tab.test.tsx index 543b6cc..0bbfb33 100644 --- a/apps/console/src/features/deployments/deployments-tab.test.tsx +++ b/apps/console/src/features/deployments/deployments-tab.test.tsx @@ -63,6 +63,11 @@ function deploymentFixture(overrides: Partial = {}): Deployment { build_status: "ready", serve_status: "failed", release_status: "draft", + release_pending: false, + pending_release_reason: null, + pending_release_requested_at: null, + screenshot_status: "pending", + screenshot_url: null, serve_resources: { cpu_millicores: 50, memory_mb: 64, disk_mb: 256 }, overcommitted: false, build_stage: null, @@ -91,15 +96,41 @@ function deploymentFixture(overrides: Partial = {}): Deployment { } it("keeps polling while build or serve work is in progress", () => { - expect(deploymentRefetchInterval({ build_status: "building", serve_status: "pending" })).toBe( - 4000, - ); - expect(deploymentRefetchInterval({ build_status: "ready", serve_status: "pending" })).toBe(4000); - expect(deploymentRefetchInterval({ build_status: "ready", serve_status: "syncing" })).toBe(4000); - expect(deploymentRefetchInterval({ build_status: "ready", serve_status: "ready" })).toBe(false); - expect(deploymentRefetchInterval({ build_status: "failed", serve_status: "pending" })).toBe( - false, - ); + expect( + deploymentRefetchInterval({ + release_pending: false, + build_status: "building", + serve_status: "pending", + }), + ).toBe(4000); + expect( + deploymentRefetchInterval({ + release_pending: false, + build_status: "ready", + serve_status: "pending", + }), + ).toBe(4000); + expect( + deploymentRefetchInterval({ + release_pending: false, + build_status: "ready", + serve_status: "syncing", + }), + ).toBe(4000); + expect( + deploymentRefetchInterval({ + release_pending: false, + build_status: "ready", + serve_status: "ready", + }), + ).toBe(false); + expect( + deploymentRefetchInterval({ + release_pending: false, + build_status: "failed", + serve_status: "pending", + }), + ).toBe(false); }); it("shows serve placement and serve failures in the serve column", async () => { diff --git a/apps/console/src/features/notifications/notification-bell.test.tsx b/apps/console/src/features/notifications/notification-bell.test.tsx index 2e876a1..5b40370 100644 --- a/apps/console/src/features/notifications/notification-bell.test.tsx +++ b/apps/console/src/features/notifications/notification-bell.test.tsx @@ -74,7 +74,7 @@ it("opens announcement content in a dialog and marks it as read", async () => { expect(await screen.findByRole("dialog")).toHaveTextContent( "The service will restart at 10:00 UTC.", ); - expect(notificationsApi.markRead.mock.calls[0]?.[0]).toBe("announcement-1"); + expect(vi.mocked(notificationsApi.markRead).mock.calls[0]?.[0]).toBe("announcement-1"); }); it("opens a compact inbox from the notification bell", async () => { @@ -132,7 +132,7 @@ it("marks an automatically opened announcement as read when it closes", async () expect(await screen.findByRole("dialog")).toHaveTextContent("Please read this update."); await user.click(screen.getByRole("button", { name: "Close" })); - expect(notificationsApi.markRead.mock.calls[0]?.[0]).toBe("announcement-auto"); + expect(vi.mocked(notificationsApi.markRead).mock.calls[0]?.[0]).toBe("announcement-auto"); }); it("marks every message as read from the inbox footer", async () => { diff --git a/apps/console/src/features/projects/project-create-route.test.tsx b/apps/console/src/features/projects/project-create-route.test.tsx index b3b05ff..fb5acb3 100644 --- a/apps/console/src/features/projects/project-create-route.test.tsx +++ b/apps/console/src/features/projects/project-create-route.test.tsx @@ -24,10 +24,23 @@ function LocationProbe() { function renderCreate(role: "member" | "viewer" = "member", isLoading = false) { vi.mocked(useTeam).mockReturnValue({ - activeTeam: { id: "team-1", name: "Acme", slug: "acme", role }, + teams: [], + error: null, + selectTeam: vi.fn(), + createTeam: vi.fn(), + refreshTeams: vi.fn(), + activeTeam: { + id: "team-1", + name: "Acme", + slug: "acme", + kind: "team", + avatar_url: null, + owner_user_id: null, + group_id: null, + }, activeRole: role, isLoading, - } as ReturnType); + }); const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); return render( @@ -172,10 +185,15 @@ describe("ProjectCreateRoute", () => { it("waits for team permissions before deciding access", () => { vi.mocked(useTeam).mockReturnValue({ + teams: [], + error: null, + selectTeam: vi.fn(), + createTeam: vi.fn(), + refreshTeams: vi.fn(), activeTeam: null, activeRole: null, isLoading: true, - } as ReturnType); + }); const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); render( @@ -191,12 +209,15 @@ describe("ProjectCreateRoute", () => { it("shows a Toast and retry action when teams fail to load", () => { vi.mocked(useTeam).mockReturnValue({ + teams: [], + selectTeam: vi.fn(), + createTeam: vi.fn(), activeTeam: null, activeRole: null, isLoading: false, error: new Error("Network unavailable"), refreshTeams: vi.fn(), - } as ReturnType); + }); const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); render( diff --git a/apps/console/src/features/projects/project-settings-build-route.tsx b/apps/console/src/features/projects/project-settings-build-route.tsx index eb8fb2a..488a3bd 100644 --- a/apps/console/src/features/projects/project-settings-build-route.tsx +++ b/apps/console/src/features/projects/project-settings-build-route.tsx @@ -68,7 +68,7 @@ export function ProjectSettingsBuildRoute() { }); const credentialMutation = useMutation({ - mutationFn: () => + mutationFn: async () => selectedCredentialId === "none" ? projectsApi.unbindSourceCredential(project.id) : projectsApi.bindSourceCredential(project.id, selectedCredentialId), diff --git a/apps/console/src/features/projects/projects-route.test.tsx b/apps/console/src/features/projects/projects-route.test.tsx index 7d63d39..9c9c6dd 100644 --- a/apps/console/src/features/projects/projects-route.test.tsx +++ b/apps/console/src/features/projects/projects-route.test.tsx @@ -16,9 +16,23 @@ vi.mock("./projects.api", async (importOriginal) => { function renderProjects(role: "member" | "viewer") { vi.mocked(useTeam).mockReturnValue({ - activeTeam: { id: "team-1", name: "Acme", slug: "acme", role }, + teams: [], + isLoading: false, + error: null, + selectTeam: vi.fn(), + createTeam: vi.fn(), + refreshTeams: vi.fn(), + activeTeam: { + id: "team-1", + name: "Acme", + slug: "acme", + kind: "team", + avatar_url: null, + owner_user_id: null, + group_id: null, + }, activeRole: role, - } as ReturnType); + }); const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); return render( diff --git a/apps/console/src/features/teams/team-switcher.test.tsx b/apps/console/src/features/teams/team-switcher.test.tsx index a68cd71..0ea7f96 100644 --- a/apps/console/src/features/teams/team-switcher.test.tsx +++ b/apps/console/src/features/teams/team-switcher.test.tsx @@ -21,10 +21,11 @@ const personal: Team = { }; const shared: Team = { ...personal, id: "shared", slug: "acme", name: "Acme", kind: "team" }; -function mockTeamContext(overrides: Record = {}) { +function mockTeamContext(overrides: Partial> = {}) { vi.mocked(useTeam).mockReturnValue({ teams: [personal, shared], activeTeam: personal, + activeRole: "owner", isLoading: false, error: null, selectTeam: vi.fn(), diff --git a/apps/console/src/layouts/app-layout.test.tsx b/apps/console/src/layouts/app-layout.test.tsx index 590edbc..1664c2f 100644 --- a/apps/console/src/layouts/app-layout.test.tsx +++ b/apps/console/src/layouts/app-layout.test.tsx @@ -30,21 +30,44 @@ function renderLayout( isLoading = false, ) { vi.mocked(useAuth).mockReturnValue({ + isLoading: false, + login: vi.fn(), + register: vi.fn(), + completeMfa: vi.fn(), + verifyEmail: vi.fn(), + updateProfile: vi.fn(), + uploadAvatar: vi.fn(), + removeAvatar: vi.fn(), user: { + avatar_url: null, + email_verified: true, id: "user-1", email: "user@example.com", display_name: "User", platform_role: platformRole, }, logout: vi.fn(), - } as ReturnType); + }); vi.mocked(useTeam).mockReturnValue({ - activeTeam: isLoading ? null : { id: "team-1", slug: "team", name: "Team", kind: "team" }, + teams: [], + selectTeam: vi.fn(), + createTeam: vi.fn(), + activeTeam: isLoading + ? null + : { + id: "team-1", + slug: "team", + name: "Team", + kind: "team", + avatar_url: null, + owner_user_id: null, + group_id: null, + }, activeRole: teamRole, error: null, isLoading, refreshTeams: vi.fn(), - } as ReturnType); + }); const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); render( diff --git a/apps/console/src/router.test.tsx b/apps/console/src/router.test.tsx index 61d338d..06c36c9 100644 --- a/apps/console/src/router.test.tsx +++ b/apps/console/src/router.test.tsx @@ -53,7 +53,13 @@ vi.mock("@/features/projects/project-create-route", () => ({ function setUser(platformRole: "admin" | "user") { vi.mocked(useAuth).mockReturnValue({ + completeMfa: vi.fn(), + verifyEmail: vi.fn(), + uploadAvatar: vi.fn(), + removeAvatar: vi.fn(), user: { + avatar_url: null, + email_verified: true, id: "user-1", email: "user@example.com", display_name: "User", @@ -64,18 +70,22 @@ function setUser(platformRole: "admin" | "user") { register: vi.fn(), updateProfile: vi.fn(), logout: vi.fn(), - } as ReturnType); + }); } function setGuest() { vi.mocked(useAuth).mockReturnValue({ + completeMfa: vi.fn(), + verifyEmail: vi.fn(), + uploadAvatar: vi.fn(), + removeAvatar: vi.fn(), user: null, isLoading: false, login: vi.fn(), register: vi.fn(), updateProfile: vi.fn(), logout: vi.fn(), - } as ReturnType); + }); } describe("Administration routing", () => { diff --git a/apps/console/tsconfig.json b/apps/console/tsconfig.json index 2abf2e4..6fe6837 100644 --- a/apps/console/tsconfig.json +++ b/apps/console/tsconfig.json @@ -15,7 +15,10 @@ "isolatedModules": true, "noEmit": true, "jsx": "react-jsx", - "paths": { "@/*": ["./src/*"] } + "paths": { + "@/*": ["./src/*"] + }, + "types": ["vite/client", "vite-plus/test/globals", "node"] }, "include": ["src", "vite.config.ts"] } diff --git a/apps/console/vite.config.ts b/apps/console/vite.config.ts index 028b56b..162fde5 100644 --- a/apps/console/vite.config.ts +++ b/apps/console/vite.config.ts @@ -8,6 +8,7 @@ export default defineConfig(({ mode }) => { const apiTarget = env.VITE_API_TARGET ?? "http://127.0.0.1:7817"; return { + lint: { options: { typeAware: true, typeCheck: true } }, test: { environment: "jsdom", globals: true, diff --git a/apps/control-api/src/domain/acme.rs b/apps/control-api/src/domain/acme.rs index 12db023..b260e03 100644 --- a/apps/control-api/src/domain/acme.rs +++ b/apps/control-api/src/domain/acme.rs @@ -1,5 +1,5 @@ //! Automatic issuance with persistent retry state and challenge publication barriers. -use super::certificates::{self, ACCOUNT_KEY, BUNDLE_KEY, PemBundle}; + use anyhow::{Context, ensure}; use base64::{ Engine, @@ -13,16 +13,13 @@ use sea_orm::{ ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, QuerySelect, Set, TransactionTrait, }; use serde_json::Value; - -use crate::domain::certificates::CertificateStatus; - -#[cfg(test)] -use serde_json::json; use time::{Duration, OffsetDateTime}; use uuid::Uuid; -use crate::infra::database::entity::managed_certificate as cert; -use crate::infra::database::entity::{node_ingress_status, project_host_binding, regional_ingress}; +use super::certificates::{self, ACCOUNT_KEY, BUNDLE_KEY, CertificateStatus, PemBundle}; +use crate::infra::database::entity::{ + managed_certificate as cert, node_ingress_status, project_host_binding, regional_ingress, +}; const LEASE_SECONDS: i64 = 600; const ATTEMPT_SECONDS: u64 = 480; @@ -362,7 +359,12 @@ async fn reconcile_record( Err(_error) => { // ACME/provider errors may contain challenge/account material; expose bounded diagnostics. active.status = Set(CertificateStatus::Failed.as_str().to_owned()); - active.error=Set(Some("Certificate issuance failed; check public HTTP access on port 80, entry acknowledgements and certificate authority settings. Automatic retry is scheduled.".to_owned())); + active.error = Set(Some( + "Certificate issuance failed; check public HTTP access on port 80, \ + entry acknowledgements and certificate authority settings. \ + Automatic retry is scheduled." + .to_owned(), + )); active.retry_at = Set(Some( OffsetDateTime::now_utc() + retry_delay(item.failure_count), )); @@ -381,7 +383,11 @@ pub async fn sweep(db: &DatabaseConnection, secret: &str) -> anyhow::Result<()> let secret = secret.to_owned(); tasks.spawn(async move { if sweep_ingress(&db, ingress.id, &secret).await.is_err() { - tracing::warn!(operation="control_api.acme.ingress_sweep_failed",ingress_id=%ingress.id,"regional certificate sweep failed; other regions will continue"); + tracing::warn!( + operation = "control_api.acme.ingress_sweep_failed", + ingress_id = %ingress.id, + "regional certificate sweep failed; other regions will continue" + ); } }); if tasks.len() >= 4 { @@ -453,7 +459,10 @@ async fn sweep_ingress( #[cfg(test)] mod tests { + use serde_json::json; + use super::*; + #[tokio::test] async fn http_validation_waits_for_every_eligible_entry_revision() { use crate::infra::database::entity::regional_ingress_health; @@ -509,6 +518,7 @@ mod tests { ); } } + #[derive(Clone)] struct MockCa { origin: String, @@ -522,8 +532,10 @@ mod tests { uri: axum::http::Uri, body: axum::body::Bytes, ) -> axum::response::Response { - use axum::response::IntoResponse; use std::sync::atomic::Ordering; + + use axum::response::IntoResponse; + let payload = if body.is_empty() { json!({}) } else { @@ -554,7 +566,7 @@ mod tests { "/nonce" => axum::http::StatusCode::OK.into_response(), "/account" => { state.accounts.fetch_add(1, Ordering::SeqCst); - axum::Json(json!({"status": "valid"})).into_response() + axum::Json(json!({ "status": "valid" })).into_response() } "/new-order" => { assert_eq!(payload["identifiers"][0]["value"], "site.example.org"); @@ -562,7 +574,7 @@ mod tests { } "/authorization" => axum::Json(json!({ "status": "valid", - "identifier": {"type": "dns", "value": "site.example.org"}, + "identifier": { "type": "dns", "value": "site.example.org" }, "challenges": [], })) .into_response(), @@ -677,6 +689,7 @@ mod tests { assert!(state.issued.load(std::sync::atomic::Ordering::SeqCst)); server.abort(); } + #[test] fn renewal_respects_expiry_manual_mode_backoff_and_active_lease() { let now = OffsetDateTime::now_utc(); @@ -702,6 +715,7 @@ mod tests { item.issuer = "manual".to_owned(); assert!(!due(&item, now)); } + #[test] fn interrupted_or_failed_forced_renewal_retries_with_auto_renew_disabled() { let now = OffsetDateTime::now_utc(); @@ -718,15 +732,19 @@ mod tests { assert!(!due(&item, now)); assert!(due(&item, now + Duration::seconds(2))); } + #[test] fn retry_delay_is_bounded_and_zero_ssl_requires_complete_eab() { assert_eq!(retry_delay(0), Duration::minutes(5)); assert_eq!(retry_delay(20), Duration::seconds(76_800)); - assert!(external_account_key(&json!({"eab_kid":"id"})).is_err()); + assert!(external_account_key(&json!({ "eab_kid": "id" })).is_err()); assert!( - external_account_key(&json!({"eab_kid":"id","eab_hmac_key":"c2VjcmV0"})) - .unwrap() - .is_some() + external_account_key(&json!({ + "eab_kid": "id", + "eab_hmac_key": "c2VjcmV0", + })) + .unwrap() + .is_some() ); } } diff --git a/apps/control-api/src/domain/certificate_settings.rs b/apps/control-api/src/domain/certificate_settings.rs index 296490f..3bdaf5f 100644 --- a/apps/control-api/src/domain/certificate_settings.rs +++ b/apps/control-api/src/domain/certificate_settings.rs @@ -1,9 +1,10 @@ -use super::{certificates, settings}; use anyhow::{Context, ensure}; use sea_orm::ConnectionTrait; use serde_json::{Value, json}; use uuid::Uuid; +use super::{certificates, settings}; + const SETTINGS_KEY: &str = "domain_https"; const SECRET_KEY: &str = "domain-https-eab-v1"; @@ -11,6 +12,7 @@ pub struct CertificateSettings { pub issuer: String, pub eab: Value, } + impl CertificateSettings { pub fn validate(&self) -> anyhow::Result<()> { ensure!( @@ -26,12 +28,14 @@ impl CertificateSettings { Ok(()) } } + async fn stored(db: &C) -> anyhow::Result { Ok(settings::get_setting(db, SETTINGS_KEY) .await? .map(|s| s.value) - .unwrap_or_else(|| json!({"issuer":"letsencrypt"}))) + .unwrap_or_else(|| json!({ "issuer": "letsencrypt" }))) } + pub async fn issuer(db: &C) -> anyhow::Result { let value = stored(db).await?; let issuer = value["issuer"] @@ -43,6 +47,7 @@ pub async fn issuer(db: &C) -> anyhow::Result { ); Ok(issuer.to_owned()) } + pub async fn load(db: &C, secret: &str) -> anyhow::Result { let value = stored(db).await?; let eab = match value.get("eab") { @@ -62,6 +67,7 @@ pub async fn load(db: &C, secret: &str) -> anyhow::Result( db: &C, settings: &CertificateSettings, @@ -72,7 +78,10 @@ pub async fn save( super::settings::set_json( db, SETTINGS_KEY, - json!({"issuer":settings.issuer,"eab":eab}), + json!({ + "issuer": settings.issuer, + "eab": eab, + }), ) .await } @@ -80,6 +89,7 @@ pub async fn save( #[cfg(test)] mod tests { use super::*; + #[test] fn lets_encrypt_needs_no_provider_credentials_and_zerossl_needs_eab() { let plain = CertificateSettings { @@ -98,13 +108,20 @@ mod tests { ); let zero = CertificateSettings { issuer: "zerossl".into(), - eab: json!({"eab_kid":"test-id","eab_hmac_key":"c2VjcmV0"}), + eab: json!({ + "eab_kid": "test-id", + "eab_hmac_key": "c2VjcmV0", + }), }; assert!(zero.validate().is_ok()); } + #[test] fn authority_credentials_are_encrypted_and_bound_to_the_setting() { - let secret = json!({"eab_kid":"test-id","eab_hmac_key":"c2VjcmV0"}); + let secret = json!({ + "eab_kid": "test-id", + "eab_hmac_key": "c2VjcmV0", + }); let encrypted = certificates::encrypt("key", Uuid::nil(), SECRET_KEY, &secret).unwrap(); assert!(!encrypted.to_string().contains("test-id")); assert_eq!( diff --git a/apps/control-api/src/domain/domain_dns.rs b/apps/control-api/src/domain/domain_dns.rs index 08c7e62..9b7431e 100644 --- a/apps/control-api/src/domain/domain_dns.rs +++ b/apps/control-api/src/domain/domain_dns.rs @@ -127,6 +127,7 @@ mod tests { server.abort(); } } + #[tokio::test] async fn accepts_cname_chains_and_rejects_a_different_entry_on_shared_ip() { for (target, expected) in [ @@ -142,11 +143,26 @@ mod tests { ( "site.example.org", "A", - json!({"Status":0,"Answer":[ - {"name":"site.example.org.","type":5,"data":"alias.example.org."}, - {"name":"alias.example.org.","type":5,"data":"entry.example.com."}, - {"name":"entry.example.com.","type":1,"data":"203.0.113.1"} - ]}), + json!({ + "Status": 0, + "Answer": [ + { + "name": "site.example.org.", + "type": 5, + "data": "alias.example.org.", + }, + { + "name": "alias.example.org.", + "type": 5, + "data": "entry.example.com.", + }, + { + "name": "entry.example.com.", + "type": 1, + "data": "203.0.113.1", + }, + ], + }), ), ( "site.example.org", @@ -169,6 +185,7 @@ mod tests { server.abort(); } } + #[tokio::test] async fn distinguishes_unresolved_customer_dns_from_missing_entry_dns() { let (resolver, server) = fixture(vec![( @@ -191,6 +208,7 @@ mod tests { ); server.abort(); } + #[tokio::test] async fn wrong_ipv6_and_resolver_failures_do_not_pass_verification() { let (resolver, server) = fixture(vec![ @@ -209,7 +227,7 @@ mod tests { "AAAA", answer("site.example.org", 28, "2001:db8::2"), ), - ("broken.example.org", "A", json!({"Status":2})), + ("broken.example.org", "A", json!({ "Status": 2 })), ]) .await; assert_eq!( diff --git a/apps/control-api/src/domain/retention.rs b/apps/control-api/src/domain/retention.rs index 159c5b9..adc2fcc 100644 --- a/apps/control-api/src/domain/retention.rs +++ b/apps/control-api/src/domain/retention.rs @@ -229,8 +229,8 @@ pub async fn sweep( && let Some(size_bytes) = artifact.size_bytes { let size_mb = (size_bytes.max(0) + BYTES_PER_MB - 1) / BYTES_PER_MB; - if size_mb > 0 { - if let Err(error) = quota + if size_mb > 0 + && let Err(error) = quota .release_once( "retention.artifact", deployment.team_id, @@ -242,9 +242,8 @@ pub async fn sweep( artifact.id, ) .await - { - tracing::warn!(operation = "artifact_retention.release_quota", %error, artifact_id = %artifact.id, "failed to release artifact quota"); - } + { + tracing::warn!(operation = "artifact_retention.release_quota", %error, artifact_id = %artifact.id, "failed to release artifact quota"); } } diff --git a/apps/control-api/src/features/api/v1/admin/regions/by_code.rs b/apps/control-api/src/features/api/v1/admin/regions/by_code.rs index 2449932..22a6f70 100644 --- a/apps/control-api/src/features/api/v1/admin/regions/by_code.rs +++ b/apps/control-api/src/features/api/v1/admin/regions/by_code.rs @@ -131,7 +131,12 @@ async fn remove( source: source.into(), })?; if referenced || item.code == "default" { - return Err(AppError::Conflict { op: OP, message: "This region is reserved or still referenced by nodes, configurations, entries, domains, or deployments.".to_owned() }); + return Err(AppError::Conflict { + op: OP, + message: "This region is reserved or still referenced by nodes, configurations, \ + entries, domains, or deployments." + .to_owned(), + }); } region::Entity::delete_by_id(item.code) .exec(&transaction) diff --git a/apps/control-api/src/features/frontend.rs b/apps/control-api/src/features/frontend.rs index 2a8312c..6756dc4 100644 --- a/apps/control-api/src/features/frontend.rs +++ b/apps/control-api/src/features/frontend.rs @@ -19,15 +19,15 @@ pub(super) async fn frontend_fallback(uri: Uri) -> Response { // Try ./public override first. let public_path = Path::new(PUBLIC_DIR).join(path); - if public_path.is_file() { - if let Ok(content) = tokio::fs::read(&public_path).await { - let mime = mime_guess::from_path(&public_path).first_or_octet_stream(); - return Response::builder() - .header(header::CONTENT_TYPE, mime.as_ref()) - .header(header::CACHE_CONTROL, "public, max-age=0") - .body(Body::from(content)) - .unwrap_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response()); - } + if public_path.is_file() + && let Ok(content) = tokio::fs::read(&public_path).await + { + let mime = mime_guess::from_path(&public_path).first_or_octet_stream(); + return Response::builder() + .header(header::CONTENT_TYPE, mime.as_ref()) + .header(header::CACHE_CONTROL, "public, max-age=0") + .body(Body::from(content)) + .unwrap_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response()); } // Try embedded asset. diff --git a/apps/control-api/src/infra/database/migrate.rs b/apps/control-api/src/infra/database/migrate.rs index dcd01fe..b390574 100644 --- a/apps/control-api/src/infra/database/migrate.rs +++ b/apps/control-api/src/infra/database/migrate.rs @@ -1,7 +1,8 @@ -use super::migration; use sea_orm::DatabaseConnection; use sea_orm_migration::{MigratorTrait, prelude::*}; +use super::migration; + #[cfg(test)] pub(crate) static MIGRATION_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); @@ -58,3077 +59,4 @@ pub async fn run(database: &DatabaseConnection) -> anyhow::Result<()> { } #[cfg(test)] -mod tests { - use std::{collections::BTreeMap, future::Future}; - - use anyhow::{Context, ensure}; - use sea_orm::{ConnectionTrait, Database, DatabaseBackend, Statement}; - use tokio::sync::oneshot; - use uuid::Uuid; - - use super::*; - - #[tokio::test] - async fn shared_postgres_migration_lock_serializes_access() { - let (started_sender, started_receiver) = oneshot::channel(); - let (release_sender, release_receiver) = oneshot::channel(); - let first = tokio::spawn(async move { - let _guard = super::MIGRATION_TEST_LOCK.lock().await; - started_sender.send(()).unwrap(); - release_receiver.await.unwrap(); - }); - started_receiver.await.unwrap(); - - let (second_attempted_sender, second_attempted_receiver) = oneshot::channel(); - let (second_acquired_sender, mut second_acquired_receiver) = oneshot::channel(); - let second = tokio::spawn(async move { - let mut lock = Box::pin(super::MIGRATION_TEST_LOCK.lock()); - let mut attempted_sender = Some(second_attempted_sender); - let _guard = std::future::poll_fn(move |context| { - if let Some(sender) = attempted_sender.take() { - sender.send(()).unwrap(); - } - lock.as_mut().poll(context) - }) - .await; - second_acquired_sender.send(()).unwrap(); - }); - - second_attempted_receiver.await.unwrap(); - assert!(second_acquired_receiver.try_recv().is_err()); - - release_sender.send(()).unwrap(); - second.await.unwrap(); - assert!(second_acquired_receiver.await.is_ok()); - first.await.unwrap(); - } - - #[derive(Debug, Eq, PartialEq)] - struct ColumnShape { - name: String, - udt_name: String, - nullable: String, - default: Option, - } - - struct PostgresMigrationDatabase { - db: DatabaseConnection, - admin: DatabaseConnection, - schema: String, - } - - impl PostgresMigrationDatabase { - async fn start(database_url: &str) -> anyhow::Result { - let admin = Database::connect(database_url).await?; - let schema = format!("gw_audit_migration_{}", Uuid::now_v7().simple()); - admin - .execute_unprepared(&format!("CREATE SCHEMA {schema}")) - .await?; - - let mut scoped_url = url::Url::parse(database_url)?; - scoped_url - .query_pairs_mut() - .append_pair("options", &format!("-csearch_path={schema}")); - let db = match Database::connect(scoped_url.as_str()).await { - Ok(db) => db, - Err(error) => { - admin - .execute_unprepared(&format!("DROP SCHEMA {schema} CASCADE")) - .await?; - return Err(error.into()); - } - }; - - Ok(Self { db, admin, schema }) - } - - async fn cleanup(self) -> anyhow::Result<()> { - self.db.close().await?; - self.admin - .execute_unprepared(&format!("DROP SCHEMA {} CASCADE", self.schema)) - .await?; - self.admin.close().await?; - Ok(()) - } - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL and disposable schema permission"] - async fn postgres_auth_version_shape_and_password_revocation() -> anyhow::Result<()> { - postgres_account_revocation(grass_cache::CacheStore::Moka( - grass_cache::MokaCache::connect(), - )) - .await - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL, GRASS_TEST_REDIS_URL and disposable schema permission"] - async fn postgres_redis_auth_version_shape_and_password_revocation() -> anyhow::Result<()> { - postgres_account_revocation(grass_cache::CacheStore::Redis( - grass_cache::RedisCache::connect(&std::env::var("GRASS_TEST_REDIS_URL")?).await?, - )) - .await - } - - async fn postgres_account_revocation( - cache_store: grass_cache::CacheStore, - ) -> anyhow::Result<()> { - use crate::{ - domain::{authentication, users}, - infra::{ - config::ControlApiConfig, - database::entity::{AuthTokenKind, PlatformRole, UserStatus}, - http::{extractors::Session, middlewares::session}, - }, - state::ControlApiState, - }; - use axum::{Router, body::Body, http::Request, middleware, routing::get}; - use grass_cache::Cache; - use std::time::Duration; - use tower::ServiceExt; - let _guard = MIGRATION_TEST_LOCK.lock().await; - let database = - PostgresMigrationDatabase::start(&std::env::var("GRASS_TEST_DATABASE_URL")?).await?; - let result: anyhow::Result<()> = async { - let db = &database.db; - Migrator::up(db, None).await?; - assert_migration_tracking(db, 35).await?; - let shapes = query_column_shapes(db, "SELECT column_name, udt_name, is_nullable, column_default FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'users' AND column_name = 'auth_version'").await?; - ensure!(shapes == vec![column("auth_version", "int8", "NO", Some("1"))], "incorrect authentication version column shape"); - let constraint = db.query_one_raw(Statement::from_string(DatabaseBackend::Postgres, "SELECT pg_get_constraintdef(oid) AS definition FROM pg_constraint WHERE conrelid = 'users'::regclass AND conname = 'users_auth_version_check'")).await?.unwrap(); - ensure!(constraint.try_get::("", "definition")?.contains("auth_version > 0")); - let user = users::create_user(db, users::CreateUserParams { email: format!("revocation-{}@example.test", Uuid::now_v7()), display_name: None, password_hash: Some(grass_crypto::hash_password("Original-password-123!")?), platform_role: PlatformRole::Admin, email_verified_at: Some(time::OffsetDateTime::now_utc()) }).await?; - ensure!(user.auth_version == 1); - let state = ControlApiState::new(ControlApiConfig::default(), "unused.toml"); - state.database.set(db.clone()).ok().unwrap(); - state.cache.set(cache_store).ok().unwrap(); - async fn protected(_session: Session) -> &'static str { "allowed" } - async fn admin(_admin: crate::infra::http::extractors::PlatformAdmin) -> &'static str { "allowed" } - let app = Router::new().route("/protected", get(protected).post(protected)) - .route("/admin", get(admin)) - .nest("/api/v1/auth", crate::features::api::v1::auth::login::router() - .merge(crate::features::api::v1::auth::password::reset::router())) - .nest("/api/v1", crate::features::api::v1::me::password::router()) - .nest("/api/v1/admin", crate::features::api::v1::admin::users::by_user_id::reset_password::router()) - .layer(middleware::from_fn_with_state(state.clone(), session::session_middleware)).with_state(state.clone()); - let cache = state.try_cache().unwrap(); - let mut current_password = "Original-password-123!"; - for (flow, next_password) in [("change", "Changed-password-123!"), ("reset", "Reset-password-123!"), ("admin", "Admin-reset-password-123!")] { - let current = users::get_user_by_id(db, user.id).await?.unwrap(); - let first = grass_session::create_session(cache, user.id, current.auth_version, Duration::from_secs(300)).await?; - let second = grass_session::create_session(cache, user.id, current.auth_version, Duration::from_secs(300)).await?; - for sid in [&first, &second] { - ensure!(session::validate_current_session(&state, sid, "test.active").await?.is_some()); - } - // A refresh may read an old session before the password transaction commits. - let key = format!("session:{second}"); - let stale_refresh = cache.get(&key).await?.unwrap(); - let (uri, body) = match flow { - "change" => ("/api/v1/me/password".into(), serde_json::json!({"current_password": current_password, "password": next_password})), - "reset" => { - let token = authentication::create_auth_token(db, user.id, AuthTokenKind::PasswordReset, time::Duration::hours(1)).await?; - ("/api/v1/auth/password/reset".into(), serde_json::json!({"token": token, "password": next_password})) - }, - _ => (format!("/api/v1/admin/users/{}/reset-password", user.id), serde_json::json!({"password": next_password})), - }; - let response = app.clone().oneshot(Request::builder().uri(uri).method("POST").header("content-type", "application/json").header("cookie", format!("session_id={first}")).body(Body::from(body.to_string()))?).await?; - ensure!(response.status().is_success(), "password flow {flow} failed with {}", response.status()); - let updated = users::get_user_by_id(db, user.id).await?.unwrap(); - ensure!(updated.auth_version == current.auth_version + 1); - // Complete that delayed cache write after revocation; DB state must still win. - ensure!(cache.update_if_present(&key, &stale_refresh, Duration::from_secs(300)).await?); - for (sid, method, path) in [(&first, "GET", "/protected"), (&second, "POST", "/protected"), (&second, "GET", "/admin")] { - let response = app.clone().oneshot(Request::builder().uri(path).method(method).header("cookie", format!("session_id={sid}")).body(Body::empty())?).await?; - ensure!(response.status().as_u16() == 401, "old session survived {flow}"); - } - ensure!(users::verify_user_password(db, &user.email, next_password).await?.is_some()); - ensure!(users::verify_user_password(db, &user.email, current_password).await?.is_none()); - let response = app.clone().oneshot(Request::builder().uri("/api/v1/auth/login").method("POST") - .extension(axum::extract::ConnectInfo(std::net::SocketAddr::from(([127, 0, 0, 1], 12345)))) - .header("content-type", "application/json") - .body(Body::from(serde_json::json!({"email": user.email, "password": next_password}).to_string()))?).await?; - ensure!(response.status().is_success(), "new password could not log in after {flow}"); - let sid = response.headers().get_all("set-cookie").iter() - .filter_map(|cookie| cookie.to_str().ok()) - .find_map(|cookie| cookie.strip_prefix("session_id=").and_then(|value| value.split(';').next())) - .context("login did not issue a session")?; - ensure!(session::validate_current_session(&state, sid, "test.login").await?.is_some()); - grass_session::revoke_session(cache, sid).await?; - current_password = next_password; - } - let current = users::get_user_by_id(db, user.id).await?.unwrap(); - let first = grass_session::create_session(cache, user.id, current.auth_version, Duration::from_secs(300)).await?; - let second = grass_session::create_session(cache, user.id, current.auth_version, Duration::from_secs(300)).await?; - ensure!(session::validate_current_session(&state, &first, "test.before-disable").await?.is_some()); - let disabled = users::update_user(db, current.clone(), users::UpdateUserParams { - display_name: None, status: Some(UserStatus::Disabled), platform_role: None, - }).await?; - ensure!(disabled.auth_version == current.auth_version + 1); - ensure!(session::validate_current_session(&state, &first, "test.disabled").await?.is_none()); - let enabled = users::update_user(db, disabled, users::UpdateUserParams { - display_name: None, status: Some(UserStatus::Active), platform_role: None, - }).await?; - ensure!(enabled.auth_version == current.auth_version + 1); - ensure!(session::validate_current_session(&state, &second, "test.reenabled").await?.is_none()); - let fresh = grass_session::create_session(cache, user.id, enabled.auth_version, Duration::from_secs(300)).await?; - ensure!(session::validate_current_session(&state, &fresh, "test.enabled").await?.is_some()); - db.execute_raw(Statement::from_sql_and_values(DatabaseBackend::Postgres, - "UPDATE users SET deleted_at = CURRENT_TIMESTAMP WHERE id = $1", [user.id.into()])).await?; - ensure!(session::validate_current_session(&state, &fresh, "test.deleted").await?.is_none()); - Ok(()) - }.await; - database.cleanup().await?; - result - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL"] - async fn postgres_region_catalog_backfills_and_enforces_references() -> anyhow::Result<()> { - let _guard = MIGRATION_TEST_LOCK.lock().await; - let database_url = std::env::var("GRASS_TEST_DATABASE_URL")?; - let database = PostgresMigrationDatabase::start(&database_url).await?; - let result: anyhow::Result<()> = async { - Migrator::up(&database.db, Some(32)).await?; - database.db.execute_unprepared("INSERT INTO regional_ingresses (id, region, hostname, created_at, updated_at) VALUES ('00000000-0000-0000-0000-000000000101', 'hk_1', 'hk.entry.example.com', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)").await?; - Migrator::up(&database.db, Some(1)).await?; - let rows = database.db.query_all_raw(Statement::from_string(DatabaseBackend::Postgres, "SELECT code FROM regions ORDER BY code".to_owned())).await?; - let codes = rows.iter().map(|r| r.try_get::("", "code")).collect::, _>>()?; - ensure!(codes == vec!["default", "hk_1"]); - let foreign_keys = object_count(&database.db, "SELECT count(*) AS count FROM pg_constraint WHERE contype = 'f' AND confrelid = 'regions'::regclass").await?; - ensure!(foreign_keys == 5); - ensure!(database.db.execute_unprepared("DELETE FROM regions WHERE code = 'hk_1'").await.is_err()); - ensure!(database.db.execute_unprepared("INSERT INTO regions (code, name) VALUES ('hk_1', 'duplicate')").await.is_err()); - ensure!(database.db.execute_unprepared("INSERT INTO regional_ingresses (id, region, hostname, created_at, updated_at) VALUES ('00000000-0000-0000-0000-000000000102', 'unknown', 'unknown.entry.example.com', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)").await.is_err()); - ensure!(database.db.execute_unprepared("INSERT INTO regional_ingresses (id, region, hostname, created_at, updated_at) VALUES ('00000000-0000-0000-0000-000000000103', 'hk_1', 'second.entry.example.com', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)").await.is_err()); - let columns = query_column_shapes(&database.db, "SELECT column_name, udt_name, is_nullable, column_default FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'regions'").await?; - ensure!(columns.iter().any(|c| c.name == "code" && c.nullable == "NO" && c.udt_name == "text")); - ensure!(columns.iter().any(|c| c.name == "name" && c.nullable == "NO")); - Ok(()) - }.await; - database.cleanup().await?; - result - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL"] - async fn postgres_domain_onboarding_migrates_and_checks_customer_dns() -> anyhow::Result<()> { - use crate::domain::{acme, certificate_settings, certificates, domain_onboarding}; - use crate::infra::database::entity::{ - HostBindingStatus, managed_certificate, project_host_binding as binding, - regional_ingress, - }; - use sea_orm::{ActiveModelTrait, EntityTrait, Set}; - use serde_json::json; - let _guard = MIGRATION_TEST_LOCK.lock().await; - let database = - PostgresMigrationDatabase::start(&std::env::var("GRASS_TEST_DATABASE_URL")?).await?; - let result: anyhow::Result<()> = async { - let db = &database.db; - Migrator::up(db, Some(33)).await?; - let owner = Uuid::now_v7(); - let actor = Uuid::now_v7(); - let mut old = crate::test_support::certificates::binding_fixture(); - old.host = "legacy.example.org".into(); - let entry_id = Uuid::now_v7(); - db.execute_unprepared(&format!(r#" - INSERT INTO users (id, email, display_name) VALUES ('{owner}', 'owner@example.org', 'Owner'), ('{actor}', 'adder@example.org', 'Adding user'); - INSERT INTO teams (id, slug, name, owner_user_id) VALUES ('{}', 'onboarding', 'Onboarding', '{owner}'); - INSERT INTO projects (id, team_id, slug, name, created_by_user_id) VALUES ('{}', '{}', 'onboarding', 'Onboarding', '{owner}'); - INSERT INTO regions (code, name) VALUES ('eu', 'Europe'); - INSERT INTO regional_ingresses (id, region, hostname, created_at, updated_at) VALUES ('{entry_id}', 'eu', 'entry.example.org', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP); - "#, old.team_id, old.project_id, old.team_id)).await?; - binding::ActiveModel::from(old.clone()).insert(db).await?; - db.execute_unprepared(&format!(r#" - INSERT INTO managed_certificates (id, ingress_id, hostname, issuer, generation) VALUES ('{entry_id}', '{entry_id}', 'entry.example.org', 'letsencrypt', '{entry_id}'); - INSERT INTO managed_certificates (id, ingress_id, host_binding_id, hostname, issuer, generation, challenge_method) VALUES ('{0}', '{entry_id}', '{0}', 'legacy.example.org', 'letsencrypt', '{0}', 'dns01'); - "#, old.id)).await?; - Migrator::up(db, None).await?; - assert_migration_tracking(db, 35).await?; - ensure!(managed_certificate::Entity::find_by_id(entry_id).one(db).await?.is_none(), "entry certificate must be removed"); - let legacy = managed_certificate::Entity::find_by_id(old.id).one(db).await?.unwrap(); - ensure!(legacy.challenge_method == "http01" && legacy.contact_email == "owner@example.org"); - let entry_columns = query_column_shapes(db, "SELECT column_name, udt_name, is_nullable, column_default FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'regional_ingresses'").await?; - ensure!(!entry_columns.iter().any(|c| c.name.starts_with("certificate_") || c.name.starts_with("dns_challenge") || c.name == "tls_enabled" || c.name == "acme_account")); - ensure!(entry_columns.iter().any(|c| c.name == "dns_checked_at" && c.udt_name == "timestamptz" && c.nullable == "YES" && c.default.is_none())); - let columns = query_column_shapes(db, "SELECT column_name, udt_name, is_nullable, column_default FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'domain_onboarding'").await?; - for name in ["checked_at", "lease_until"] { - ensure!(columns.iter().any(|c| c.name == name && c.udt_name == "timestamptz" && c.nullable == "YES" && c.default.is_none())); - } - ensure!(columns.iter().any(|c| c.name == "next_check_at" && c.nullable == "NO" && c.default.is_some())); - ensure!(columns.iter().any(|c| c.name == "contact_email" && c.udt_name == "text" && c.nullable == "NO")); - ensure!(object_count(db, "SELECT count(*) AS count FROM pg_constraint WHERE conrelid = 'domain_onboarding'::regclass AND contype = 'f'").await? == 2); - ensure!(object_count(db, "SELECT count(*) AS count FROM pg_indexes WHERE schemaname = current_schema() AND indexname = 'ix_domain_onboarding_due'").await? == 1); - ensure!(db.execute_unprepared("UPDATE managed_certificates SET host_binding_id = NULL").await.is_err()); - ensure!(db.execute_unprepared("UPDATE managed_certificates SET challenge_method = 'dns01'").await.is_err()); - ensure!(db.execute_unprepared("UPDATE domain_onboarding SET dns_status = 'invalid'").await.is_err()); - let mut custom = old.clone(); - custom.id = Uuid::now_v7(); - custom.host = "site.example.org".into(); - custom.status = HostBindingStatus::Pending; - custom.ownership_status = "pending".into(); - binding::ActiveModel::from(custom.clone()).insert(db).await?; - domain_onboarding::create(db, &custom, actor).await?; - let contact = domain_onboarding::get(db, custom.id).await?.unwrap(); - ensure!(contact.created_by_user_id == Some(actor) && contact.contact_email == "adder@example.org"); - - let entry = regional_ingress::Entity::find_by_id(entry_id).one(db).await?.unwrap(); - let token = crate::domain::ingress::dns_verification_token("secret", custom.id, &custom.host); - for (address, txt, expected, active) in [ - (None, "wrong", "unresolved", false), - (Some("203.0.113.9"), token.as_str(), "mismatch", false), - (Some("203.0.113.1"), "wrong", "ready", false), - (Some("203.0.113.1"), token.as_str(), "ready", true), - ] { - let mut records = vec![ - ("entry.example.org", "A", crate::test_support::dns::answer("entry.example.org", 1, "203.0.113.1")), - ("_grass.site.example.org", "TXT", crate::test_support::dns::answer("_grass.site.example.org", 16, &format!("\"{txt}\""))), - ]; - if let Some(address) = address { records.push(("site.example.org", "A", crate::test_support::dns::answer("site.example.org", 1, address))); } - let (resolver, server) = crate::test_support::dns::fixture(records).await; - // Immediate checks and scheduled checks use the same persisted workflow. - domain_onboarding::run_check_with_resolver(db, custom.id, "secret", true, &resolver).await?; - server.abort(); - let check = domain_onboarding::get(db, custom.id).await?.unwrap(); - let bound = binding::Entity::find_by_id(custom.id).one(db).await?.unwrap(); - ensure!(check.dns_status == expected, "unexpected DNS state: {}", check.dns_status); - ensure!((bound.status == HostBindingStatus::Active) == active); - ensure!(check.lease_until.is_none()); - let interval = check.next_check_at - check.checked_at.unwrap(); - ensure!((60..=120).contains(&interval.whole_seconds())); - } - // The scheduler creates only customer-domain certificates, even without a deployment. - acme::sweep(db, "secret").await?; - let cert = managed_certificate::Entity::find_by_id(custom.id).one(db).await?.unwrap(); - ensure!(cert.hostname == custom.host && cert.contact_email == "adder@example.org" && cert.auto_renew && cert.challenge_method == "http01"); - ensure!(cert.status == "pending", "no eligible entry nodes means no external order"); - ensure!(managed_certificate::Entity::find_by_id(entry_id).one(db).await?.is_none()); - let again = certificates::ensure_record(db, &entry, Some(&custom)).await?; - ensure!(again.id == cert.id && again.generation == cert.generation); - // Not-yet-due checks do not query DNS; leases also protect forced checks. - let (resolver, server) = crate::test_support::dns::fixture(vec![]).await; - let before = domain_onboarding::get(db, custom.id).await?.unwrap(); - domain_onboarding::run_check_with_resolver(db, custom.id, "secret", false, &resolver).await?; - ensure!(domain_onboarding::get(db, custom.id).await?.unwrap() == before); - db.execute_unprepared(&format!("UPDATE domain_onboarding SET lease_until = CURRENT_TIMESTAMP + INTERVAL '1 minute' WHERE binding_id = '{}'", custom.id)).await?; - domain_onboarding::run_check_with_resolver(db, custom.id, "secret", true, &resolver).await?; - ensure!(domain_onboarding::get(db, custom.id).await?.unwrap().dns_status == "ready"); - // Simulate a restart after an abandoned lease. The next scheduled check recovers. - db.execute_unprepared(&format!("UPDATE domain_onboarding SET lease_until = CURRENT_TIMESTAMP - INTERVAL '1 second', next_check_at = CURRENT_TIMESTAMP - INTERVAL '1 second' WHERE binding_id = '{}'", custom.id)).await?; - domain_onboarding::run_check_with_resolver(db, custom.id, "secret", false, &resolver).await?; - ensure!(domain_onboarding::get(db, custom.id).await?.unwrap().dns_status == "entry_unavailable"); - server.abort(); - let settings = certificate_settings::CertificateSettings { issuer: "zerossl".into(), eab: json!({"eab_kid":"test-id","eab_hmac_key":"c2VjcmV0"}) }; - certificate_settings::save(db, &settings, "secret").await?; - ensure!(certificate_settings::issuer(db).await? == "zerossl"); - let loaded = certificate_settings::load(db, "secret").await?; - ensure!(loaded.eab == settings.eab); - let stored = crate::domain::settings::get_setting(db, "domain_https").await?.unwrap(); - ensure!(!stored.value.to_string().contains("c2VjcmV0")); - let mut disabled: binding::ActiveModel = binding::Entity::find_by_id(custom.id).one(db).await?.unwrap().into(); - disabled.status = Set(HostBindingStatus::Disabled); - disabled.update(db).await?; - let (resolver, server) = crate::test_support::dns::fixture(vec![]).await; - domain_onboarding::run_check_with_resolver(db, custom.id, "secret", true, &resolver).await?; - ensure!(binding::Entity::find_by_id(custom.id).one(db).await?.unwrap().status == HostBindingStatus::Disabled); - server.abort(); - // Down/up restores the legacy shape, while reapplication produces the same new constraints. - Migrator::down(db, Some(2)).await?; - assert_migration_tracking(db, 33).await?; - Migrator::up(db, None).await?; - assert_migration_tracking(db, 35).await?; - Ok(()) - }.await; - database.cleanup().await?; - result - } - - #[test] - fn registers_audit_foundation_migration() { - let migrations = Migrator::migrations(); - - assert_eq!(migrations.len(), 35); - assert_eq!( - migrations.get(11).expect("twelfth migration").name(), - "m20260729_000012_audit_foundation" - ); - - let sql = migration::m20260729_000012_audit_foundation::UP_SQL; - assert!(sql.contains("CREATE TYPE audit_actor_type")); - assert!(sql.contains("CREATE TYPE audit_event_visibility")); - assert!(sql.contains("ADD COLUMN request_id UUID NULL")); - assert!(sql.contains("ADD COLUMN changes JSONB NOT NULL DEFAULT '{}'")); - assert!( - sql.contains("ADD COLUMN pending_release_audit_visibility audit_event_visibility NULL") - ); - assert!(sql.contains("SET pending_release_audit_visibility = 'platform'")); - assert!(sql.contains("ck_deployments_pending_release_audit_visibility")); - assert!(sql.contains("actor_user_id IS NULL OR actor_type = 'user'")); - assert!(sql.contains("actor_node_id IS NULL OR actor_type = 'node'")); - assert!(sql.contains("actor_type NOT IN ('anonymous', 'system')")); - assert!(sql.contains("WHEN actor_user_id IS NOT NULL THEN 'user'")); - assert!(sql.contains("COALESCE(metadata ->> 'platform_admin', 'false') <> 'true'")); - assert!(sql.contains("COALESCE(metadata ->> 'completed_after_sync', 'false') <> 'true'")); - assert!(sql.contains("'team.quota_plan_overridden'")); - } - - #[test] - fn registers_team_group_review_policy_migration() { - let migrations = Migrator::migrations(); - - assert_eq!(migrations.len(), 35); - assert_eq!( - migrations.get(12).expect("thirteenth migration").name(), - "m20260729_000013_team_group_review_policy" - ); - } - - #[test] - fn registers_node_config_sync_migration() { - let migrations = Migrator::migrations(); - - assert_eq!(migrations.len(), 35); - assert_eq!( - migrations.get(13).expect("fourteenth migration").name(), - "m20260729_000014_node_config_sync" - ); - } - - #[test] - fn registers_node_deletion_queue_migration() { - let migrations = Migrator::migrations(); - - assert_eq!(migrations.len(), 35); - assert_eq!( - migrations.get(14).expect("fifteenth migration").name(), - "m20260729_000015_node_deletion_queue" - ); - } - - #[test] - fn registers_domain_review_policy_after_node_deletion_queue() { - let migrations = Migrator::migrations(); - - assert_eq!(migrations.len(), 35); - assert_eq!( - migrations.get(14).expect("fifteenth migration").name(), - "m20260729_000015_node_deletion_queue" - ); - assert_eq!( - migrations.get(15).expect("sixteenth migration").name(), - "m20260730_000016_domain_review_policy" - ); - } - - #[test] - fn registers_project_notifications_after_domain_review_policy() { - let migrations = Migrator::migrations(); - - assert_eq!(migrations.len(), 35); - assert_eq!( - migrations.get(15).expect("sixteenth migration").name(), - "m20260730_000016_domain_review_policy" - ); - assert_eq!( - migrations.get(16).expect("seventeenth migration").name(), - "m20260731_000017_project_notifications" - ); - assert_eq!( - migrations.get(17).expect("eighteenth migration").name(), - "m20260801_000018_artifact_retention" - ); - assert_eq!( - migrations.get(22).expect("twenty-third migration").name(), - "m20260804_000023_mfa_policy" - ); - } - - #[test] - fn registers_scoped_codes_after_authentication_migrations() { - let migrations = Migrator::migrations(); - - assert_eq!(migrations.len(), 35); - assert_eq!( - migrations.get(23).expect("twenty-fourth migration").name(), - "m20260806_000024_scoped_codes" - ); - } - - #[test] - fn registers_registration_allowlist_after_scoped_codes() { - let migrations = Migrator::migrations(); - - assert_eq!(migrations.len(), 35); - assert_eq!( - migrations.get(24).expect("twenty-fifth migration").name(), - "m20260806_000025_registration_allowlist" - ); - } - - #[test] - fn registers_avatar_versions_after_registration_allowlist() { - let migrations = Migrator::migrations(); - - assert_eq!(migrations.len(), 35); - assert_eq!( - migrations.get(25).expect("twenty-sixth migration").name(), - "m20260807_000026_avatars" - ); - } - - #[test] - fn registers_object_storage_after_deployment_screenshots() { - let migrations = Migrator::migrations(); - - assert_eq!(migrations.len(), 35); - assert_eq!( - migrations.get(26).expect("twenty-seventh migration").name(), - "m20260807_000027_deployment_screenshots" - ); - assert_eq!( - migrations.get(27).expect("twenty-eighth migration").name(), - "m20260808_000028_object_storage" - ); - assert_eq!( - migrations.last().expect("last migration").name(), - "m20260912_000035_user_auth_version" - ); - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL"] - async fn postgres_media_schema_matches_domain_and_is_reversible() -> anyhow::Result<()> { - let _migration_guard = MIGRATION_TEST_LOCK.lock().await; - let database_url = std::env::var("GRASS_TEST_DATABASE_URL") - .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); - let test_db = PostgresMigrationDatabase::start(&database_url).await?; - - let verification = async { - Migrator::up(&test_db.db, Some(31)).await?; - assert_migration_tracking(&test_db.db, 31).await?; - assert_avatar_schema(&test_db.db).await?; - assert_screenshot_schema(&test_db.db).await?; - assert_object_storage_schema(&test_db.db).await?; - - Migrator::down(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 30).await?; - assert_regional_ingress_lifecycle_absent(&test_db.db).await?; - assert_avatar_schema(&test_db.db).await?; - assert_screenshot_schema(&test_db.db).await?; - assert_object_storage_schema(&test_db.db).await?; - - Migrator::down(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 29).await?; - assert_regional_ingress_schema_absent(&test_db.db).await?; - assert_avatar_schema(&test_db.db).await?; - assert_screenshot_schema(&test_db.db).await?; - assert_object_storage_schema(&test_db.db).await?; - - Migrator::down(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 28).await?; - assert_avatar_schema(&test_db.db).await?; - assert_screenshot_schema(&test_db.db).await?; - assert_object_storage_schema(&test_db.db).await?; - - Migrator::down(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 27).await?; - assert_avatar_schema(&test_db.db).await?; - assert_screenshot_schema(&test_db.db).await?; - assert_object_storage_schema_absent(&test_db.db).await?; - - Migrator::up(&test_db.db, Some(4)).await?; - assert_migration_tracking(&test_db.db, 31).await?; - assert_avatar_schema(&test_db.db).await?; - assert_screenshot_schema(&test_db.db).await?; - assert_object_storage_schema(&test_db.db).await - } - .await; - let cleanup = test_db.cleanup().await; - - match (verification, cleanup) { - (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context( - format!("disposable schema cleanup also failed: {cleanup_error:#}"), - )), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Ok(()), Ok(())) => Ok(()), - } - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL"] - async fn postgres_regional_ingress_schema_matches_domain_and_is_reversible() - -> anyhow::Result<()> { - let _migration_guard = MIGRATION_TEST_LOCK.lock().await; - let database_url = std::env::var("GRASS_TEST_DATABASE_URL") - .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); - let test_db = PostgresMigrationDatabase::start(&database_url).await?; - - let verification = async { - Migrator::up(&test_db.db, Some(31)).await?; - assert_migration_tracking(&test_db.db, 31).await?; - assert_regional_ingress_schema(&test_db.db).await?; - - Migrator::down(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 30).await?; - assert_regional_ingress_lifecycle_absent(&test_db.db).await?; - - Migrator::down(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 29).await?; - assert_regional_ingress_schema_absent(&test_db.db).await?; - - Migrator::up(&test_db.db, Some(2)).await?; - assert_migration_tracking(&test_db.db, 31).await?; - assert_regional_ingress_schema(&test_db.db).await - } - .await; - let cleanup = test_db.cleanup().await; - - match (verification, cleanup) { - (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context( - format!("disposable schema cleanup also failed: {cleanup_error:#}"), - )), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Ok(()), Ok(())) => Ok(()), - } - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL"] - async fn postgres_registration_allowlist_schema_matches_domain_and_is_reversible() - -> anyhow::Result<()> { - let _migration_guard = MIGRATION_TEST_LOCK.lock().await; - let database_url = std::env::var("GRASS_TEST_DATABASE_URL") - .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); - let test_db = PostgresMigrationDatabase::start(&database_url).await?; - - let verification = async { - Migrator::up(&test_db.db, Some(25)).await?; - assert_migration_tracking(&test_db.db, 25).await?; - assert_registration_allowlist_schema(&test_db.db).await?; - - Migrator::down(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 24).await?; - assert_registration_allowlist_schema_absent(&test_db.db).await?; - - Migrator::up(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 25).await?; - assert_registration_allowlist_schema(&test_db.db).await - } - .await; - let cleanup = test_db.cleanup().await; - - match (verification, cleanup) { - (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context( - format!("disposable schema cleanup also failed: {cleanup_error:#}"), - )), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Ok(()), Ok(())) => Ok(()), - } - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL"] - async fn postgres_notification_and_announcement_schema_matches_the_domain_model_and_is_reversible() - -> anyhow::Result<()> { - let _migration_guard = MIGRATION_TEST_LOCK.lock().await; - let database_url = std::env::var("GRASS_TEST_DATABASE_URL") - .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); - let test_db = PostgresMigrationDatabase::start(&database_url).await?; - - let verification = async { - Migrator::up(&test_db.db, Some(19)).await?; - assert_migration_tracking(&test_db.db, 19).await?; - - Migrator::up(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 20).await?; - assert_notification_content_schema(&test_db.db).await?; - - Migrator::up(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 21).await?; - assert_announcement_schema(&test_db.db).await?; - - Migrator::down(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 20).await?; - assert_announcement_schema_absent(&test_db.db).await?; - assert_notification_content_schema(&test_db.db).await?; - - Migrator::up(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 21).await?; - assert_announcement_schema(&test_db.db).await - } - .await; - let cleanup = test_db.cleanup().await; - - match (verification, cleanup) { - (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context( - format!("disposable schema cleanup also failed: {cleanup_error:#}"), - )), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Ok(()), Ok(())) => Ok(()), - } - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL"] - async fn postgres_audit_foundation_migration_upgrades_v11_and_is_reversible() - -> anyhow::Result<()> { - let _migration_guard = MIGRATION_TEST_LOCK.lock().await; - let database_url = std::env::var("GRASS_TEST_DATABASE_URL") - .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); - let test_db = PostgresMigrationDatabase::start(&database_url).await?; - - let verification = verify_audit_foundation_migration(&test_db.db).await; - let cleanup = test_db.cleanup().await; - - match (verification, cleanup) { - (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context( - format!("disposable schema cleanup also failed: {cleanup_error:#}"), - )), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Ok(()), Ok(())) => Ok(()), - } - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL"] - async fn postgres_node_deletion_queue_schema_matches_domain_and_is_reversible() - -> anyhow::Result<()> { - let _migration_guard = MIGRATION_TEST_LOCK.lock().await; - let database_url = std::env::var("GRASS_TEST_DATABASE_URL") - .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); - let test_db = PostgresMigrationDatabase::start(&database_url).await?; - - let verification = async { - Migrator::up(&test_db.db, Some(15)).await?; - assert_migration_tracking(&test_db.db, 15).await?; - assert_node_deletion_schema(&test_db.db).await?; - - Migrator::down(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 14).await?; - assert_node_deletion_schema_absent(&test_db.db).await?; - - Migrator::up(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 15).await?; - assert_node_deletion_schema(&test_db.db).await - } - .await; - let cleanup = test_db.cleanup().await; - - match (verification, cleanup) { - (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context( - format!("disposable schema cleanup also failed: {cleanup_error:#}"), - )), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Ok(()), Ok(())) => Ok(()), - } - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL"] - async fn postgres_project_notification_schema_backfills_and_is_reversible() -> anyhow::Result<()> - { - let _migration_guard = MIGRATION_TEST_LOCK.lock().await; - let database_url = std::env::var("GRASS_TEST_DATABASE_URL") - .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); - let test_db = PostgresMigrationDatabase::start(&database_url).await?; - - let verification = async { - Migrator::up(&test_db.db, Some(16)).await?; - assert_migration_tracking(&test_db.db, 16).await?; - let (user_id, project_id) = seed_project_notification_fixture(&test_db.db).await?; - - Migrator::up(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 17).await?; - assert_project_notification_schema(&test_db.db, user_id, project_id).await?; - - Migrator::down(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 16).await?; - assert_project_notification_schema_absent(&test_db.db).await?; - - Migrator::up(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 17).await?; - assert_project_notification_schema(&test_db.db, user_id, project_id).await - } - .await; - let cleanup = test_db.cleanup().await; - - match (verification, cleanup) { - (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context( - format!("disposable schema cleanup also failed: {cleanup_error:#}"), - )), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Ok(()), Ok(())) => Ok(()), - } - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL"] - async fn postgres_authentication_schema_matches_domain_and_is_reversible() -> anyhow::Result<()> - { - let _migration_guard = MIGRATION_TEST_LOCK.lock().await; - let database_url = std::env::var("GRASS_TEST_DATABASE_URL") - .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); - let test_db = PostgresMigrationDatabase::start(&database_url).await?; - - let verification = async { - Migrator::up(&test_db.db, Some(21)).await?; - assert_migration_tracking(&test_db.db, 21).await?; - let user_id = seed_authentication_fixture(&test_db.db).await?; - - Migrator::up(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 22).await?; - assert_authentication_schema(&test_db.db, user_id).await?; - - Migrator::down(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 21).await?; - assert_authentication_schema_absent(&test_db.db).await?; - - Migrator::up(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 22).await?; - assert_authentication_schema(&test_db.db, user_id).await - } - .await; - let cleanup = test_db.cleanup().await; - - match (verification, cleanup) { - (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context( - format!("disposable schema cleanup also failed: {cleanup_error:#}"), - )), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Ok(()), Ok(())) => Ok(()), - } - } - - #[tokio::test] - #[ignore = "requires GRASS_TEST_DATABASE_URL"] - async fn postgres_mfa_policy_schema_migrates_legacy_scope_and_is_reversible() - -> anyhow::Result<()> { - let _migration_guard = MIGRATION_TEST_LOCK.lock().await; - let database_url = std::env::var("GRASS_TEST_DATABASE_URL") - .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); - let test_db = PostgresMigrationDatabase::start(&database_url).await?; - - let verification = async { - Migrator::up(&test_db.db, Some(22)).await?; - assert_migration_tracking(&test_db.db, 22).await?; - let user_id = seed_legacy_mfa_policy_fixture(&test_db.db).await?; - - Migrator::up(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 23).await?; - assert_mfa_policy_schema(&test_db.db, user_id).await?; - - Migrator::down(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 22).await?; - assert_mfa_policy_schema_absent(&test_db.db, user_id).await?; - - Migrator::up(&test_db.db, Some(1)).await?; - assert_migration_tracking(&test_db.db, 23).await?; - assert_mfa_policy_schema(&test_db.db, user_id).await - } - .await; - let cleanup = test_db.cleanup().await; - - match (verification, cleanup) { - (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context( - format!("disposable schema cleanup also failed: {cleanup_error:#}"), - )), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Ok(()), Ok(())) => Ok(()), - } - } - - async fn seed_legacy_mfa_policy_fixture(db: &DatabaseConnection) -> anyhow::Result { - let user_id = Uuid::now_v7(); - let setting_id = Uuid::now_v7(); - db.execute_unprepared(&format!( - r#" -INSERT INTO users (id, email, display_name, email_verified_at) -VALUES ('{user_id}'::uuid, 'mfa-policy-migration@example.invalid', 'MFA Policy Migration', NOW()); - -INSERT INTO system_settings (id, key, value_kind, value, is_secret) -VALUES ( - '{setting_id}'::uuid, - 'auth.mfa_policy', - 'json', - jsonb_build_object( - 'allowed_factors', jsonb_build_array('totp', 'email'), - 'enforcement', 'selected_users', - 'selected_user_ids', jsonb_build_array('{user_id}'::text) - ), - false -); -"# - )) - .await?; - Ok(user_id) - } - - async fn assert_mfa_policy_schema( - db: &DatabaseConnection, - selected_user_id: Uuid, - ) -> anyhow::Result<()> { - let columns = query_column_shapes( - db, - r#" -SELECT column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name = 'user_mfa_policies' -ORDER BY ordinal_position -"#, - ) - .await?; - ensure!( - columns - == vec![ - column("user_id", "uuid", "NO", None), - column("inherit_platform", "bool", "NO", Some("true")), - column("minimum_factors", "int2", "NO", Some("0")), - column("required_factors", "jsonb", "NO", Some("'[]'::jsonb")), - column("created_at", "timestamptz", "NO", None), - column("updated_at", "timestamptz", "NO", None), - ], - "unexpected user MFA policy columns: {columns:#?}" - ); - - let constraints = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT conname, pg_get_constraintdef(oid) AS definition -FROM pg_constraint -WHERE conrelid = 'user_mfa_policies'::regclass -ORDER BY conname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "conname")?, - row.try_get::("", "definition")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - constraints.len() == 4, - "missing user MFA policy constraints" - ); - ensure!( - constraints["ck_user_mfa_policies_minimum_factors"].contains("minimum_factors <= 2") - ); - ensure!( - constraints["ck_user_mfa_policies_required_factors"] - .contains("jsonb_typeof(required_factors)") - && constraints["ck_user_mfa_policies_required_factors"].contains("'array'") - ); - ensure!( - constraints - .values() - .any(|definition| definition.contains("FOREIGN KEY (user_id)") - && definition.contains("REFERENCES users(id) ON DELETE CASCADE")) - ); - - let row = db - .query_one_raw(Statement::from_sql_and_values( - DatabaseBackend::Postgres, - r#" -SELECT - setting.value AS platform_policy, - policy.inherit_platform, - policy.minimum_factors, - policy.required_factors -FROM system_settings AS setting -JOIN user_mfa_policies AS policy ON policy.user_id = $1 -WHERE setting.key = 'auth.mfa_policy' -"#, - [selected_user_id.into()], - )) - .await? - .context("migrated MFA policy row is missing")?; - let platform_policy = row.try_get::("", "platform_policy")?; - ensure!(platform_policy["enforcement"] == "none"); - ensure!(platform_policy["minimum_factors"] == 0); - ensure!(platform_policy["required_factors"] == serde_json::json!([])); - ensure!(platform_policy.get("selected_user_ids").is_none()); - ensure!(!row.try_get::("", "inherit_platform")?); - ensure!(row.try_get::("", "minimum_factors")? == 1); - ensure!(row.try_get::("", "required_factors")? == serde_json::json!([])); - Ok(()) - } - - async fn assert_mfa_policy_schema_absent( - db: &DatabaseConnection, - selected_user_id: Uuid, - ) -> anyhow::Result<()> { - ensure!( - object_count( - db, - "SELECT count(*)::bigint AS count FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'user_mfa_policies'", - ) - .await? - == 0 - ); - let row = db - .query_one_raw(Statement::from_string( - DatabaseBackend::Postgres, - "SELECT value FROM system_settings WHERE key = 'auth.mfa_policy'", - )) - .await? - .context("legacy MFA policy row is missing after down migration")?; - let policy = row.try_get::("", "value")?; - ensure!(policy["enforcement"] == "selected_users"); - ensure!(policy["selected_user_ids"] == serde_json::json!([selected_user_id])); - ensure!(policy.get("minimum_factors").is_none()); - ensure!(policy.get("required_factors").is_none()); - Ok(()) - } - - async fn seed_authentication_fixture(db: &DatabaseConnection) -> anyhow::Result { - let user_id = Uuid::now_v7(); - let credential_id = Uuid::now_v7(); - db.execute_unprepared(&format!( - r#" -INSERT INTO users (id, email, display_name) -VALUES ('{user_id}'::uuid, 'authentication-migration@example.invalid', 'Authentication Migration'); - -INSERT INTO user_password_credentials (id, user_id, password_hash) -VALUES ('{credential_id}'::uuid, '{user_id}'::uuid, 'migration-password-hash'); -"# - )) - .await?; - Ok(user_id) - } - - async fn assert_authentication_schema( - db: &DatabaseConnection, - seeded_user_id: Uuid, - ) -> anyhow::Result<()> { - let columns = query_column_shapes( - db, - r#" -SELECT column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND ( - (table_name = 'users' AND column_name = 'email_verified_at') OR - (table_name = 'user_auth_tokens' AND column_name = 'used_at') OR - (table_name = 'user_mfa_factors' AND column_name IN ('verified_at', 'last_used_at')) - ) -ORDER BY table_name, ordinal_position -"#, - ) - .await?; - ensure!( - columns.len() == 4, - "missing authentication lifecycle columns" - ); - for column in columns { - ensure!( - column.udt_name == "timestamptz", - "unexpected column type: {column:?}" - ); - ensure!( - column.nullable == "YES", - "lifecycle column is not nullable: {column:?}" - ); - ensure!( - column.default.is_none(), - "lifecycle column has a default: {column:?}" - ); - } - - let enum_rows = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT t.typname, string_agg(e.enumlabel, ',' ORDER BY e.enumsortorder) AS labels -FROM pg_type t -JOIN pg_enum e ON e.enumtypid = t.oid -WHERE t.typname IN ('identity_provider_kind', 'auth_token_kind', 'mfa_factor_kind') -GROUP BY t.typname -ORDER BY t.typname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "typname")?, - row.try_get::("", "labels")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!(enum_rows.get("identity_provider_kind") == Some(&"oidc,github".to_owned())); - ensure!( - enum_rows.get("auth_token_kind") - == Some(&"email_verification,password_reset".to_owned()) - ); - ensure!(enum_rows.get("mfa_factor_kind") == Some(&"totp,email".to_owned())); - - let indexes = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT indexname, indexdef -FROM pg_indexes -WHERE schemaname = current_schema() - AND indexname IN ( - 'ix_user_external_identities_user_id', - 'ix_user_auth_tokens_live', - 'ix_user_mfa_factors_verified', - 'ix_user_password_history_recent' - ) -ORDER BY indexname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "indexname")?, - row.try_get::("", "indexdef")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!(indexes.len() == 4, "missing authentication indexes"); - ensure!(indexes["ix_user_auth_tokens_live"].contains("WHERE (used_at IS NULL)")); - ensure!( - indexes["ix_user_mfa_factors_verified"].contains("WHERE (verified_at IS NOT NULL)") - ); - - let constraints = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT conname, pg_get_constraintdef(oid) AS definition -FROM pg_constraint -WHERE conrelid IN ( - 'auth_identity_providers'::regclass, - 'user_external_identities'::regclass, - 'user_auth_tokens'::regclass, - 'user_mfa_factors'::regclass, - 'user_password_history'::regclass -) - AND contype = 'f' -ORDER BY conname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "conname")?, - row.try_get::("", "definition")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - constraints.len() == 6, - "missing authentication foreign keys" - ); - ensure!( - constraints - .values() - .filter(|definition| definition.contains("ON DELETE CASCADE")) - .count() - == 5 - ); - ensure!( - constraints - .values() - .filter(|definition| definition.contains("ON DELETE SET NULL")) - .count() - == 1 - ); - - let backfill = db - .query_one_raw(Statement::from_string( - DatabaseBackend::Postgres, - format!( - r#" -SELECT - u.email_verified_at = u.created_at AS email_backfilled, - h.password_hash -FROM users u -JOIN user_password_history h ON h.user_id = u.id -WHERE u.id = '{seeded_user_id}'::uuid -"# - ), - )) - .await? - .context("authentication backfill row is missing")?; - ensure!(backfill.try_get::("", "email_backfilled")?); - ensure!(backfill.try_get::("", "password_hash")? == "migration-password-hash"); - Ok(()) - } - - async fn assert_authentication_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> { - let row = db - .query_one_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT - to_regclass('auth_identity_providers') IS NULL AND - to_regclass('user_external_identities') IS NULL AND - to_regclass('user_auth_tokens') IS NULL AND - to_regclass('user_mfa_factors') IS NULL AND - to_regclass('user_password_history') IS NULL AS tables_absent, - NOT EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_schema = current_schema() - AND table_name = 'users' - AND column_name = 'email_verified_at' - ) AS column_absent, - NOT EXISTS ( - SELECT 1 FROM pg_type - WHERE typname IN ('identity_provider_kind', 'auth_token_kind', 'mfa_factor_kind') - ) AS types_absent -"#, - )) - .await? - .context("authentication absence query returned no row")?; - ensure!(row.try_get::("", "tables_absent")?); - ensure!(row.try_get::("", "column_absent")?); - ensure!(row.try_get::("", "types_absent")?); - Ok(()) - } - - async fn seed_project_notification_fixture( - db: &DatabaseConnection, - ) -> anyhow::Result<(Uuid, Uuid)> { - let user_id = Uuid::now_v7(); - let team_id = Uuid::now_v7(); - let project_id = Uuid::now_v7(); - let audit_id = Uuid::now_v7(); - db.execute_unprepared(&format!( - r#" -INSERT INTO users (id, email, display_name) -VALUES ('{user_id}'::uuid, 'notification-migration@example.invalid', 'Notification Migration'); - -INSERT INTO teams (id, slug, name, owner_user_id) -VALUES ('{team_id}'::uuid, 'notification-migration', 'Notification Migration', '{user_id}'::uuid); - -INSERT INTO projects (id, team_id, slug, name) -VALUES ('{project_id}'::uuid, '{team_id}'::uuid, 'notification-migration', 'Notification Migration'); - -INSERT INTO audit_events ( - id, - actor_user_id, - actor_type, - visibility, - action, - target_type, - target_id, - result, - metadata, - team_id -) -VALUES ( - '{audit_id}'::uuid, - '{user_id}'::uuid, - 'user', - 'team', - 'project.created', - 'project', - '{project_id}'::uuid, - 'success', - '{{}}'::jsonb, - '{team_id}'::uuid -); -"# - )) - .await?; - Ok((user_id, project_id)) - } - - async fn assert_project_notification_schema( - db: &DatabaseConnection, - expected_creator_id: Uuid, - project_id: Uuid, - ) -> anyhow::Result<()> { - let project_columns = query_column_shapes( - db, - r#" -SELECT column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name = 'projects' - AND column_name = 'created_by_user_id' -"#, - ) - .await?; - ensure!( - project_columns - == vec![ColumnShape { - name: "created_by_user_id".to_owned(), - udt_name: "uuid".to_owned(), - nullable: "YES".to_owned(), - default: None, - }], - "unexpected Project creator column: {project_columns:?}" - ); - - let notification_columns = query_column_shapes( - db, - r#" -SELECT column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name = 'user_notifications' -ORDER BY ordinal_position -"#, - ) - .await?; - ensure!( - notification_columns.len() == 13, - "expected 13 notification columns, found {}", - notification_columns.len() - ); - let shapes = notification_columns - .into_iter() - .map(|column| (column.name.clone(), column)) - .collect::>(); - ensure!( - shapes.get("recipient_user_id") - == Some(&ColumnShape { - name: "recipient_user_id".to_owned(), - udt_name: "uuid".to_owned(), - nullable: "NO".to_owned(), - default: None, - }) - ); - ensure!( - shapes.get("read_at") - == Some(&ColumnShape { - name: "read_at".to_owned(), - udt_name: "timestamptz".to_owned(), - nullable: "YES".to_owned(), - default: None, - }) - ); - - let constraints = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT conname, pg_get_constraintdef(oid) AS definition -FROM pg_constraint -WHERE conname IN ( - 'fk_projects_created_by_user_id', - 'fk_user_notifications_recipient_user_id', - 'fk_user_notifications_actor_user_id', - 'fk_user_notifications_team_id', - 'fk_user_notifications_project_id' -) -ORDER BY conname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "conname")?, - row.try_get::("", "definition")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!(constraints.len() == 5, "missing notification foreign keys"); - ensure!( - constraints["fk_user_notifications_recipient_user_id"].contains("ON DELETE CASCADE") - ); - for name in [ - "fk_projects_created_by_user_id", - "fk_user_notifications_actor_user_id", - "fk_user_notifications_team_id", - "fk_user_notifications_project_id", - ] { - ensure!( - constraints[name].contains("ON DELETE SET NULL"), - "{name} must preserve notification history" - ); - } - - let indexes = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT indexname, indexdef -FROM pg_indexes -WHERE schemaname = current_schema() - AND indexname IN ( - 'ix_projects_created_by_user_id', - 'ix_user_notifications_recipient_created', - 'ix_user_notifications_recipient_unread' - ) -ORDER BY indexname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "indexname")?, - row.try_get::("", "indexdef")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!(indexes.len() == 3, "missing notification indexes"); - ensure!(indexes["ix_user_notifications_recipient_unread"].contains("read_at IS NULL")); - - let row = db - .query_one_raw(Statement::from_sql_and_values( - DatabaseBackend::Postgres, - "SELECT created_by_user_id FROM projects WHERE id = $1", - [project_id.into()], - )) - .await? - .context("Project creator backfill query returned no row")?; - ensure!( - row.try_get::("", "created_by_user_id")? == expected_creator_id, - "Project creator was not backfilled from the creation audit" - ); - Ok(()) - } - - async fn assert_project_notification_schema_absent( - db: &DatabaseConnection, - ) -> anyhow::Result<()> { - let row = db - .query_one_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT - to_regclass('user_notifications') IS NULL AS notifications_absent, - NOT EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = current_schema() - AND table_name = 'projects' - AND column_name = 'created_by_user_id' - ) AS creator_absent -"#, - )) - .await? - .context("notification absence query returned no row")?; - ensure!(row.try_get::("", "notifications_absent")?); - ensure!(row.try_get::("", "creator_absent")?); - Ok(()) - } - - async fn assert_notification_content_schema(db: &DatabaseConnection) -> anyhow::Result<()> { - let columns = query_column_shapes( - db, - r#" -SELECT column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name = 'user_notifications' - AND column_name IN ('project_name', 'project_slug', 'title', 'content') -ORDER BY ordinal_position -"#, - ) - .await?; - ensure!( - columns - == vec![ - ColumnShape { - name: "project_name".to_owned(), - udt_name: "text".to_owned(), - nullable: "YES".to_owned(), - default: None, - }, - ColumnShape { - name: "project_slug".to_owned(), - udt_name: "text".to_owned(), - nullable: "YES".to_owned(), - default: None, - }, - ColumnShape { - name: "title".to_owned(), - udt_name: "text".to_owned(), - nullable: "YES".to_owned(), - default: None, - }, - ColumnShape { - name: "content".to_owned(), - udt_name: "text".to_owned(), - nullable: "YES".to_owned(), - default: None, - }, - ], - "unexpected notification content columns: {columns:?}" - ); - - let row = db - .query_one_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT pg_get_constraintdef(oid) AS definition -FROM pg_constraint -WHERE conname = 'ck_user_notifications_announcement_content' -"#, - )) - .await? - .context("announcement content constraint was not created")?; - let definition = row.try_get::("", "definition")?; - ensure!(definition.contains("site.announcement")); - ensure!(definition.contains("team_id IS NULL")); - ensure!(definition.contains("project_id IS NULL")); - Ok(()) - } - - async fn assert_announcement_schema(db: &DatabaseConnection) -> anyhow::Result<()> { - let columns = query_column_shapes( - db, - r#" -SELECT column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name = 'announcements' -ORDER BY ordinal_position -"#, - ) - .await?; - ensure!( - columns - == vec![ - column("id", "uuid", "NO", None), - column("title", "text", "NO", None), - column("content", "text", "NO", None), - column("auto_popup", "bool", "NO", Some("false")), - column("created_by_user_id", "uuid", "YES", None), - column("published_at", "timestamptz", "NO", None), - ], - "unexpected announcement columns: {columns:#?}" - ); - - let notification_columns = query_column_shapes( - db, - r#" -SELECT column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name = 'user_notifications' - AND column_name = 'announcement_id' -"#, - ) - .await?; - ensure!( - notification_columns == vec![column("announcement_id", "uuid", "YES", None)], - "unexpected notification announcement column: {notification_columns:#?}" - ); - - let constraints = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT conname, pg_get_constraintdef(oid) AS definition -FROM pg_constraint -WHERE conname IN ( - 'ck_announcements_title_length', - 'ck_announcements_content_length', - 'ck_user_notifications_announcement_content' -) -ORDER BY conname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "conname")?, - row.try_get::("", "definition")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!(constraints.len() == 3, "missing announcement constraints"); - ensure!(constraints["ck_announcements_title_length"].contains("120")); - ensure!(constraints["ck_announcements_content_length"].contains("10000")); - ensure!( - constraints["ck_user_notifications_announcement_content"] - .contains("announcement_id IS NOT NULL") - ); - - let foreign_keys = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT pg_get_constraintdef(oid) AS definition -FROM pg_constraint -WHERE contype = 'f' - AND conrelid IN ('announcements'::regclass, 'user_notifications'::regclass) -"#, - )) - .await? - .into_iter() - .map(|row| row.try_get::("", "definition")) - .collect::, sea_orm::DbErr>>()?; - ensure!( - foreign_keys - .iter() - .any(|definition| definition.contains("REFERENCES announcements") - && definition.contains("ON DELETE CASCADE")), - "notification announcement foreign key is not cascading" - ); - ensure!( - foreign_keys - .iter() - .any(|definition| definition.contains("REFERENCES users") - && definition.contains("ON DELETE SET NULL")), - "announcement creator foreign key is not nullable" - ); - - let indexes = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT indexname -FROM pg_indexes -WHERE schemaname = current_schema() - AND indexname = 'ix_announcements_published_at' -"#, - )) - .await?; - ensure!(indexes.len() == 1, "announcement history index is missing"); - Ok(()) - } - - async fn assert_announcement_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> { - let row = db - .query_one_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT - to_regclass('announcements') IS NULL AS table_absent, - NOT EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = current_schema() - AND table_name = 'user_notifications' - AND column_name = 'announcement_id' - ) AS notification_column_absent, - EXISTS ( - SELECT 1 - FROM pg_constraint - WHERE conname = 'ck_user_notifications_announcement_content' - ) AS legacy_constraint_present -"#, - )) - .await? - .context("announcement absence query returned no row")?; - ensure!(row.try_get::("", "table_absent")?); - ensure!(row.try_get::("", "notification_column_absent")?); - ensure!(row.try_get::("", "legacy_constraint_present")?); - Ok(()) - } - - async fn assert_node_deletion_schema(db: &DatabaseConnection) -> anyhow::Result<()> { - let enums = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT t.typname, string_agg(e.enumlabel, ',' ORDER BY e.enumsortorder) AS labels -FROM pg_type t -JOIN pg_enum e ON e.enumtypid = t.oid -JOIN pg_namespace n ON n.oid = t.typnamespace -WHERE n.nspname = current_schema() - AND t.typname IN ('node_deletion_status', 'node_deployment_migration_status') -GROUP BY t.typname -ORDER BY t.typname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "typname")?, - row.try_get::("", "labels")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - enums - == vec![ - ( - "node_deletion_status".to_owned(), - "queued,migrating,draining,deleting,failed,completed".to_owned(), - ), - ( - "node_deployment_migration_status".to_owned(), - "pending,syncing,ready,failed".to_owned(), - ), - ], - "node deletion enum values did not match the domain model: {enums:?}" - ); - - let columns = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT table_name, column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name IN ('node_deletion_jobs', 'node_deployment_migrations') -ORDER BY table_name, ordinal_position -"#, - )) - .await?; - ensure!( - columns.len() == 22, - "expected 22 node deletion columns, found {}", - columns.len() - ); - let shapes = columns - .into_iter() - .map(|row| { - Ok(( - format!( - "{}.{}", - row.try_get::("", "table_name")?, - row.try_get::("", "column_name")?, - ), - ( - row.try_get::("", "udt_name")?, - row.try_get::("", "is_nullable")?, - row.try_get::>("", "column_default")?, - ), - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - shapes.get("node_deletion_jobs.status") - == Some(&( - "node_deletion_status".to_owned(), - "NO".to_owned(), - Some("'queued'::node_deletion_status".to_owned()), - )) - ); - ensure!( - shapes.get("node_deletion_jobs.completed_at") - == Some(&("timestamptz".to_owned(), "YES".to_owned(), None)) - ); - ensure!( - shapes.get("node_deployment_migrations.status") - == Some(&( - "node_deployment_migration_status".to_owned(), - "NO".to_owned(), - Some("'pending'::node_deployment_migration_status".to_owned()), - )) - ); - ensure!( - shapes.get("node_deployment_migrations.ready_at") - == Some(&("timestamptz".to_owned(), "YES".to_owned(), None)) - ); - - let constraints = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT conname, pg_get_constraintdef(oid) AS definition -FROM pg_constraint -WHERE conrelid IN ('node_deletion_jobs'::regclass, 'node_deployment_migrations'::regclass) -ORDER BY conname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "conname")?, - row.try_get::("", "definition")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - for expected in [ - "ck_node_deletion_jobs_distinct_target", - "ck_node_deletion_jobs_progress_nonnegative", - "ck_node_deletion_jobs_progress_bounded", - "ck_node_deletion_jobs_completed_at", - "ck_node_deployment_migrations_distinct_nodes", - "ck_node_deployment_migrations_ready_at", - "ux_node_deployment_migrations_job_deployment", - ] { - ensure!( - constraints.contains_key(expected), - "missing constraint {expected}" - ); - } - ensure!( - constraints.values().any(|definition| definition - .contains("FOREIGN KEY (target_node_id)") - && definition.contains("REFERENCES nodes(id) ON DELETE RESTRICT")), - "target Node foreign keys must prevent deleting an active migration target" - ); - ensure!( - constraints.values().any(|definition| definition - .contains("FOREIGN KEY (requested_by_user_id)") - && definition.contains("ON DELETE SET NULL")), - "requester foreign key must preserve deletion history" - ); - - let indexes = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT indexname, indexdef -FROM pg_indexes -WHERE schemaname = current_schema() - AND indexname IN ( - 'ux_node_deletion_jobs_active_node', - 'ix_node_deletion_jobs_queue', - 'ix_node_deployment_migrations_target' - ) -ORDER BY indexname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "indexname")?, - row.try_get::("", "indexdef")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!(indexes.len() == 3, "expected three queue indexes"); - ensure!( - indexes["ux_node_deletion_jobs_active_node"].contains("UNIQUE INDEX") - && indexes["ux_node_deletion_jobs_active_node"] - .contains("status <> 'completed'::node_deletion_status") - ); - ensure!( - indexes["ix_node_deployment_migrations_target"] - .contains("'ready'::node_deployment_migration_status") - ); - Ok(()) - } - - async fn assert_node_deletion_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> { - let row = db - .query_one_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT - to_regclass('node_deletion_jobs') IS NULL AS jobs_absent, - to_regclass('node_deployment_migrations') IS NULL AS migrations_absent, - NOT EXISTS ( - SELECT 1 FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace - WHERE n.nspname = current_schema() - AND t.typname IN ('node_deletion_status', 'node_deployment_migration_status') - ) AS enums_absent -"#, - )) - .await? - .context("node deletion absence query returned no row")?; - ensure!(row.try_get::("", "jobs_absent")?); - ensure!(row.try_get::("", "migrations_absent")?); - ensure!(row.try_get::("", "enums_absent")?); - Ok(()) - } - - async fn verify_audit_foundation_migration(db: &DatabaseConnection) -> anyhow::Result<()> { - Migrator::up(db, Some(11)).await?; - assert_migration_tracking(db, 11).await?; - - let user_id = Uuid::now_v7(); - let team_id = Uuid::now_v7(); - let project_id = Uuid::now_v7(); - let deployment_id = Uuid::now_v7(); - seed_v11_audit_fixtures(db, user_id, team_id, project_id, deployment_id).await?; - - Migrator::up(db, Some(1)).await?; - assert_migration_tracking(db, 12).await?; - assert_audit_enum_shapes(db).await?; - assert_audit_column_shapes(db).await?; - assert_audit_constraints(db).await?; - assert_audit_indexes(db).await?; - assert_audit_backfill(db, deployment_id).await?; - - Migrator::down(db, Some(1)).await?; - assert_migration_tracking(db, 11).await?; - assert_audit_foundation_objects_absent(db).await?; - - Migrator::up(db, None).await?; - assert_migration_tracking(db, 35).await?; - assert_audit_foundation_objects_restored(db).await?; - - Ok(()) - } - - async fn seed_v11_audit_fixtures( - db: &DatabaseConnection, - user_id: Uuid, - team_id: Uuid, - project_id: Uuid, - deployment_id: Uuid, - ) -> anyhow::Result<()> { - db.execute_unprepared(&format!( - r#" -INSERT INTO users (id, email, display_name) -VALUES ('{user_id}'::uuid, 'audit-migration@example.invalid', 'Audit Migration'); - -INSERT INTO teams (id, slug, name, owner_user_id) -VALUES ('{team_id}'::uuid, 'audit-migration', 'Audit Migration', '{user_id}'::uuid); - -INSERT INTO projects (id, team_id, slug, name) -VALUES ('{project_id}'::uuid, '{team_id}'::uuid, 'audit-migration', 'Audit Migration'); - -INSERT INTO audit_events (actor_user_id, action, target_type, metadata, team_id) -VALUES - ('{user_id}'::uuid, 'project.updated', 'project', '{{}}'::jsonb, '{team_id}'::uuid), - (NULL, 'project.deleted', 'project', '{{"platform_admin": true}}'::jsonb, '{team_id}'::uuid), - ('{user_id}'::uuid, 'deployment.release.completed', 'deployment', '{{"completed_after_sync": true}}'::jsonb, '{team_id}'::uuid), - (NULL, 'team.quota_plan_overridden', 'team', '{{}}'::jsonb, '{team_id}'::uuid); - -INSERT INTO deployments ( - id, - project_id, - team_id, - pending_release_reason, - pending_release_actor_user_id, - pending_release_requested_at -) -VALUES ( - '{deployment_id}'::uuid, - '{project_id}'::uuid, - '{team_id}'::uuid, - 'rollback', - '{user_id}'::uuid, - CURRENT_TIMESTAMP -); -"# - )) - .await?; - - Ok(()) - } - - async fn assert_migration_tracking( - db: &DatabaseConnection, - applied_count: usize, - ) -> anyhow::Result<()> { - let applied = Migrator::get_applied_migrations(db).await?; - let pending = Migrator::get_pending_migrations(db).await?; - // Historical shape tests stop at their target migration. Later migrations - // remain pending even when that historical phase is fully applied. - let pending_count = Migrator::migrations().len() - applied_count; - - ensure!( - applied.len() == applied_count, - "expected {applied_count} applied migrations, found {}", - applied.len() - ); - ensure!( - pending.len() == pending_count, - "expected {pending_count} pending migrations, found {}", - pending.len() - ); - if applied_count >= 12 { - ensure!( - applied.get(11).map(|migration| migration.name()) - == Some("m20260729_000012_audit_foundation"), - "audit foundation migration was not the twelfth applied migration" - ); - } - if pending_count > 0 { - let expected = Migrator::migrations() - .get(applied_count) - .map(|migration| migration.name().to_owned()); - ensure!( - pending.first().map(|migration| migration.name()) == expected.as_deref(), - "migration tracking did not expose the next registered migration first" - ); - } - - Ok(()) - } - - async fn assert_avatar_schema(db: &DatabaseConnection) -> anyhow::Result<()> { - let rows = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT table_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND column_name = 'avatar_version' - AND table_name IN ('teams', 'users') -ORDER BY table_name -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "table_name")?, - row.try_get::("", "udt_name")?, - row.try_get::("", "is_nullable")?, - row.try_get::>("", "column_default")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - rows == vec![ - ( - "teams".to_owned(), - "uuid".to_owned(), - "YES".to_owned(), - None, - ), - ( - "users".to_owned(), - "uuid".to_owned(), - "YES".to_owned(), - None, - ), - ], - "unexpected avatar columns: {rows:#?}" - ); - Ok(()) - } - - async fn assert_screenshot_schema(db: &DatabaseConnection) -> anyhow::Result<()> { - let enum_rows = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT t.typname, string_agg(e.enumlabel, ',' ORDER BY e.enumsortorder) AS labels -FROM pg_type t -JOIN pg_enum e ON e.enumtypid = t.oid -JOIN pg_namespace n ON n.oid = t.typnamespace -WHERE n.nspname = current_schema() - AND t.typname IN ('deployment_artifact_kind', 'deployment_screenshot_status') -GROUP BY t.typname -ORDER BY t.typname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "typname")?, - row.try_get::("", "labels")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - enum_rows - == vec![ - ( - "deployment_artifact_kind".to_owned(), - "grass_output,build_log,static_site,screenshot".to_owned(), - ), - ( - "deployment_screenshot_status".to_owned(), - "pending,running,succeeded,failed".to_owned(), - ), - ], - "unexpected screenshot enum values: {enum_rows:#?}" - ); - - let columns = query_column_shapes( - db, - r#" -SELECT column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name = 'deployment_screenshot_jobs' -ORDER BY ordinal_position -"#, - ) - .await?; - ensure!( - columns.len() == 8 - && columns[0] == column("deployment_id", "uuid", "NO", None) - && columns[1].name == "status" - && columns[1].udt_name == "deployment_screenshot_status" - && columns[1].nullable == "NO" - && columns[1] - .default - .as_deref() - .is_some_and(|value| value.contains("'pending'")) - && columns[2].name == "attempt_count" - && columns[2].udt_name == "int4" - && columns[2].nullable == "NO" - && columns[2].default.as_deref() == Some("0") - && columns[3] == column("next_attempt_at", "timestamptz", "NO", None) - && columns[4] == column("last_error", "text", "YES", None) - && columns[5] == column("artifact_id", "uuid", "YES", None) - && columns[6] == column("created_at", "timestamptz", "NO", None) - && columns[7] == column("updated_at", "timestamptz", "NO", None), - "unexpected screenshot job columns: {columns:#?}" - ); - - let constraints = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT conname, pg_get_constraintdef(oid) AS definition -FROM pg_constraint -WHERE conrelid = 'deployment_screenshot_jobs'::regclass -ORDER BY conname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "conname")?, - row.try_get::("", "definition")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - constraints - .get("deployment_screenshot_jobs_deployment_id_fkey") - .is_some_and(|value| value.contains("ON DELETE CASCADE")), - "screenshot deployment foreign key must cascade" - ); - ensure!( - constraints - .get("deployment_screenshot_jobs_artifact_id_fkey") - .is_some_and(|value| value.contains("ON DELETE CASCADE")), - "screenshot artifact foreign key must cascade" - ); - ensure!( - constraints - .get("ck_deployment_screenshot_attempt_count") - .is_some_and(|value| value.contains("attempt_count <= 4")), - "screenshot attempt constraint is missing" - ); - ensure!( - constraints - .get("ck_deployment_screenshot_artifact") - .is_some_and(|value| value.contains("artifact_id IS NOT NULL")), - "screenshot artifact state constraint is missing" - ); - - let index = db - .query_one_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT indexdef -FROM pg_indexes -WHERE schemaname = current_schema() - AND indexname = 'ix_deployment_screenshot_jobs_due' -"#, - )) - .await? - .context("screenshot due-job index is missing")?; - let index = index.try_get::("", "indexdef")?; - ensure!( - index.contains("next_attempt_at, deployment_id") - && index.contains("status = 'pending'"), - "unexpected screenshot due-job index: {index}" - ); - Ok(()) - } - - async fn assert_object_storage_schema(db: &DatabaseConnection) -> anyhow::Result<()> { - let table_count = object_count( - db, - r#" -SELECT count(*)::bigint AS count -FROM information_schema.tables -WHERE table_schema = current_schema() - AND table_name IN ('storage_migration_jobs', 'storage_migration_objects') -"#, - ) - .await?; - ensure!( - table_count == 2, - "object storage migration tables are incomplete" - ); - - let enum_count = object_count( - db, - r#" -SELECT count(*)::bigint AS count -FROM pg_type t -JOIN pg_namespace n ON n.oid = t.typnamespace -WHERE n.nspname = current_schema() - AND t.typname IN ('storage_migration_status', 'storage_migration_object_status') -"#, - ) - .await?; - ensure!( - enum_count == 2, - "object storage migration enums are incomplete" - ); - - let index_count = object_count( - db, - r#" -SELECT count(*)::bigint AS count -FROM pg_indexes -WHERE schemaname = current_schema() - AND indexname IN ('ux_storage_migration_jobs_active', 'ix_storage_migration_objects_due') -"#, - ) - .await?; - ensure!( - index_count == 2, - "object storage migration indexes are incomplete" - ); - Ok(()) - } - - async fn assert_object_storage_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> { - ensure!( - object_count( - db, - r#" -SELECT count(*)::bigint AS count -FROM information_schema.tables -WHERE table_schema = current_schema() - AND table_name IN ('storage_migration_jobs', 'storage_migration_objects') -"#, - ) - .await? - == 0 - ); - ensure!( - object_count( - db, - r#" -SELECT count(*)::bigint AS count -FROM pg_type t -JOIN pg_namespace n ON n.oid = t.typnamespace -WHERE n.nspname = current_schema() - AND t.typname IN ('storage_migration_status', 'storage_migration_object_status') -"#, - ) - .await? - == 0 - ); - ensure!( - object_count( - db, - r#" -SELECT count(*)::bigint AS count -FROM pg_indexes -WHERE schemaname = current_schema() - AND indexname IN ('ux_storage_migration_jobs_active', 'ix_storage_migration_objects_due') -"#, - ) - .await? - == 0 - ); - Ok(()) - } - - async fn assert_regional_ingress_schema(db: &DatabaseConnection) -> anyhow::Result<()> { - let columns = query_column_shapes( - db, - r#" -SELECT column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name = 'regional_ingresses' -ORDER BY ordinal_position -"#, - ) - .await?; - ensure!( - columns - == vec![ - column("id", "uuid", "NO", None), - column("region", "text", "NO", None), - column("hostname", "varchar", "NO", None), - column("enabled", "bool", "NO", Some("true")), - column("health_check_path", "text", "NO", Some("'/health'::text")), - column("health_check_interval_seconds", "int4", "NO", Some("30")), - column("origin_host_preservation", "bool", "NO", Some("true")), - column("tls_enabled", "bool", "NO", Some("true")), - column( - "certificate_issuer", - "text", - "NO", - Some("'letsencrypt'::text") - ), - column("certificate_auto_renew", "bool", "NO", Some("true")), - column("certificate_status", "text", "NO", Some("'pending'::text")), - column("certificate_expires_at", "timestamptz", "YES", None), - column("certificate_error", "text", "YES", None), - column("dns_challenge_provider", "text", "YES", None), - column("dns_challenge_config", "jsonb", "NO", Some("'{}'::jsonb")), - column( - "dns_challenge_status", - "text", - "NO", - Some("'not_configured'::text") - ), - column("dns_challenge_record_name", "text", "YES", None), - column("dns_challenge_record_value", "text", "YES", None), - column("deleted_at", "timestamptz", "YES", None), - column("created_at", "timestamptz", "NO", None), - column("updated_at", "timestamptz", "NO", None), - column("acme_account", "jsonb", "YES", None), - column("certificate_bundle", "jsonb", "YES", None), - column("certificate_issued_at", "timestamptz", "YES", None), - ], - "unexpected regional_ingresses column shapes: {columns:#?}" - ); - - let constraints = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT conname, pg_get_constraintdef(oid) AS definition -FROM pg_constraint -WHERE conrelid = 'regional_ingresses'::regclass - AND conname LIKE 'ck_regional_ingresses_%' -ORDER BY conname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "conname")?, - row.try_get::("", "definition")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - constraints.len() == 7, - "unexpected ingress constraints: {constraints:#?}" - ); - ensure!(constraints["ck_regional_ingresses_region_nonempty"].contains("char_length")); - ensure!(constraints["ck_regional_ingresses_hostname_nonempty"].contains("char_length")); - // PostgreSQL deparses LIKE and BETWEEN into their underlying operators. - ensure!( - constraints["ck_regional_ingresses_health_path"] - .contains("health_check_path ~~ '/%'::text") - ); - ensure!( - constraints["ck_regional_ingresses_health_interval"] - .contains("health_check_interval_seconds >= 5") - && constraints["ck_regional_ingresses_health_interval"] - .contains("health_check_interval_seconds <= 3600") - ); - ensure!(constraints["ck_regional_ingresses_certificate_issuer"].contains("letsencrypt")); - ensure!( - constraints["ck_regional_ingresses_certificate_status"].contains("certificate_status") - ); - ensure!( - constraints["ck_regional_ingresses_dns_challenge_status"].contains("not_configured") - ); - - let indexes = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT indexname, indexdef -FROM pg_indexes -WHERE schemaname = current_schema() - AND indexname IN ( - 'ux_regional_ingresses_region_active', - 'ux_regional_ingresses_hostname_active', - 'ix_regional_ingresses_enabled', - 'ix_host_sources_region', - 'ix_project_host_bindings_region' - ) -ORDER BY indexname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "indexname")?, - row.try_get::("", "indexdef")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - indexes.len() == 5, - "unexpected ingress indexes: {indexes:#?}" - ); - ensure!(indexes["ux_regional_ingresses_region_active"].contains("UNIQUE")); - ensure!(indexes["ux_regional_ingresses_region_active"].contains("deleted_at IS NULL")); - ensure!(indexes["ux_regional_ingresses_hostname_active"].contains("UNIQUE")); - ensure!(indexes["ix_regional_ingresses_enabled"].contains("(region, enabled)")); - ensure!(indexes["ix_host_sources_region"].contains("(region)")); - ensure!(indexes["ix_project_host_bindings_region"].contains("(region)")); - ensure!( - object_count( - db, - "SELECT count(*)::bigint AS count FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'regional_ingress_health'", - ) - .await? - == 1, - "regional ingress health table is missing" - ); - Ok(()) - } - - async fn assert_regional_ingress_lifecycle_absent( - db: &DatabaseConnection, - ) -> anyhow::Result<()> { - ensure!( - object_count( - db, - "SELECT count(*)::bigint AS count FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'regional_ingress_health'", - ) - .await? - == 0, - "regional ingress health table remained after lifecycle down migration" - ); - ensure!( - object_count( - db, - "SELECT count(*)::bigint AS count FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'regional_ingresses' AND column_name IN ('acme_account', 'certificate_bundle', 'certificate_issued_at')", - ) - .await? - == 0, - "regional ingress lifecycle columns remained after lifecycle down migration" - ); - Ok(()) - } - - async fn assert_regional_ingress_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> { - ensure!( - object_count( - db, - r#" -SELECT count(*)::bigint AS count -FROM information_schema.tables -WHERE table_schema = current_schema() - AND table_name = 'regional_ingresses' -"#, - ) - .await? - == 0, - "regional_ingresses table remained after down migration" - ); - ensure!( - object_count( - db, - r#" -SELECT count(*)::bigint AS count -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name IN ('host_sources', 'project_host_bindings') - AND column_name = 'region' -"#, - ) - .await? - == 0, - "regional host columns remained after down migration" - ); - ensure!( - object_count( - db, - r#" -SELECT count(*)::bigint AS count -FROM pg_indexes -WHERE schemaname = current_schema() - AND indexname IN ( - 'ux_regional_ingresses_region_active', - 'ux_regional_ingresses_hostname_active', - 'ix_regional_ingresses_enabled', - 'ix_host_sources_region', - 'ix_project_host_bindings_region' - ) -"#, - ) - .await? - == 0, - "regional ingress indexes remained after down migration" - ); - Ok(()) - } - - async fn assert_registration_allowlist_schema(db: &DatabaseConnection) -> anyhow::Result<()> { - let columns = query_column_shapes( - db, - r#" -SELECT column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name = 'registration_email_allowlist' -ORDER BY ordinal_position -"#, - ) - .await?; - ensure!( - columns - == vec![ - column("id", "uuid", "NO", None), - column("email", "text", "NO", None), - column("created_by_user_id", "uuid", "YES", None), - column("created_at", "timestamptz", "NO", None), - ], - "unexpected registration allowlist columns: {columns:#?}" - ); - - let definitions = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT conname, pg_get_constraintdef(oid) AS definition -FROM pg_constraint -WHERE conrelid = 'registration_email_allowlist'::regclass -ORDER BY conname -"#, - )) - .await? - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "conname")?, - row.try_get::("", "definition")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - definitions - .get("registration_email_allowlist_email_key") - .is_some_and(|definition| definition.starts_with("UNIQUE (email)")), - "registration allowlist email unique constraint is missing" - ); - ensure!( - definitions - .get("registration_email_allowlist_created_by_user_id_fkey") - .is_some_and(|definition| definition.contains("ON DELETE SET NULL")), - "registration allowlist creator foreign key is missing" - ); - ensure!( - definitions - .get("ck_registration_email_allowlist_email") - .is_some_and(|definition| definition.contains("email = lower(email)")), - "registration allowlist normalization check is missing" - ); - - let index = db - .query_one_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT indexdef -FROM pg_indexes -WHERE schemaname = current_schema() - AND indexname = 'ix_registration_email_allowlist_created' -"#, - )) - .await? - .context("registration allowlist creation index is missing")?; - let index_definition = index.try_get::("", "indexdef")?; - ensure!( - index_definition.contains("created_at DESC, id DESC"), - "unexpected registration allowlist index: {index_definition}" - ); - Ok(()) - } - - async fn assert_registration_allowlist_schema_absent( - db: &DatabaseConnection, - ) -> anyhow::Result<()> { - let row = db - .query_one_raw(Statement::from_string( - DatabaseBackend::Postgres, - "SELECT to_regclass('registration_email_allowlist') IS NULL AS absent", - )) - .await? - .context("registration allowlist absence query returned no row")?; - ensure!(row.try_get::("", "absent")?); - Ok(()) - } - - async fn assert_audit_enum_shapes(db: &DatabaseConnection) -> anyhow::Result<()> { - let rows = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT t.typname, string_agg(e.enumlabel, ',' ORDER BY e.enumsortorder) AS labels -FROM pg_type t -JOIN pg_enum e ON e.enumtypid = t.oid -JOIN pg_namespace n ON n.oid = t.typnamespace -WHERE n.nspname = current_schema() - AND t.typname IN ('audit_actor_type', 'audit_event_visibility') -GROUP BY t.typname -ORDER BY t.typname -"#, - )) - .await?; - let enum_shapes = rows - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "typname")?, - row.try_get::("", "labels")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - - ensure!( - enum_shapes - == vec![ - ( - "audit_actor_type".to_owned(), - "anonymous,user,system,node".to_owned(), - ), - ( - "audit_event_visibility".to_owned(), - "platform,team".to_owned(), - ), - ], - "unexpected audit enum shapes: {enum_shapes:?}" - ); - Ok(()) - } - - async fn assert_audit_column_shapes(db: &DatabaseConnection) -> anyhow::Result<()> { - let audit_columns = query_column_shapes( - db, - r#" -SELECT column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name = 'audit_events' - AND column_name IN ( - 'actor_type', - 'actor_node_id', - 'visibility', - 'request_id', - 'source_ip', - 'user_agent', - 'http_method', - 'request_path', - 'status_code', - 'duration_ms', - 'changes' - ) -ORDER BY column_name -"#, - ) - .await?; - ensure!( - audit_columns - == vec![ - column("actor_node_id", "uuid", "YES", None), - column( - "actor_type", - "audit_actor_type", - "NO", - Some("'system'::audit_actor_type"), - ), - column("changes", "jsonb", "NO", Some("'{}'::jsonb")), - column("duration_ms", "int8", "YES", None), - column("http_method", "text", "YES", None), - column("request_id", "uuid", "YES", None), - column("request_path", "text", "YES", None), - column("source_ip", "text", "YES", None), - column("status_code", "int4", "YES", None), - column("user_agent", "text", "YES", None), - column( - "visibility", - "audit_event_visibility", - "NO", - Some("'platform'::audit_event_visibility"), - ), - ], - "unexpected audit_events column shapes: {audit_columns:#?}" - ); - - let deployment_columns = query_column_shapes( - db, - r#" -SELECT column_name, udt_name, is_nullable, column_default -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name = 'deployments' - AND column_name = 'pending_release_audit_visibility' -"#, - ) - .await?; - ensure!( - deployment_columns - == vec![column( - "pending_release_audit_visibility", - "audit_event_visibility", - "YES", - None, - )], - "unexpected deployment provenance column shape: {deployment_columns:#?}" - ); - - Ok(()) - } - - fn column(name: &str, udt_name: &str, nullable: &str, default: Option<&str>) -> ColumnShape { - ColumnShape { - name: name.to_owned(), - udt_name: udt_name.to_owned(), - nullable: nullable.to_owned(), - default: default.map(str::to_owned), - } - } - - async fn query_column_shapes( - db: &DatabaseConnection, - sql: &str, - ) -> anyhow::Result> { - db.query_all_raw(Statement::from_string(DatabaseBackend::Postgres, sql)) - .await? - .into_iter() - .map(|row| { - Ok(ColumnShape { - name: row.try_get::("", "column_name")?, - udt_name: row.try_get::("", "udt_name")?, - nullable: row.try_get::("", "is_nullable")?, - default: row.try_get::>("", "column_default")?, - }) - }) - .collect::, sea_orm::DbErr>>() - .map_err(Into::into) - } - - async fn assert_audit_constraints(db: &DatabaseConnection) -> anyhow::Result<()> { - let rows = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT conname, contype::text AS constraint_type, pg_get_constraintdef(oid) AS definition -FROM pg_constraint -WHERE conrelid = 'audit_events'::regclass - AND conname IN ( - 'fk_audit_events_actor_node_id', - 'ck_audit_events_actor_identity', - 'ck_audit_events_status_code', - 'ck_audit_events_duration_ms' - ) -ORDER BY conname -"#, - )) - .await?; - let constraints = rows - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "conname")?, - ( - row.try_get::("", "constraint_type")?, - row.try_get::("", "definition")?, - ), - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - constraints.len() == 4, - "expected four audit constraints, found {constraints:#?}" - ); - - let (kind, definition) = constraints - .get("fk_audit_events_actor_node_id") - .context("missing actor node foreign key")?; - ensure!(kind == "f", "actor node constraint is not a foreign key"); - ensure!( - definition.contains("FOREIGN KEY (actor_node_id)") - && definition.contains("REFERENCES nodes(id)") - && definition.contains("ON DELETE SET NULL"), - "unexpected actor node foreign key: {definition}" - ); - - let (kind, definition) = constraints - .get("ck_audit_events_actor_identity") - .context("missing actor identity constraint")?; - ensure!(kind == "c", "actor identity constraint is not a check"); - ensure!( - definition.contains("actor_user_id IS NULL") - && definition.contains("actor_type = 'user'") - && definition.contains("actor_node_id IS NULL") - && definition.contains("actor_type = 'node'") - && definition.contains("actor_type <> ALL"), - "unexpected actor identity check: {definition}" - ); - - let (kind, definition) = constraints - .get("ck_audit_events_status_code") - .context("missing status code constraint")?; - ensure!(kind == "c", "status code constraint is not a check"); - ensure!( - definition.contains("status_code IS NULL") - && definition.contains("status_code >= 100") - && definition.contains("status_code <= 599"), - "unexpected status code check: {definition}" - ); - - let (kind, definition) = constraints - .get("ck_audit_events_duration_ms") - .context("missing duration constraint")?; - ensure!(kind == "c", "duration constraint is not a check"); - ensure!( - definition.contains("duration_ms IS NULL") && definition.contains("duration_ms >= 0"), - "unexpected duration check: {definition}" - ); - - let deployment_constraint = db - .query_one_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT contype::text AS constraint_type, pg_get_constraintdef(oid) AS definition -FROM pg_constraint -WHERE conrelid = 'deployments'::regclass - AND conname = 'ck_deployments_pending_release_audit_visibility' -"#, - )) - .await? - .context("missing pending release audit visibility constraint")?; - let kind = deployment_constraint.try_get::("", "constraint_type")?; - let definition = deployment_constraint.try_get::("", "definition")?; - ensure!(kind == "c", "pending release constraint is not a check"); - ensure!( - definition.contains("pending_release_reason IS NULL") - && definition.contains("pending_release_audit_visibility IS NULL"), - "unexpected pending release audit visibility check: {definition}" - ); - - Ok(()) - } - - async fn assert_audit_indexes(db: &DatabaseConnection) -> anyhow::Result<()> { - let rows = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT indexname, indexdef -FROM pg_indexes -WHERE schemaname = current_schema() - AND tablename = 'audit_events' - AND indexname IN ( - 'ux_audit_events_request_id', - 'ix_audit_events_visibility_created_at', - 'ix_audit_events_actor_created_at', - 'ix_audit_events_actor_node_created_at', - 'ix_audit_events_created_at' - ) -ORDER BY indexname -"#, - )) - .await?; - let indexes = rows - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "indexname")?, - row.try_get::("", "indexdef")?, - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - indexes.len() == 5, - "expected five audit indexes, found {indexes:#?}" - ); - - let request = indexes - .get("ux_audit_events_request_id") - .context("missing request id index")?; - ensure!( - request.contains("CREATE UNIQUE INDEX") - && request.contains("(request_id)") - && request.contains("request_id IS NOT NULL"), - "unexpected request id index: {request}" - ); - ensure_index( - &indexes, - "ix_audit_events_visibility_created_at", - "(visibility, created_at DESC)", - None, - )?; - ensure_index( - &indexes, - "ix_audit_events_actor_created_at", - "(actor_user_id, created_at DESC)", - Some("actor_user_id IS NOT NULL"), - )?; - ensure_index( - &indexes, - "ix_audit_events_actor_node_created_at", - "(actor_node_id, created_at DESC)", - Some("actor_node_id IS NOT NULL"), - )?; - ensure_index(&indexes, "ix_audit_events_created_at", "(created_at)", None)?; - - Ok(()) - } - - fn ensure_index( - indexes: &BTreeMap, - name: &str, - columns: &str, - predicate: Option<&str>, - ) -> anyhow::Result<()> { - let definition = indexes - .get(name) - .with_context(|| format!("missing index {name}"))?; - ensure!( - definition.contains(columns), - "index {name} has unexpected columns: {definition}" - ); - if let Some(predicate) = predicate { - ensure!( - definition.contains(predicate), - "index {name} has unexpected predicate: {definition}" - ); - } - Ok(()) - } - - async fn assert_audit_backfill( - db: &DatabaseConnection, - deployment_id: Uuid, - ) -> anyhow::Result<()> { - let rows = db - .query_all_raw(Statement::from_string( - DatabaseBackend::Postgres, - r#" -SELECT action, actor_type::text AS actor_type, visibility::text AS visibility -FROM audit_events -WHERE action IN ( - 'project.updated', - 'project.deleted', - 'deployment.release.completed', - 'team.quota_plan_overridden' -) -ORDER BY action -"#, - )) - .await?; - let backfilled = rows - .into_iter() - .map(|row| { - Ok(( - row.try_get::("", "action")?, - ( - row.try_get::("", "actor_type")?, - row.try_get::("", "visibility")?, - ), - )) - }) - .collect::, sea_orm::DbErr>>()?; - ensure!( - backfilled - == BTreeMap::from([ - ( - "deployment.release.completed".to_owned(), - ("user".to_owned(), "platform".to_owned()), - ), - ( - "project.deleted".to_owned(), - ("system".to_owned(), "platform".to_owned()), - ), - ( - "project.updated".to_owned(), - ("user".to_owned(), "team".to_owned()), - ), - ( - "team.quota_plan_overridden".to_owned(), - ("system".to_owned(), "platform".to_owned()), - ), - ]), - "unexpected audit backfill: {backfilled:#?}" - ); - - let deployment_visibility = db - .query_one_raw(Statement::from_string( - DatabaseBackend::Postgres, - format!( - "SELECT pending_release_audit_visibility::text AS visibility FROM deployments WHERE id = '{deployment_id}'::uuid" - ), - )) - .await? - .context("seeded pending release deployment is missing")? - .try_get::("", "visibility")?; - ensure!( - deployment_visibility == "platform", - "pending release backfilled to {deployment_visibility:?} instead of platform" - ); - - Ok(()) - } - - async fn assert_audit_foundation_objects_absent(db: &DatabaseConnection) -> anyhow::Result<()> { - let column_count = object_count( - db, - r#" -SELECT count(*)::bigint AS count -FROM information_schema.columns -WHERE table_schema = current_schema() - AND ( - (table_name = 'audit_events' AND column_name IN ( - 'actor_type', 'actor_node_id', 'visibility', 'request_id', 'source_ip', - 'user_agent', 'http_method', 'request_path', 'status_code', 'duration_ms', 'changes' - )) - OR (table_name = 'deployments' AND column_name = 'pending_release_audit_visibility') - ) -"#, - ) - .await?; - ensure!( - column_count == 0, - "audit foundation columns remained after down migration" - ); - - let enum_count = audit_enum_count(db).await?; - ensure!( - enum_count == 0, - "audit foundation enum types remained after down migration" - ); - Ok(()) - } - - async fn assert_audit_foundation_objects_restored( - db: &DatabaseConnection, - ) -> anyhow::Result<()> { - let column_count = object_count( - db, - r#" -SELECT count(*)::bigint AS count -FROM information_schema.columns -WHERE table_schema = current_schema() - AND ( - (table_name = 'audit_events' AND column_name IN ( - 'actor_type', 'actor_node_id', 'visibility', 'request_id', 'source_ip', - 'user_agent', 'http_method', 'request_path', 'status_code', 'duration_ms', 'changes' - )) - OR (table_name = 'deployments' AND column_name = 'pending_release_audit_visibility') - ) -"#, - ) - .await?; - ensure!( - column_count == 12, - "audit foundation columns were not restored after reapplying migration" - ); - - let enum_count = audit_enum_count(db).await?; - ensure!( - enum_count == 2, - "audit foundation enum types were not restored after reapplying migration" - ); - Ok(()) - } - - async fn audit_enum_count(db: &DatabaseConnection) -> anyhow::Result { - object_count( - db, - r#" -SELECT count(*)::bigint AS count -FROM pg_type t -JOIN pg_namespace n ON n.oid = t.typnamespace -WHERE n.nspname = current_schema() - AND t.typname IN ('audit_actor_type', 'audit_event_visibility') -"#, - ) - .await - } - - async fn object_count(db: &DatabaseConnection, sql: &str) -> anyhow::Result { - db.query_one_raw(Statement::from_string(DatabaseBackend::Postgres, sql)) - .await? - .context("count query returned no row")? - .try_get::("", "count") - .map_err(Into::into) - } -} +mod tests; diff --git a/apps/control-api/src/infra/database/migrate/tests/allowlist.rs b/apps/control-api/src/infra/database/migrate/tests/allowlist.rs new file mode 100644 index 0000000..f7dd291 --- /dev/null +++ b/apps/control-api/src/infra/database/migrate/tests/allowlist.rs @@ -0,0 +1,139 @@ +use std::collections::BTreeMap; + +use anyhow::{Context, ensure}; +use sea_orm::{ConnectionTrait, DatabaseBackend, DatabaseConnection, Statement}; +use sea_orm_migration::MigratorTrait; + +use super::super::{MIGRATION_TEST_LOCK, Migrator}; +use super::support::{ + PostgresMigrationDatabase, assert_migration_tracking, column, query_column_shapes, +}; + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL"] +async fn postgres_registration_allowlist_schema_matches_domain_and_is_reversible() +-> anyhow::Result<()> { + let _migration_guard = MIGRATION_TEST_LOCK.lock().await; + let database_url = std::env::var("GRASS_TEST_DATABASE_URL") + .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); + let test_db = PostgresMigrationDatabase::start(&database_url).await?; + + let verification = async { + Migrator::up(&test_db.db, Some(25)).await?; + assert_migration_tracking(&test_db.db, 25).await?; + assert_registration_allowlist_schema(&test_db.db).await?; + + Migrator::down(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 24).await?; + assert_registration_allowlist_schema_absent(&test_db.db).await?; + + Migrator::up(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 25).await?; + assert_registration_allowlist_schema(&test_db.db).await + } + .await; + let cleanup = test_db.cleanup().await; + + match (verification, cleanup) { + (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context(format!( + "disposable schema cleanup also failed: {cleanup_error:#}" + ))), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Ok(()), Ok(())) => Ok(()), + } +} + +async fn assert_registration_allowlist_schema(db: &DatabaseConnection) -> anyhow::Result<()> { + let columns = query_column_shapes( + db, + r#" +SELECT column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'registration_email_allowlist' +ORDER BY ordinal_position +"#, + ) + .await?; + ensure!( + columns + == vec![ + column("id", "uuid", "NO", None), + column("email", "text", "NO", None), + column("created_by_user_id", "uuid", "YES", None), + column("created_at", "timestamptz", "NO", None), + ], + "unexpected registration allowlist columns: {columns:#?}" + ); + + let definitions = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT conname, pg_get_constraintdef(oid) AS definition +FROM pg_constraint +WHERE conrelid = 'registration_email_allowlist'::regclass +ORDER BY conname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "conname")?, + row.try_get::("", "definition")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + definitions + .get("registration_email_allowlist_email_key") + .is_some_and(|definition| definition.starts_with("UNIQUE (email)")), + "registration allowlist email unique constraint is missing" + ); + ensure!( + definitions + .get("registration_email_allowlist_created_by_user_id_fkey") + .is_some_and(|definition| definition.contains("ON DELETE SET NULL")), + "registration allowlist creator foreign key is missing" + ); + ensure!( + definitions + .get("ck_registration_email_allowlist_email") + .is_some_and(|definition| definition.contains("email = lower(email)")), + "registration allowlist normalization check is missing" + ); + + let index = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT indexdef +FROM pg_indexes +WHERE schemaname = current_schema() + AND indexname = 'ix_registration_email_allowlist_created' +"#, + )) + .await? + .context("registration allowlist creation index is missing")?; + let index_definition = index.try_get::("", "indexdef")?; + ensure!( + index_definition.contains("created_at DESC, id DESC"), + "unexpected registration allowlist index: {index_definition}" + ); + Ok(()) +} + +async fn assert_registration_allowlist_schema_absent( + db: &DatabaseConnection, +) -> anyhow::Result<()> { + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + "SELECT to_regclass('registration_email_allowlist') IS NULL AS absent", + )) + .await? + .context("registration allowlist absence query returned no row")?; + ensure!(row.try_get::("", "absent")?); + Ok(()) +} diff --git a/apps/control-api/src/infra/database/migrate/tests/audit.rs b/apps/control-api/src/infra/database/migrate/tests/audit.rs new file mode 100644 index 0000000..43e1deb --- /dev/null +++ b/apps/control-api/src/infra/database/migrate/tests/audit.rs @@ -0,0 +1,544 @@ +use std::collections::BTreeMap; + +use anyhow::{Context, ensure}; +use sea_orm::{ConnectionTrait, DatabaseBackend, DatabaseConnection, Statement}; +use sea_orm_migration::MigratorTrait; +use uuid::Uuid; + +use super::super::{MIGRATION_TEST_LOCK, Migrator}; +use super::support::{ + PostgresMigrationDatabase, assert_migration_tracking, column, ensure_index, object_count, + query_column_shapes, +}; + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL"] +async fn postgres_audit_foundation_migration_upgrades_v11_and_is_reversible() -> anyhow::Result<()> +{ + let _migration_guard = MIGRATION_TEST_LOCK.lock().await; + let database_url = std::env::var("GRASS_TEST_DATABASE_URL") + .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); + let test_db = PostgresMigrationDatabase::start(&database_url).await?; + + let verification = verify_audit_foundation_migration(&test_db.db).await; + let cleanup = test_db.cleanup().await; + + match (verification, cleanup) { + (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context(format!( + "disposable schema cleanup also failed: {cleanup_error:#}" + ))), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Ok(()), Ok(())) => Ok(()), + } +} + +async fn verify_audit_foundation_migration(db: &DatabaseConnection) -> anyhow::Result<()> { + Migrator::up(db, Some(11)).await?; + assert_migration_tracking(db, 11).await?; + + let user_id = Uuid::now_v7(); + let team_id = Uuid::now_v7(); + let project_id = Uuid::now_v7(); + let deployment_id = Uuid::now_v7(); + seed_v11_audit_fixtures(db, user_id, team_id, project_id, deployment_id).await?; + + Migrator::up(db, Some(1)).await?; + assert_migration_tracking(db, 12).await?; + assert_audit_enum_shapes(db).await?; + assert_audit_column_shapes(db).await?; + assert_audit_constraints(db).await?; + assert_audit_indexes(db).await?; + assert_audit_backfill(db, deployment_id).await?; + + Migrator::down(db, Some(1)).await?; + assert_migration_tracking(db, 11).await?; + assert_audit_foundation_objects_absent(db).await?; + + Migrator::up(db, None).await?; + assert_migration_tracking(db, 35).await?; + assert_audit_foundation_objects_restored(db).await?; + + Ok(()) +} + +async fn seed_v11_audit_fixtures( + db: &DatabaseConnection, + user_id: Uuid, + team_id: Uuid, + project_id: Uuid, + deployment_id: Uuid, +) -> anyhow::Result<()> { + db.execute_unprepared(&format!( + r#" +INSERT INTO users (id, email, display_name) +VALUES ('{user_id}'::uuid, 'audit-migration@example.invalid', 'Audit Migration'); + +INSERT INTO teams (id, slug, name, owner_user_id) +VALUES ('{team_id}'::uuid, 'audit-migration', 'Audit Migration', '{user_id}'::uuid); + +INSERT INTO projects (id, team_id, slug, name) +VALUES ('{project_id}'::uuid, '{team_id}'::uuid, 'audit-migration', 'Audit Migration'); + +INSERT INTO audit_events (actor_user_id, action, target_type, metadata, team_id) +VALUES +('{user_id}'::uuid, 'project.updated', 'project', '{{}}'::jsonb, '{team_id}'::uuid), +(NULL, 'project.deleted', 'project', '{{"platform_admin": true}}'::jsonb, '{team_id}'::uuid), +('{user_id}'::uuid, 'deployment.release.completed', 'deployment', '{{"completed_after_sync": true}}'::jsonb, '{team_id}'::uuid), +(NULL, 'team.quota_plan_overridden', 'team', '{{}}'::jsonb, '{team_id}'::uuid); + +INSERT INTO deployments ( +id, +project_id, +team_id, +pending_release_reason, +pending_release_actor_user_id, +pending_release_requested_at +) +VALUES ( +'{deployment_id}'::uuid, +'{project_id}'::uuid, +'{team_id}'::uuid, +'rollback', +'{user_id}'::uuid, +CURRENT_TIMESTAMP +); +"# + )) + .await?; + + Ok(()) +} + +async fn assert_audit_enum_shapes(db: &DatabaseConnection) -> anyhow::Result<()> { + let rows = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT t.typname, string_agg(e.enumlabel, ',' ORDER BY e.enumsortorder) AS labels +FROM pg_type t +JOIN pg_enum e ON e.enumtypid = t.oid +JOIN pg_namespace n ON n.oid = t.typnamespace +WHERE n.nspname = current_schema() + AND t.typname IN ('audit_actor_type', 'audit_event_visibility') +GROUP BY t.typname +ORDER BY t.typname +"#, + )) + .await?; + let enum_shapes = rows + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "typname")?, + row.try_get::("", "labels")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + + ensure!( + enum_shapes + == vec![ + ( + "audit_actor_type".to_owned(), + "anonymous,user,system,node".to_owned(), + ), + ( + "audit_event_visibility".to_owned(), + "platform,team".to_owned(), + ), + ], + "unexpected audit enum shapes: {enum_shapes:?}" + ); + Ok(()) +} + +async fn assert_audit_column_shapes(db: &DatabaseConnection) -> anyhow::Result<()> { + let audit_columns = query_column_shapes( + db, + r#" +SELECT column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'audit_events' + AND column_name IN ( +'actor_type', +'actor_node_id', +'visibility', +'request_id', +'source_ip', +'user_agent', +'http_method', +'request_path', +'status_code', +'duration_ms', +'changes' + ) +ORDER BY column_name +"#, + ) + .await?; + ensure!( + audit_columns + == vec![ + column("actor_node_id", "uuid", "YES", None), + column( + "actor_type", + "audit_actor_type", + "NO", + Some("'system'::audit_actor_type"), + ), + column("changes", "jsonb", "NO", Some("'{}'::jsonb")), + column("duration_ms", "int8", "YES", None), + column("http_method", "text", "YES", None), + column("request_id", "uuid", "YES", None), + column("request_path", "text", "YES", None), + column("source_ip", "text", "YES", None), + column("status_code", "int4", "YES", None), + column("user_agent", "text", "YES", None), + column( + "visibility", + "audit_event_visibility", + "NO", + Some("'platform'::audit_event_visibility"), + ), + ], + "unexpected audit_events column shapes: {audit_columns:#?}" + ); + + let deployment_columns = query_column_shapes( + db, + r#" +SELECT column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'deployments' + AND column_name = 'pending_release_audit_visibility' +"#, + ) + .await?; + ensure!( + deployment_columns + == vec![column( + "pending_release_audit_visibility", + "audit_event_visibility", + "YES", + None, + )], + "unexpected deployment provenance column shape: {deployment_columns:#?}" + ); + + Ok(()) +} + +async fn assert_audit_constraints(db: &DatabaseConnection) -> anyhow::Result<()> { + let rows = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT conname, contype::text AS constraint_type, pg_get_constraintdef(oid) AS definition +FROM pg_constraint +WHERE conrelid = 'audit_events'::regclass + AND conname IN ( +'fk_audit_events_actor_node_id', +'ck_audit_events_actor_identity', +'ck_audit_events_status_code', +'ck_audit_events_duration_ms' + ) +ORDER BY conname +"#, + )) + .await?; + let constraints = rows + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "conname")?, + ( + row.try_get::("", "constraint_type")?, + row.try_get::("", "definition")?, + ), + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + constraints.len() == 4, + "expected four audit constraints, found {constraints:#?}" + ); + + let (kind, definition) = constraints + .get("fk_audit_events_actor_node_id") + .context("missing actor node foreign key")?; + ensure!(kind == "f", "actor node constraint is not a foreign key"); + ensure!( + definition.contains("FOREIGN KEY (actor_node_id)") + && definition.contains("REFERENCES nodes(id)") + && definition.contains("ON DELETE SET NULL"), + "unexpected actor node foreign key: {definition}" + ); + + let (kind, definition) = constraints + .get("ck_audit_events_actor_identity") + .context("missing actor identity constraint")?; + ensure!(kind == "c", "actor identity constraint is not a check"); + ensure!( + definition.contains("actor_user_id IS NULL") + && definition.contains("actor_type = 'user'") + && definition.contains("actor_node_id IS NULL") + && definition.contains("actor_type = 'node'") + && definition.contains("actor_type <> ALL"), + "unexpected actor identity check: {definition}" + ); + + let (kind, definition) = constraints + .get("ck_audit_events_status_code") + .context("missing status code constraint")?; + ensure!(kind == "c", "status code constraint is not a check"); + ensure!( + definition.contains("status_code IS NULL") + && definition.contains("status_code >= 100") + && definition.contains("status_code <= 599"), + "unexpected status code check: {definition}" + ); + + let (kind, definition) = constraints + .get("ck_audit_events_duration_ms") + .context("missing duration constraint")?; + ensure!(kind == "c", "duration constraint is not a check"); + ensure!( + definition.contains("duration_ms IS NULL") && definition.contains("duration_ms >= 0"), + "unexpected duration check: {definition}" + ); + + let deployment_constraint = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT contype::text AS constraint_type, pg_get_constraintdef(oid) AS definition +FROM pg_constraint +WHERE conrelid = 'deployments'::regclass + AND conname = 'ck_deployments_pending_release_audit_visibility' +"#, + )) + .await? + .context("missing pending release audit visibility constraint")?; + let kind = deployment_constraint.try_get::("", "constraint_type")?; + let definition = deployment_constraint.try_get::("", "definition")?; + ensure!(kind == "c", "pending release constraint is not a check"); + ensure!( + definition.contains("pending_release_reason IS NULL") + && definition.contains("pending_release_audit_visibility IS NULL"), + "unexpected pending release audit visibility check: {definition}" + ); + + Ok(()) +} + +async fn assert_audit_indexes(db: &DatabaseConnection) -> anyhow::Result<()> { + let rows = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT indexname, indexdef +FROM pg_indexes +WHERE schemaname = current_schema() + AND tablename = 'audit_events' + AND indexname IN ( +'ux_audit_events_request_id', +'ix_audit_events_visibility_created_at', +'ix_audit_events_actor_created_at', +'ix_audit_events_actor_node_created_at', +'ix_audit_events_created_at' + ) +ORDER BY indexname +"#, + )) + .await?; + let indexes = rows + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "indexname")?, + row.try_get::("", "indexdef")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + indexes.len() == 5, + "expected five audit indexes, found {indexes:#?}" + ); + + let request = indexes + .get("ux_audit_events_request_id") + .context("missing request id index")?; + ensure!( + request.contains("CREATE UNIQUE INDEX") + && request.contains("(request_id)") + && request.contains("request_id IS NOT NULL"), + "unexpected request id index: {request}" + ); + ensure_index( + &indexes, + "ix_audit_events_visibility_created_at", + "(visibility, created_at DESC)", + None, + )?; + ensure_index( + &indexes, + "ix_audit_events_actor_created_at", + "(actor_user_id, created_at DESC)", + Some("actor_user_id IS NOT NULL"), + )?; + ensure_index( + &indexes, + "ix_audit_events_actor_node_created_at", + "(actor_node_id, created_at DESC)", + Some("actor_node_id IS NOT NULL"), + )?; + ensure_index(&indexes, "ix_audit_events_created_at", "(created_at)", None)?; + + Ok(()) +} + +async fn assert_audit_backfill(db: &DatabaseConnection, deployment_id: Uuid) -> anyhow::Result<()> { + let rows = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT action, actor_type::text AS actor_type, visibility::text AS visibility +FROM audit_events +WHERE action IN ( +'project.updated', +'project.deleted', +'deployment.release.completed', +'team.quota_plan_overridden' +) +ORDER BY action +"#, + )) + .await?; + let backfilled = rows + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "action")?, + ( + row.try_get::("", "actor_type")?, + row.try_get::("", "visibility")?, + ), + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + backfilled + == BTreeMap::from([ + ( + "deployment.release.completed".to_owned(), + ("user".to_owned(), "platform".to_owned()), + ), + ( + "project.deleted".to_owned(), + ("system".to_owned(), "platform".to_owned()), + ), + ( + "project.updated".to_owned(), + ("user".to_owned(), "team".to_owned()), + ), + ( + "team.quota_plan_overridden".to_owned(), + ("system".to_owned(), "platform".to_owned()), + ), + ]), + "unexpected audit backfill: {backfilled:#?}" + ); + + let deployment_visibility = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + format!( + "SELECT pending_release_audit_visibility::text AS visibility FROM deployments WHERE id = '{deployment_id}'::uuid" + ), + )) + .await? + .context("seeded pending release deployment is missing")? + .try_get::("", "visibility")?; + ensure!( + deployment_visibility == "platform", + "pending release backfilled to {deployment_visibility:?} instead of platform" + ); + + Ok(()) +} + +async fn assert_audit_foundation_objects_absent(db: &DatabaseConnection) -> anyhow::Result<()> { + let column_count = object_count( + db, + r#" +SELECT count(*)::bigint AS count +FROM information_schema.columns +WHERE table_schema = current_schema() + AND ( +(table_name = 'audit_events' AND column_name IN ( + 'actor_type', 'actor_node_id', 'visibility', 'request_id', 'source_ip', + 'user_agent', 'http_method', 'request_path', 'status_code', 'duration_ms', 'changes' +)) +OR (table_name = 'deployments' AND column_name = 'pending_release_audit_visibility') + ) +"#, + ) + .await?; + ensure!( + column_count == 0, + "audit foundation columns remained after down migration" + ); + + let enum_count = audit_enum_count(db).await?; + ensure!( + enum_count == 0, + "audit foundation enum types remained after down migration" + ); + Ok(()) +} + +async fn assert_audit_foundation_objects_restored(db: &DatabaseConnection) -> anyhow::Result<()> { + let column_count = object_count( + db, + r#" +SELECT count(*)::bigint AS count +FROM information_schema.columns +WHERE table_schema = current_schema() + AND ( +(table_name = 'audit_events' AND column_name IN ( + 'actor_type', 'actor_node_id', 'visibility', 'request_id', 'source_ip', + 'user_agent', 'http_method', 'request_path', 'status_code', 'duration_ms', 'changes' +)) +OR (table_name = 'deployments' AND column_name = 'pending_release_audit_visibility') + ) +"#, + ) + .await?; + ensure!( + column_count == 12, + "audit foundation columns were not restored after reapplying migration" + ); + + let enum_count = audit_enum_count(db).await?; + ensure!( + enum_count == 2, + "audit foundation enum types were not restored after reapplying migration" + ); + Ok(()) +} + +async fn audit_enum_count(db: &DatabaseConnection) -> anyhow::Result { + object_count( + db, + r#" +SELECT count(*)::bigint AS count +FROM pg_type t +JOIN pg_namespace n ON n.oid = t.typnamespace +WHERE n.nspname = current_schema() + AND t.typname IN ('audit_actor_type', 'audit_event_visibility') +"#, + ) + .await +} diff --git a/apps/control-api/src/infra/database/migrate/tests/authentication.rs b/apps/control-api/src/infra/database/migrate/tests/authentication.rs new file mode 100644 index 0000000..a4a2e58 --- /dev/null +++ b/apps/control-api/src/infra/database/migrate/tests/authentication.rs @@ -0,0 +1,443 @@ +use std::collections::BTreeMap; + +use anyhow::{Context, ensure}; +use sea_orm::{ConnectionTrait, DatabaseBackend, DatabaseConnection, Statement}; +use sea_orm_migration::MigratorTrait; +use uuid::Uuid; + +use super::super::{MIGRATION_TEST_LOCK, Migrator}; +use super::support::{ + PostgresMigrationDatabase, assert_migration_tracking, column, object_count, query_column_shapes, +}; + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL"] +async fn postgres_authentication_schema_matches_domain_and_is_reversible() -> anyhow::Result<()> { + let _migration_guard = MIGRATION_TEST_LOCK.lock().await; + let database_url = std::env::var("GRASS_TEST_DATABASE_URL") + .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); + let test_db = PostgresMigrationDatabase::start(&database_url).await?; + + let verification = async { + Migrator::up(&test_db.db, Some(21)).await?; + assert_migration_tracking(&test_db.db, 21).await?; + let user_id = seed_authentication_fixture(&test_db.db).await?; + + Migrator::up(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 22).await?; + assert_authentication_schema(&test_db.db, user_id).await?; + + Migrator::down(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 21).await?; + assert_authentication_schema_absent(&test_db.db).await?; + + Migrator::up(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 22).await?; + assert_authentication_schema(&test_db.db, user_id).await + } + .await; + let cleanup = test_db.cleanup().await; + + match (verification, cleanup) { + (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context(format!( + "disposable schema cleanup also failed: {cleanup_error:#}" + ))), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Ok(()), Ok(())) => Ok(()), + } +} + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL"] +async fn postgres_mfa_policy_schema_migrates_legacy_scope_and_is_reversible() -> anyhow::Result<()> +{ + let _migration_guard = MIGRATION_TEST_LOCK.lock().await; + let database_url = std::env::var("GRASS_TEST_DATABASE_URL") + .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); + let test_db = PostgresMigrationDatabase::start(&database_url).await?; + + let verification = async { + Migrator::up(&test_db.db, Some(22)).await?; + assert_migration_tracking(&test_db.db, 22).await?; + let user_id = seed_legacy_mfa_policy_fixture(&test_db.db).await?; + + Migrator::up(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 23).await?; + assert_mfa_policy_schema(&test_db.db, user_id).await?; + + Migrator::down(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 22).await?; + assert_mfa_policy_schema_absent(&test_db.db, user_id).await?; + + Migrator::up(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 23).await?; + assert_mfa_policy_schema(&test_db.db, user_id).await + } + .await; + let cleanup = test_db.cleanup().await; + + match (verification, cleanup) { + (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context(format!( + "disposable schema cleanup also failed: {cleanup_error:#}" + ))), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Ok(()), Ok(())) => Ok(()), + } +} + +async fn seed_legacy_mfa_policy_fixture(db: &DatabaseConnection) -> anyhow::Result { + let user_id = Uuid::now_v7(); + let setting_id = Uuid::now_v7(); + db.execute_unprepared(&format!( + r#" +INSERT INTO users (id, email, display_name, email_verified_at) +VALUES ('{user_id}'::uuid, 'mfa-policy-migration@example.invalid', 'MFA Policy Migration', NOW()); + +INSERT INTO system_settings (id, key, value_kind, value, is_secret) +VALUES ( +'{setting_id}'::uuid, +'auth.mfa_policy', +'json', +jsonb_build_object( + 'allowed_factors', jsonb_build_array('totp', 'email'), + 'enforcement', 'selected_users', + 'selected_user_ids', jsonb_build_array('{user_id}'::text) +), +false +); +"# + )) + .await?; + Ok(user_id) +} + +async fn assert_mfa_policy_schema( + db: &DatabaseConnection, + selected_user_id: Uuid, +) -> anyhow::Result<()> { + let columns = query_column_shapes( + db, + r#" +SELECT column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'user_mfa_policies' +ORDER BY ordinal_position +"#, + ) + .await?; + ensure!( + columns + == vec![ + column("user_id", "uuid", "NO", None), + column("inherit_platform", "bool", "NO", Some("true")), + column("minimum_factors", "int2", "NO", Some("0")), + column("required_factors", "jsonb", "NO", Some("'[]'::jsonb")), + column("created_at", "timestamptz", "NO", None), + column("updated_at", "timestamptz", "NO", None), + ], + "unexpected user MFA policy columns: {columns:#?}" + ); + + let constraints = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT conname, pg_get_constraintdef(oid) AS definition +FROM pg_constraint +WHERE conrelid = 'user_mfa_policies'::regclass +ORDER BY conname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "conname")?, + row.try_get::("", "definition")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + constraints.len() == 4, + "missing user MFA policy constraints" + ); + ensure!(constraints["ck_user_mfa_policies_minimum_factors"].contains("minimum_factors <= 2")); + ensure!( + constraints["ck_user_mfa_policies_required_factors"] + .contains("jsonb_typeof(required_factors)") + && constraints["ck_user_mfa_policies_required_factors"].contains("'array'") + ); + ensure!( + constraints + .values() + .any(|definition| definition.contains("FOREIGN KEY (user_id)") + && definition.contains("REFERENCES users(id) ON DELETE CASCADE")) + ); + + let row = db + .query_one_raw(Statement::from_sql_and_values( + DatabaseBackend::Postgres, + r#" +SELECT +setting.value AS platform_policy, +policy.inherit_platform, +policy.minimum_factors, +policy.required_factors +FROM system_settings AS setting +JOIN user_mfa_policies AS policy ON policy.user_id = $1 +WHERE setting.key = 'auth.mfa_policy' +"#, + [selected_user_id.into()], + )) + .await? + .context("migrated MFA policy row is missing")?; + let platform_policy = row.try_get::("", "platform_policy")?; + ensure!(platform_policy["enforcement"] == "none"); + ensure!(platform_policy["minimum_factors"] == 0); + ensure!(platform_policy["required_factors"] == serde_json::json!([])); + ensure!(platform_policy.get("selected_user_ids").is_none()); + ensure!(!row.try_get::("", "inherit_platform")?); + ensure!(row.try_get::("", "minimum_factors")? == 1); + ensure!(row.try_get::("", "required_factors")? == serde_json::json!([])); + Ok(()) +} + +async fn assert_mfa_policy_schema_absent( + db: &DatabaseConnection, + selected_user_id: Uuid, +) -> anyhow::Result<()> { + ensure!( + object_count( + db, + "SELECT count(*)::bigint AS count FROM information_schema.tables WHERE \ + table_schema = current_schema() AND table_name = 'user_mfa_policies'", + ) + .await? + == 0 + ); + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + "SELECT value FROM system_settings WHERE key = 'auth.mfa_policy'", + )) + .await? + .context("legacy MFA policy row is missing after down migration")?; + let policy = row.try_get::("", "value")?; + ensure!(policy["enforcement"] == "selected_users"); + ensure!(policy["selected_user_ids"] == serde_json::json!([selected_user_id])); + ensure!(policy.get("minimum_factors").is_none()); + ensure!(policy.get("required_factors").is_none()); + Ok(()) +} + +async fn seed_authentication_fixture(db: &DatabaseConnection) -> anyhow::Result { + let user_id = Uuid::now_v7(); + let credential_id = Uuid::now_v7(); + db.execute_unprepared(&format!( + r#" +INSERT INTO users (id, email, display_name) +VALUES ('{user_id}'::uuid, 'authentication-migration@example.invalid', 'Authentication Migration'); + +INSERT INTO user_password_credentials (id, user_id, password_hash) +VALUES ('{credential_id}'::uuid, '{user_id}'::uuid, 'migration-password-hash'); +"# + )) + .await?; + Ok(user_id) +} + +async fn assert_authentication_schema( + db: &DatabaseConnection, + seeded_user_id: Uuid, +) -> anyhow::Result<()> { + let columns = query_column_shapes( + db, + r#" +SELECT column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND ( +(table_name = 'users' AND column_name = 'email_verified_at') OR +(table_name = 'user_auth_tokens' AND column_name = 'used_at') OR +(table_name = 'user_mfa_factors' AND column_name IN ('verified_at', 'last_used_at')) + ) +ORDER BY table_name, ordinal_position +"#, + ) + .await?; + ensure!( + columns.len() == 4, + "missing authentication lifecycle columns" + ); + for column in columns { + ensure!( + column.udt_name == "timestamptz", + "unexpected column type: {column:?}" + ); + ensure!( + column.nullable == "YES", + "lifecycle column is not nullable: {column:?}" + ); + ensure!( + column.default.is_none(), + "lifecycle column has a default: {column:?}" + ); + } + + let enum_rows = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT t.typname, string_agg(e.enumlabel, ',' ORDER BY e.enumsortorder) AS labels +FROM pg_type t +JOIN pg_enum e ON e.enumtypid = t.oid +JOIN pg_namespace n ON n.oid = t.typnamespace +WHERE n.nspname = current_schema() + AND t.typname IN ('identity_provider_kind', 'auth_token_kind', 'mfa_factor_kind') +GROUP BY t.typname +ORDER BY t.typname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "typname")?, + row.try_get::("", "labels")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!(enum_rows.get("identity_provider_kind") == Some(&"oidc,github".to_owned())); + ensure!( + enum_rows.get("auth_token_kind") == Some(&"email_verification,password_reset".to_owned()) + ); + ensure!(enum_rows.get("mfa_factor_kind") == Some(&"totp,email".to_owned())); + + let indexes = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT indexname, indexdef +FROM pg_indexes +WHERE schemaname = current_schema() + AND indexname IN ( +'ix_user_external_identities_user_id', +'ix_user_auth_tokens_live', +'ix_user_mfa_factors_verified', +'ix_user_password_history_recent' + ) +ORDER BY indexname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "indexname")?, + row.try_get::("", "indexdef")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!(indexes.len() == 4, "missing authentication indexes"); + ensure!(indexes["ix_user_auth_tokens_live"].contains("WHERE (used_at IS NULL)")); + ensure!(indexes["ix_user_mfa_factors_verified"].contains("WHERE (verified_at IS NOT NULL)")); + + let constraints = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT conname, pg_get_constraintdef(oid) AS definition +FROM pg_constraint +WHERE conrelid IN ( +'auth_identity_providers'::regclass, +'user_external_identities'::regclass, +'user_auth_tokens'::regclass, +'user_mfa_factors'::regclass, +'user_password_history'::regclass +) + AND contype = 'f' +ORDER BY conname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "conname")?, + row.try_get::("", "definition")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + constraints.len() == 6, + "missing authentication foreign keys" + ); + ensure!( + constraints + .values() + .filter(|definition| definition.contains("ON DELETE CASCADE")) + .count() + == 5 + ); + ensure!( + constraints + .values() + .filter(|definition| definition.contains("ON DELETE SET NULL")) + .count() + == 1 + ); + + let backfill = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + format!( + r#" +SELECT +u.email_verified_at = u.created_at AS email_backfilled, +h.password_hash +FROM users u +JOIN user_password_history h ON h.user_id = u.id +WHERE u.id = '{seeded_user_id}'::uuid +"# + ), + )) + .await? + .context("authentication backfill row is missing")?; + ensure!(backfill.try_get::("", "email_backfilled")?); + ensure!(backfill.try_get::("", "password_hash")? == "migration-password-hash"); + Ok(()) +} + +async fn assert_authentication_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> { + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT +to_regclass('auth_identity_providers') IS NULL AND +to_regclass('user_external_identities') IS NULL AND +to_regclass('user_auth_tokens') IS NULL AND +to_regclass('user_mfa_factors') IS NULL AND +to_regclass('user_password_history') IS NULL AS tables_absent, +NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'users' + AND column_name = 'email_verified_at' +) AS column_absent, +NOT EXISTS ( + SELECT 1 FROM pg_type t + JOIN pg_namespace n ON n.oid = t.typnamespace + WHERE n.nspname = current_schema() + AND t.typname IN ('identity_provider_kind', 'auth_token_kind', 'mfa_factor_kind') +) AS types_absent +"#, + )) + .await? + .context("authentication absence query returned no row")?; + ensure!(row.try_get::("", "tables_absent")?); + ensure!(row.try_get::("", "column_absent")?); + ensure!(row.try_get::("", "types_absent")?); + Ok(()) +} diff --git a/apps/control-api/src/infra/database/migrate/tests/ingress.rs b/apps/control-api/src/infra/database/migrate/tests/ingress.rs new file mode 100644 index 0000000..ea9ac73 --- /dev/null +++ b/apps/control-api/src/infra/database/migrate/tests/ingress.rs @@ -0,0 +1,273 @@ +use std::collections::BTreeMap; + +use anyhow::ensure; +use sea_orm::{ConnectionTrait, DatabaseBackend, DatabaseConnection, Statement}; +use sea_orm_migration::MigratorTrait; + +use super::super::{MIGRATION_TEST_LOCK, Migrator}; +use super::support::{ + PostgresMigrationDatabase, assert_migration_tracking, column, object_count, query_column_shapes, +}; + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL"] +async fn postgres_regional_ingress_schema_matches_domain_and_is_reversible() -> anyhow::Result<()> { + let _migration_guard = MIGRATION_TEST_LOCK.lock().await; + let database_url = std::env::var("GRASS_TEST_DATABASE_URL") + .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); + let test_db = PostgresMigrationDatabase::start(&database_url).await?; + + let verification = async { + Migrator::up(&test_db.db, Some(31)).await?; + assert_migration_tracking(&test_db.db, 31).await?; + assert_regional_ingress_schema(&test_db.db).await?; + + Migrator::down(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 30).await?; + assert_regional_ingress_lifecycle_absent(&test_db.db).await?; + + Migrator::down(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 29).await?; + assert_regional_ingress_schema_absent(&test_db.db).await?; + + Migrator::up(&test_db.db, Some(2)).await?; + assert_migration_tracking(&test_db.db, 31).await?; + assert_regional_ingress_schema(&test_db.db).await + } + .await; + let cleanup = test_db.cleanup().await; + + match (verification, cleanup) { + (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context(format!( + "disposable schema cleanup also failed: {cleanup_error:#}" + ))), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Ok(()), Ok(())) => Ok(()), + } +} + +async fn assert_regional_ingress_schema(db: &DatabaseConnection) -> anyhow::Result<()> { + let columns = query_column_shapes( + db, + r#" +SELECT column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'regional_ingresses' +ORDER BY ordinal_position +"#, + ) + .await?; + ensure!( + columns + == vec![ + column("id", "uuid", "NO", None), + column("region", "text", "NO", None), + column("hostname", "varchar", "NO", None), + column("enabled", "bool", "NO", Some("true")), + column("health_check_path", "text", "NO", Some("'/health'::text")), + column("health_check_interval_seconds", "int4", "NO", Some("30")), + column("origin_host_preservation", "bool", "NO", Some("true")), + column("tls_enabled", "bool", "NO", Some("true")), + column( + "certificate_issuer", + "text", + "NO", + Some("'letsencrypt'::text") + ), + column("certificate_auto_renew", "bool", "NO", Some("true")), + column("certificate_status", "text", "NO", Some("'pending'::text")), + column("certificate_expires_at", "timestamptz", "YES", None), + column("certificate_error", "text", "YES", None), + column("dns_challenge_provider", "text", "YES", None), + column("dns_challenge_config", "jsonb", "NO", Some("'{}'::jsonb")), + column( + "dns_challenge_status", + "text", + "NO", + Some("'not_configured'::text") + ), + column("dns_challenge_record_name", "text", "YES", None), + column("dns_challenge_record_value", "text", "YES", None), + column("deleted_at", "timestamptz", "YES", None), + column("created_at", "timestamptz", "NO", None), + column("updated_at", "timestamptz", "NO", None), + column("acme_account", "jsonb", "YES", None), + column("certificate_bundle", "jsonb", "YES", None), + column("certificate_issued_at", "timestamptz", "YES", None), + ], + "unexpected regional_ingresses column shapes: {columns:#?}" + ); + + let constraints = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT conname, pg_get_constraintdef(oid) AS definition +FROM pg_constraint +WHERE conrelid = 'regional_ingresses'::regclass + AND conname LIKE 'ck_regional_ingresses_%' +ORDER BY conname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "conname")?, + row.try_get::("", "definition")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + constraints.len() == 7, + "unexpected ingress constraints: {constraints:#?}" + ); + ensure!(constraints["ck_regional_ingresses_region_nonempty"].contains("char_length")); + ensure!(constraints["ck_regional_ingresses_hostname_nonempty"].contains("char_length")); + // PostgreSQL deparses LIKE and BETWEEN into their underlying operators. + ensure!( + constraints["ck_regional_ingresses_health_path"] + .contains("health_check_path ~~ '/%'::text") + ); + ensure!( + constraints["ck_regional_ingresses_health_interval"] + .contains("health_check_interval_seconds >= 5") + && constraints["ck_regional_ingresses_health_interval"] + .contains("health_check_interval_seconds <= 3600") + ); + ensure!(constraints["ck_regional_ingresses_certificate_issuer"].contains("letsencrypt")); + ensure!(constraints["ck_regional_ingresses_certificate_status"].contains("certificate_status")); + ensure!(constraints["ck_regional_ingresses_dns_challenge_status"].contains("not_configured")); + + let indexes = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT indexname, indexdef +FROM pg_indexes +WHERE schemaname = current_schema() + AND indexname IN ( +'ux_regional_ingresses_region_active', +'ux_regional_ingresses_hostname_active', +'ix_regional_ingresses_enabled', +'ix_host_sources_region', +'ix_project_host_bindings_region' + ) +ORDER BY indexname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "indexname")?, + row.try_get::("", "indexdef")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + indexes.len() == 5, + "unexpected ingress indexes: {indexes:#?}" + ); + ensure!(indexes["ux_regional_ingresses_region_active"].contains("UNIQUE")); + ensure!(indexes["ux_regional_ingresses_region_active"].contains("deleted_at IS NULL")); + ensure!(indexes["ux_regional_ingresses_hostname_active"].contains("UNIQUE")); + ensure!(indexes["ix_regional_ingresses_enabled"].contains("(region, enabled)")); + ensure!(indexes["ix_host_sources_region"].contains("(region)")); + ensure!(indexes["ix_project_host_bindings_region"].contains("(region)")); + ensure!( + object_count( + db, + "SELECT count(*)::bigint AS count FROM information_schema.tables WHERE \ + table_schema = current_schema() AND table_name = 'regional_ingress_health'", + ) + .await? + == 1, + "regional ingress health table is missing" + ); + Ok(()) +} + +pub(super) async fn assert_regional_ingress_lifecycle_absent( + db: &DatabaseConnection, +) -> anyhow::Result<()> { + ensure!( + object_count( + db, + "SELECT count(*)::bigint AS count FROM information_schema.tables WHERE \ + table_schema = current_schema() AND table_name = 'regional_ingress_health'", + ) + .await? + == 0, + "regional ingress health table remained after lifecycle down migration" + ); + ensure!( + object_count( + db, + "SELECT count(*)::bigint AS count FROM information_schema.columns WHERE \ + table_schema = current_schema() AND table_name = 'regional_ingresses' AND \ + column_name IN ('acme_account', 'certificate_bundle', \ + 'certificate_issued_at')", + ) + .await? + == 0, + "regional ingress lifecycle columns remained after lifecycle down migration" + ); + Ok(()) +} + +pub(super) async fn assert_regional_ingress_schema_absent( + db: &DatabaseConnection, +) -> anyhow::Result<()> { + ensure!( + object_count( + db, + r#" +SELECT count(*)::bigint AS count +FROM information_schema.tables +WHERE table_schema = current_schema() + AND table_name = 'regional_ingresses' +"#, + ) + .await? + == 0, + "regional_ingresses table remained after down migration" + ); + ensure!( + object_count( + db, + r#" +SELECT count(*)::bigint AS count +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name IN ('host_sources', 'project_host_bindings') + AND column_name = 'region' +"#, + ) + .await? + == 0, + "regional host columns remained after down migration" + ); + ensure!( + object_count( + db, + r#" +SELECT count(*)::bigint AS count +FROM pg_indexes +WHERE schemaname = current_schema() + AND indexname IN ( +'ux_regional_ingresses_region_active', +'ux_regional_ingresses_hostname_active', +'ix_regional_ingresses_enabled', +'ix_host_sources_region', +'ix_project_host_bindings_region' + ) +"#, + ) + .await? + == 0, + "regional ingress indexes remained after down migration" + ); + Ok(()) +} diff --git a/apps/control-api/src/infra/database/migrate/tests/media.rs b/apps/control-api/src/infra/database/migrate/tests/media.rs new file mode 100644 index 0000000..d14ae6a --- /dev/null +++ b/apps/control-api/src/infra/database/migrate/tests/media.rs @@ -0,0 +1,345 @@ +use std::collections::BTreeMap; + +use anyhow::{Context, ensure}; +use sea_orm::{ConnectionTrait, DatabaseBackend, DatabaseConnection, Statement}; +use sea_orm_migration::MigratorTrait; + +use super::super::{MIGRATION_TEST_LOCK, Migrator}; +use super::ingress::{ + assert_regional_ingress_lifecycle_absent, assert_regional_ingress_schema_absent, +}; +use super::support::{ + PostgresMigrationDatabase, assert_migration_tracking, column, object_count, query_column_shapes, +}; + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL"] +async fn postgres_media_schema_matches_domain_and_is_reversible() -> anyhow::Result<()> { + let _migration_guard = MIGRATION_TEST_LOCK.lock().await; + let database_url = std::env::var("GRASS_TEST_DATABASE_URL") + .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); + let test_db = PostgresMigrationDatabase::start(&database_url).await?; + + let verification = async { + Migrator::up(&test_db.db, Some(31)).await?; + assert_migration_tracking(&test_db.db, 31).await?; + assert_avatar_schema(&test_db.db).await?; + assert_screenshot_schema(&test_db.db).await?; + assert_object_storage_schema(&test_db.db).await?; + + Migrator::down(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 30).await?; + assert_regional_ingress_lifecycle_absent(&test_db.db).await?; + assert_avatar_schema(&test_db.db).await?; + assert_screenshot_schema(&test_db.db).await?; + assert_object_storage_schema(&test_db.db).await?; + + Migrator::down(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 29).await?; + assert_regional_ingress_schema_absent(&test_db.db).await?; + assert_avatar_schema(&test_db.db).await?; + assert_screenshot_schema(&test_db.db).await?; + assert_object_storage_schema(&test_db.db).await?; + + Migrator::down(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 28).await?; + assert_avatar_schema(&test_db.db).await?; + assert_screenshot_schema(&test_db.db).await?; + assert_object_storage_schema(&test_db.db).await?; + + Migrator::down(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 27).await?; + assert_avatar_schema(&test_db.db).await?; + assert_screenshot_schema(&test_db.db).await?; + assert_object_storage_schema_absent(&test_db.db).await?; + + Migrator::up(&test_db.db, Some(4)).await?; + assert_migration_tracking(&test_db.db, 31).await?; + assert_avatar_schema(&test_db.db).await?; + assert_screenshot_schema(&test_db.db).await?; + assert_object_storage_schema(&test_db.db).await + } + .await; + let cleanup = test_db.cleanup().await; + + match (verification, cleanup) { + (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context(format!( + "disposable schema cleanup also failed: {cleanup_error:#}" + ))), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Ok(()), Ok(())) => Ok(()), + } +} + +async fn assert_avatar_schema(db: &DatabaseConnection) -> anyhow::Result<()> { + let rows = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT table_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND column_name = 'avatar_version' + AND table_name IN ('teams', 'users') +ORDER BY table_name +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "table_name")?, + row.try_get::("", "udt_name")?, + row.try_get::("", "is_nullable")?, + row.try_get::>("", "column_default")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + rows == vec![ + ( + "teams".to_owned(), + "uuid".to_owned(), + "YES".to_owned(), + None, + ), + ( + "users".to_owned(), + "uuid".to_owned(), + "YES".to_owned(), + None, + ), + ], + "unexpected avatar columns: {rows:#?}" + ); + Ok(()) +} + +async fn assert_screenshot_schema(db: &DatabaseConnection) -> anyhow::Result<()> { + let enum_rows = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT t.typname, string_agg(e.enumlabel, ',' ORDER BY e.enumsortorder) AS labels +FROM pg_type t +JOIN pg_enum e ON e.enumtypid = t.oid +JOIN pg_namespace n ON n.oid = t.typnamespace +WHERE n.nspname = current_schema() + AND t.typname IN ('deployment_artifact_kind', 'deployment_screenshot_status') +GROUP BY t.typname +ORDER BY t.typname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "typname")?, + row.try_get::("", "labels")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + enum_rows + == vec![ + ( + "deployment_artifact_kind".to_owned(), + "grass_output,build_log,static_site,screenshot".to_owned(), + ), + ( + "deployment_screenshot_status".to_owned(), + "pending,running,succeeded,failed".to_owned(), + ), + ], + "unexpected screenshot enum values: {enum_rows:#?}" + ); + + let columns = query_column_shapes( + db, + r#" +SELECT column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'deployment_screenshot_jobs' +ORDER BY ordinal_position +"#, + ) + .await?; + ensure!( + columns.len() == 8 + && columns[0] == column("deployment_id", "uuid", "NO", None) + && columns[1].name == "status" + && columns[1].udt_name == "deployment_screenshot_status" + && columns[1].nullable == "NO" + && columns[1] + .default + .as_deref() + .is_some_and(|value| value.contains("'pending'")) + && columns[2].name == "attempt_count" + && columns[2].udt_name == "int4" + && columns[2].nullable == "NO" + && columns[2].default.as_deref() == Some("0") + && columns[3] == column("next_attempt_at", "timestamptz", "NO", None) + && columns[4] == column("last_error", "text", "YES", None) + && columns[5] == column("artifact_id", "uuid", "YES", None) + && columns[6] == column("created_at", "timestamptz", "NO", None) + && columns[7] == column("updated_at", "timestamptz", "NO", None), + "unexpected screenshot job columns: {columns:#?}" + ); + + let constraints = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT conname, pg_get_constraintdef(oid) AS definition +FROM pg_constraint +WHERE conrelid = 'deployment_screenshot_jobs'::regclass +ORDER BY conname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "conname")?, + row.try_get::("", "definition")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + constraints + .get("deployment_screenshot_jobs_deployment_id_fkey") + .is_some_and(|value| value.contains("ON DELETE CASCADE")), + "screenshot deployment foreign key must cascade" + ); + ensure!( + constraints + .get("deployment_screenshot_jobs_artifact_id_fkey") + .is_some_and(|value| value.contains("ON DELETE CASCADE")), + "screenshot artifact foreign key must cascade" + ); + ensure!( + constraints + .get("ck_deployment_screenshot_attempt_count") + .is_some_and(|value| value.contains("attempt_count <= 4")), + "screenshot attempt constraint is missing" + ); + ensure!( + constraints + .get("ck_deployment_screenshot_artifact") + .is_some_and(|value| value.contains("artifact_id IS NOT NULL")), + "screenshot artifact state constraint is missing" + ); + + let index = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT indexdef +FROM pg_indexes +WHERE schemaname = current_schema() + AND indexname = 'ix_deployment_screenshot_jobs_due' +"#, + )) + .await? + .context("screenshot due-job index is missing")?; + let index = index.try_get::("", "indexdef")?; + ensure!( + index.contains("next_attempt_at, deployment_id") && index.contains("status = 'pending'"), + "unexpected screenshot due-job index: {index}" + ); + Ok(()) +} + +async fn assert_object_storage_schema(db: &DatabaseConnection) -> anyhow::Result<()> { + let table_count = object_count( + db, + r#" +SELECT count(*)::bigint AS count +FROM information_schema.tables +WHERE table_schema = current_schema() + AND table_name IN ('storage_migration_jobs', 'storage_migration_objects') +"#, + ) + .await?; + ensure!( + table_count == 2, + "object storage migration tables are incomplete" + ); + + let enum_count = object_count( + db, + r#" +SELECT count(*)::bigint AS count +FROM pg_type t +JOIN pg_namespace n ON n.oid = t.typnamespace +WHERE n.nspname = current_schema() + AND t.typname IN ('storage_migration_status', 'storage_migration_object_status') +"#, + ) + .await?; + ensure!( + enum_count == 2, + "object storage migration enums are incomplete" + ); + + let index_count = object_count( + db, + r#" +SELECT count(*)::bigint AS count +FROM pg_indexes +WHERE schemaname = current_schema() + AND indexname IN ('ux_storage_migration_jobs_active', 'ix_storage_migration_objects_due') +"#, + ) + .await?; + ensure!( + index_count == 2, + "object storage migration indexes are incomplete" + ); + Ok(()) +} + +async fn assert_object_storage_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> { + ensure!( + object_count( + db, + r#" +SELECT count(*)::bigint AS count +FROM information_schema.tables +WHERE table_schema = current_schema() + AND table_name IN ('storage_migration_jobs', 'storage_migration_objects') +"#, + ) + .await? + == 0 + ); + ensure!( + object_count( + db, + r#" +SELECT count(*)::bigint AS count +FROM pg_type t +JOIN pg_namespace n ON n.oid = t.typnamespace +WHERE n.nspname = current_schema() + AND t.typname IN ('storage_migration_status', 'storage_migration_object_status') +"#, + ) + .await? + == 0 + ); + ensure!( + object_count( + db, + r#" +SELECT count(*)::bigint AS count +FROM pg_indexes +WHERE schemaname = current_schema() + AND indexname IN ('ux_storage_migration_jobs_active', 'ix_storage_migration_objects_due') +"#, + ) + .await? + == 0 + ); + Ok(()) +} diff --git a/apps/control-api/src/infra/database/migrate/tests/mod.rs b/apps/control-api/src/infra/database/migrate/tests/mod.rs new file mode 100644 index 0000000..25301d8 --- /dev/null +++ b/apps/control-api/src/infra/database/migrate/tests/mod.rs @@ -0,0 +1,13 @@ +//! Migration registration, schema, upgrade/rollback and domain regression suites. + +mod allowlist; +mod audit; +mod authentication; +mod ingress; +mod media; +mod nodes; +mod notifications; +mod regions; +mod registration; +mod revocation; +mod support; diff --git a/apps/control-api/src/infra/database/migrate/tests/nodes.rs b/apps/control-api/src/infra/database/migrate/tests/nodes.rs new file mode 100644 index 0000000..58aea43 --- /dev/null +++ b/apps/control-api/src/infra/database/migrate/tests/nodes.rs @@ -0,0 +1,247 @@ +use std::collections::BTreeMap; + +use anyhow::{Context, ensure}; +use sea_orm::{ConnectionTrait, DatabaseBackend, DatabaseConnection, Statement}; +use sea_orm_migration::MigratorTrait; + +use super::super::{MIGRATION_TEST_LOCK, Migrator}; +use super::support::{PostgresMigrationDatabase, assert_migration_tracking}; + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL"] +async fn postgres_node_deletion_queue_schema_matches_domain_and_is_reversible() -> anyhow::Result<()> +{ + let _migration_guard = MIGRATION_TEST_LOCK.lock().await; + let database_url = std::env::var("GRASS_TEST_DATABASE_URL") + .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); + let test_db = PostgresMigrationDatabase::start(&database_url).await?; + + let verification = async { + Migrator::up(&test_db.db, Some(15)).await?; + assert_migration_tracking(&test_db.db, 15).await?; + assert_node_deletion_schema(&test_db.db).await?; + + Migrator::down(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 14).await?; + assert_node_deletion_schema_absent(&test_db.db).await?; + + Migrator::up(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 15).await?; + assert_node_deletion_schema(&test_db.db).await + } + .await; + let cleanup = test_db.cleanup().await; + + match (verification, cleanup) { + (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context(format!( + "disposable schema cleanup also failed: {cleanup_error:#}" + ))), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Ok(()), Ok(())) => Ok(()), + } +} + +async fn assert_node_deletion_schema(db: &DatabaseConnection) -> anyhow::Result<()> { + let enums = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT t.typname, string_agg(e.enumlabel, ',' ORDER BY e.enumsortorder) AS labels +FROM pg_type t +JOIN pg_enum e ON e.enumtypid = t.oid +JOIN pg_namespace n ON n.oid = t.typnamespace +WHERE n.nspname = current_schema() + AND t.typname IN ('node_deletion_status', 'node_deployment_migration_status') +GROUP BY t.typname +ORDER BY t.typname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "typname")?, + row.try_get::("", "labels")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + enums + == vec![ + ( + "node_deletion_status".to_owned(), + "queued,migrating,draining,deleting,failed,completed".to_owned(), + ), + ( + "node_deployment_migration_status".to_owned(), + "pending,syncing,ready,failed".to_owned(), + ), + ], + "node deletion enum values did not match the domain model: {enums:?}" + ); + + let columns = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT table_name, column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name IN ('node_deletion_jobs', 'node_deployment_migrations') +ORDER BY table_name, ordinal_position +"#, + )) + .await?; + ensure!( + columns.len() == 22, + "expected 22 node deletion columns, found {}", + columns.len() + ); + let shapes = columns + .into_iter() + .map(|row| { + Ok(( + format!( + "{}.{}", + row.try_get::("", "table_name")?, + row.try_get::("", "column_name")?, + ), + ( + row.try_get::("", "udt_name")?, + row.try_get::("", "is_nullable")?, + row.try_get::>("", "column_default")?, + ), + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!( + shapes.get("node_deletion_jobs.status") + == Some(&( + "node_deletion_status".to_owned(), + "NO".to_owned(), + Some("'queued'::node_deletion_status".to_owned()), + )) + ); + ensure!( + shapes.get("node_deletion_jobs.completed_at") + == Some(&("timestamptz".to_owned(), "YES".to_owned(), None)) + ); + ensure!( + shapes.get("node_deployment_migrations.status") + == Some(&( + "node_deployment_migration_status".to_owned(), + "NO".to_owned(), + Some("'pending'::node_deployment_migration_status".to_owned()), + )) + ); + ensure!( + shapes.get("node_deployment_migrations.ready_at") + == Some(&("timestamptz".to_owned(), "YES".to_owned(), None)) + ); + + let constraints = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT conname, pg_get_constraintdef(oid) AS definition +FROM pg_constraint +WHERE conrelid IN ('node_deletion_jobs'::regclass, 'node_deployment_migrations'::regclass) +ORDER BY conname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "conname")?, + row.try_get::("", "definition")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + for expected in [ + "ck_node_deletion_jobs_distinct_target", + "ck_node_deletion_jobs_progress_nonnegative", + "ck_node_deletion_jobs_progress_bounded", + "ck_node_deletion_jobs_completed_at", + "ck_node_deployment_migrations_distinct_nodes", + "ck_node_deployment_migrations_ready_at", + "ux_node_deployment_migrations_job_deployment", + ] { + ensure!( + constraints.contains_key(expected), + "missing constraint {expected}" + ); + } + ensure!( + constraints.values().any( + |definition| definition.contains("FOREIGN KEY (target_node_id)") + && definition.contains("REFERENCES nodes(id) ON DELETE RESTRICT") + ), + "target Node foreign keys must prevent deleting an active migration target" + ); + ensure!( + constraints.values().any(|definition| definition + .contains("FOREIGN KEY (requested_by_user_id)") + && definition.contains("ON DELETE SET NULL")), + "requester foreign key must preserve deletion history" + ); + + let indexes = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT indexname, indexdef +FROM pg_indexes +WHERE schemaname = current_schema() + AND indexname IN ( +'ux_node_deletion_jobs_active_node', +'ix_node_deletion_jobs_queue', +'ix_node_deployment_migrations_target' + ) +ORDER BY indexname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "indexname")?, + row.try_get::("", "indexdef")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!(indexes.len() == 3, "expected three queue indexes"); + ensure!( + indexes["ux_node_deletion_jobs_active_node"].contains("UNIQUE INDEX") + && indexes["ux_node_deletion_jobs_active_node"] + .contains("status <> 'completed'::node_deletion_status") + ); + ensure!( + indexes["ix_node_deployment_migrations_target"] + .contains("'ready'::node_deployment_migration_status") + ); + Ok(()) +} + +async fn assert_node_deletion_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> { + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT + to_regclass('node_deletion_jobs') IS NULL AS jobs_absent, + to_regclass('node_deployment_migrations') IS NULL AS migrations_absent, + NOT EXISTS ( +SELECT 1 FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace +WHERE n.nspname = current_schema() + AND t.typname IN ('node_deletion_status', 'node_deployment_migration_status') + ) AS enums_absent +"#, + )) + .await? + .context("node deletion absence query returned no row")?; + ensure!(row.try_get::("", "jobs_absent")?); + ensure!(row.try_get::("", "migrations_absent")?); + ensure!(row.try_get::("", "enums_absent")?); + Ok(()) +} diff --git a/apps/control-api/src/infra/database/migrate/tests/notifications.rs b/apps/control-api/src/infra/database/migrate/tests/notifications.rs new file mode 100644 index 0000000..7967bac --- /dev/null +++ b/apps/control-api/src/infra/database/migrate/tests/notifications.rs @@ -0,0 +1,515 @@ +use std::collections::BTreeMap; + +use anyhow::{Context, ensure}; +use sea_orm::{ConnectionTrait, DatabaseBackend, DatabaseConnection, Statement}; +use sea_orm_migration::MigratorTrait; +use uuid::Uuid; + +use super::super::{MIGRATION_TEST_LOCK, Migrator}; +use super::support::{ + ColumnShape, PostgresMigrationDatabase, assert_migration_tracking, column, query_column_shapes, +}; + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL"] +async fn postgres_notification_and_announcement_schema_matches_the_domain_model_and_is_reversible() +-> anyhow::Result<()> { + let _migration_guard = MIGRATION_TEST_LOCK.lock().await; + let database_url = std::env::var("GRASS_TEST_DATABASE_URL") + .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); + let test_db = PostgresMigrationDatabase::start(&database_url).await?; + + let verification = async { + Migrator::up(&test_db.db, Some(19)).await?; + assert_migration_tracking(&test_db.db, 19).await?; + + Migrator::up(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 20).await?; + assert_notification_content_schema(&test_db.db).await?; + + Migrator::up(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 21).await?; + assert_announcement_schema(&test_db.db).await?; + + Migrator::down(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 20).await?; + assert_announcement_schema_absent(&test_db.db).await?; + assert_notification_content_schema(&test_db.db).await?; + + Migrator::up(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 21).await?; + assert_announcement_schema(&test_db.db).await + } + .await; + let cleanup = test_db.cleanup().await; + + match (verification, cleanup) { + (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context(format!( + "disposable schema cleanup also failed: {cleanup_error:#}" + ))), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Ok(()), Ok(())) => Ok(()), + } +} + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL"] +async fn postgres_project_notification_schema_backfills_and_is_reversible() -> anyhow::Result<()> { + let _migration_guard = MIGRATION_TEST_LOCK.lock().await; + let database_url = std::env::var("GRASS_TEST_DATABASE_URL") + .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test"); + let test_db = PostgresMigrationDatabase::start(&database_url).await?; + + let verification = async { + Migrator::up(&test_db.db, Some(16)).await?; + assert_migration_tracking(&test_db.db, 16).await?; + let (user_id, project_id) = seed_project_notification_fixture(&test_db.db).await?; + + Migrator::up(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 17).await?; + assert_project_notification_schema(&test_db.db, user_id, project_id).await?; + + Migrator::down(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 16).await?; + assert_project_notification_schema_absent(&test_db.db).await?; + + Migrator::up(&test_db.db, Some(1)).await?; + assert_migration_tracking(&test_db.db, 17).await?; + assert_project_notification_schema(&test_db.db, user_id, project_id).await + } + .await; + let cleanup = test_db.cleanup().await; + + match (verification, cleanup) { + (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context(format!( + "disposable schema cleanup also failed: {cleanup_error:#}" + ))), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Ok(()), Ok(())) => Ok(()), + } +} + +async fn seed_project_notification_fixture( + db: &DatabaseConnection, +) -> anyhow::Result<(Uuid, Uuid)> { + let user_id = Uuid::now_v7(); + let team_id = Uuid::now_v7(); + let project_id = Uuid::now_v7(); + let audit_id = Uuid::now_v7(); + db.execute_unprepared(&format!( + r#" +INSERT INTO users (id, email, display_name) +VALUES ('{user_id}'::uuid, 'notification-migration@example.invalid', 'Notification Migration'); + +INSERT INTO teams (id, slug, name, owner_user_id) +VALUES ('{team_id}'::uuid, 'notification-migration', 'Notification Migration', '{user_id}'::uuid); + +INSERT INTO projects (id, team_id, slug, name) +VALUES ('{project_id}'::uuid, '{team_id}'::uuid, 'notification-migration', 'Notification Migration'); + +INSERT INTO audit_events ( +id, +actor_user_id, +actor_type, +visibility, +action, +target_type, +target_id, +result, +metadata, +team_id +) +VALUES ( +'{audit_id}'::uuid, +'{user_id}'::uuid, +'user', +'team', +'project.created', +'project', +'{project_id}'::uuid, +'success', +'{{}}'::jsonb, +'{team_id}'::uuid +); +"# + )) + .await?; + Ok((user_id, project_id)) +} + +async fn assert_project_notification_schema( + db: &DatabaseConnection, + expected_creator_id: Uuid, + project_id: Uuid, +) -> anyhow::Result<()> { + let project_columns = query_column_shapes( + db, + r#" +SELECT column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'projects' + AND column_name = 'created_by_user_id' +"#, + ) + .await?; + ensure!( + project_columns + == vec![ColumnShape { + name: "created_by_user_id".to_owned(), + udt_name: "uuid".to_owned(), + nullable: "YES".to_owned(), + default: None, + }], + "unexpected Project creator column: {project_columns:?}" + ); + + let notification_columns = query_column_shapes( + db, + r#" +SELECT column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'user_notifications' +ORDER BY ordinal_position +"#, + ) + .await?; + ensure!( + notification_columns.len() == 13, + "expected 13 notification columns, found {}", + notification_columns.len() + ); + let shapes = notification_columns + .into_iter() + .map(|column| (column.name.clone(), column)) + .collect::>(); + ensure!( + shapes.get("recipient_user_id") + == Some(&ColumnShape { + name: "recipient_user_id".to_owned(), + udt_name: "uuid".to_owned(), + nullable: "NO".to_owned(), + default: None, + }) + ); + ensure!( + shapes.get("read_at") + == Some(&ColumnShape { + name: "read_at".to_owned(), + udt_name: "timestamptz".to_owned(), + nullable: "YES".to_owned(), + default: None, + }) + ); + + let constraints = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT conname, pg_get_constraintdef(oid) AS definition +FROM pg_constraint +WHERE conname IN ( +'fk_projects_created_by_user_id', +'fk_user_notifications_recipient_user_id', +'fk_user_notifications_actor_user_id', +'fk_user_notifications_team_id', +'fk_user_notifications_project_id' +) +ORDER BY conname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "conname")?, + row.try_get::("", "definition")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!(constraints.len() == 5, "missing notification foreign keys"); + ensure!(constraints["fk_user_notifications_recipient_user_id"].contains("ON DELETE CASCADE")); + for name in [ + "fk_projects_created_by_user_id", + "fk_user_notifications_actor_user_id", + "fk_user_notifications_team_id", + "fk_user_notifications_project_id", + ] { + ensure!( + constraints[name].contains("ON DELETE SET NULL"), + "{name} must preserve notification history" + ); + } + + let indexes = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT indexname, indexdef +FROM pg_indexes +WHERE schemaname = current_schema() + AND indexname IN ( +'ix_projects_created_by_user_id', +'ix_user_notifications_recipient_created', +'ix_user_notifications_recipient_unread' + ) +ORDER BY indexname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "indexname")?, + row.try_get::("", "indexdef")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!(indexes.len() == 3, "missing notification indexes"); + ensure!(indexes["ix_user_notifications_recipient_unread"].contains("read_at IS NULL")); + + let row = db + .query_one_raw(Statement::from_sql_and_values( + DatabaseBackend::Postgres, + "SELECT created_by_user_id FROM projects WHERE id = $1", + [project_id.into()], + )) + .await? + .context("Project creator backfill query returned no row")?; + ensure!( + row.try_get::("", "created_by_user_id")? == expected_creator_id, + "Project creator was not backfilled from the creation audit" + ); + Ok(()) +} + +async fn assert_project_notification_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> { + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT + to_regclass('user_notifications') IS NULL AS notifications_absent, + NOT EXISTS ( +SELECT 1 +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'projects' + AND column_name = 'created_by_user_id' + ) AS creator_absent +"#, + )) + .await? + .context("notification absence query returned no row")?; + ensure!(row.try_get::("", "notifications_absent")?); + ensure!(row.try_get::("", "creator_absent")?); + Ok(()) +} + +async fn assert_notification_content_schema(db: &DatabaseConnection) -> anyhow::Result<()> { + let columns = query_column_shapes( + db, + r#" +SELECT column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'user_notifications' + AND column_name IN ('project_name', 'project_slug', 'title', 'content') +ORDER BY ordinal_position +"#, + ) + .await?; + ensure!( + columns + == vec![ + ColumnShape { + name: "project_name".to_owned(), + udt_name: "text".to_owned(), + nullable: "YES".to_owned(), + default: None, + }, + ColumnShape { + name: "project_slug".to_owned(), + udt_name: "text".to_owned(), + nullable: "YES".to_owned(), + default: None, + }, + ColumnShape { + name: "title".to_owned(), + udt_name: "text".to_owned(), + nullable: "YES".to_owned(), + default: None, + }, + ColumnShape { + name: "content".to_owned(), + udt_name: "text".to_owned(), + nullable: "YES".to_owned(), + default: None, + }, + ], + "unexpected notification content columns: {columns:?}" + ); + + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT pg_get_constraintdef(oid) AS definition +FROM pg_constraint +WHERE conname = 'ck_user_notifications_announcement_content' +"#, + )) + .await? + .context("announcement content constraint was not created")?; + let definition = row.try_get::("", "definition")?; + ensure!(definition.contains("site.announcement")); + ensure!(definition.contains("team_id IS NULL")); + ensure!(definition.contains("project_id IS NULL")); + Ok(()) +} + +async fn assert_announcement_schema(db: &DatabaseConnection) -> anyhow::Result<()> { + let columns = query_column_shapes( + db, + r#" +SELECT column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'announcements' +ORDER BY ordinal_position +"#, + ) + .await?; + ensure!( + columns + == vec![ + column("id", "uuid", "NO", None), + column("title", "text", "NO", None), + column("content", "text", "NO", None), + column("auto_popup", "bool", "NO", Some("false")), + column("created_by_user_id", "uuid", "YES", None), + column("published_at", "timestamptz", "NO", None), + ], + "unexpected announcement columns: {columns:#?}" + ); + + let notification_columns = query_column_shapes( + db, + r#" +SELECT column_name, udt_name, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'user_notifications' + AND column_name = 'announcement_id' +"#, + ) + .await?; + ensure!( + notification_columns == vec![column("announcement_id", "uuid", "YES", None)], + "unexpected notification announcement column: {notification_columns:#?}" + ); + + let constraints = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT conname, pg_get_constraintdef(oid) AS definition +FROM pg_constraint +WHERE conname IN ( +'ck_announcements_title_length', +'ck_announcements_content_length', +'ck_user_notifications_announcement_content' +) +ORDER BY conname +"#, + )) + .await? + .into_iter() + .map(|row| { + Ok(( + row.try_get::("", "conname")?, + row.try_get::("", "definition")?, + )) + }) + .collect::, sea_orm::DbErr>>()?; + ensure!(constraints.len() == 3, "missing announcement constraints"); + ensure!(constraints["ck_announcements_title_length"].contains("120")); + ensure!(constraints["ck_announcements_content_length"].contains("10000")); + ensure!( + constraints["ck_user_notifications_announcement_content"] + .contains("announcement_id IS NOT NULL") + ); + + let foreign_keys = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT pg_get_constraintdef(oid) AS definition +FROM pg_constraint +WHERE contype = 'f' + AND conrelid IN ('announcements'::regclass, 'user_notifications'::regclass) +"#, + )) + .await? + .into_iter() + .map(|row| row.try_get::("", "definition")) + .collect::, sea_orm::DbErr>>()?; + ensure!( + foreign_keys + .iter() + .any(|definition| definition.contains("REFERENCES announcements") + && definition.contains("ON DELETE CASCADE")), + "notification announcement foreign key is not cascading" + ); + ensure!( + foreign_keys + .iter() + .any(|definition| definition.contains("REFERENCES users") + && definition.contains("ON DELETE SET NULL")), + "announcement creator foreign key is not nullable" + ); + + let indexes = db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT indexname +FROM pg_indexes +WHERE schemaname = current_schema() + AND indexname = 'ix_announcements_published_at' +"#, + )) + .await?; + ensure!(indexes.len() == 1, "announcement history index is missing"); + Ok(()) +} + +async fn assert_announcement_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> { + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + r#" +SELECT + to_regclass('announcements') IS NULL AS table_absent, + NOT EXISTS ( +SELECT 1 +FROM information_schema.columns +WHERE table_schema = current_schema() + AND table_name = 'user_notifications' + AND column_name = 'announcement_id' + ) AS notification_column_absent, + EXISTS ( +SELECT 1 +FROM pg_constraint +WHERE conname = 'ck_user_notifications_announcement_content' + ) AS legacy_constraint_present +"#, + )) + .await? + .context("announcement absence query returned no row")?; + ensure!(row.try_get::("", "table_absent")?); + ensure!(row.try_get::("", "notification_column_absent")?); + ensure!(row.try_get::("", "legacy_constraint_present")?); + Ok(()) +} diff --git a/apps/control-api/src/infra/database/migrate/tests/regions.rs b/apps/control-api/src/infra/database/migrate/tests/regions.rs new file mode 100644 index 0000000..d79df55 --- /dev/null +++ b/apps/control-api/src/infra/database/migrate/tests/regions.rs @@ -0,0 +1,416 @@ +use anyhow::ensure; +use sea_orm::{ConnectionTrait, DatabaseBackend, Statement}; +use sea_orm_migration::MigratorTrait; +use uuid::Uuid; + +use super::super::{MIGRATION_TEST_LOCK, Migrator}; +use super::support::{ + PostgresMigrationDatabase, assert_migration_tracking, object_count, query_column_shapes, +}; + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL"] +async fn postgres_region_catalog_backfills_and_enforces_references() -> anyhow::Result<()> { + let _guard = MIGRATION_TEST_LOCK.lock().await; + let database_url = std::env::var("GRASS_TEST_DATABASE_URL")?; + let database = PostgresMigrationDatabase::start(&database_url).await?; + let result: anyhow::Result<()> = async { + Migrator::up(&database.db, Some(32)).await?; + database + .db + .execute_unprepared( + "INSERT INTO regional_ingresses (id, region, hostname, created_at, \ + updated_at) VALUES ('00000000-0000-0000-0000-000000000101', 'hk_1', \ + 'hk.entry.example.com', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)", + ) + .await?; + Migrator::up(&database.db, Some(1)).await?; + let rows = database + .db + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + "SELECT code FROM regions ORDER BY code".to_owned(), + )) + .await?; + let codes = rows + .iter() + .map(|r| r.try_get::("", "code")) + .collect::, _>>()?; + ensure!(codes == vec!["default", "hk_1"]); + let foreign_keys = object_count( + &database.db, + "SELECT count(*) AS count FROM pg_constraint WHERE contype = 'f' AND \ + confrelid = 'regions'::regclass", + ) + .await?; + ensure!(foreign_keys == 5); + ensure!( + database + .db + .execute_unprepared("DELETE FROM regions WHERE code = 'hk_1'") + .await + .is_err() + ); + ensure!( + database + .db + .execute_unprepared("INSERT INTO regions (code, name) VALUES ('hk_1', 'duplicate')") + .await + .is_err() + ); + ensure!( + database + .db + .execute_unprepared( + "INSERT INTO regional_ingresses (id, region, hostname, created_at, \ + updated_at) VALUES ('00000000-0000-0000-0000-000000000102', 'unknown', \ + 'unknown.entry.example.com', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" + ) + .await + .is_err() + ); + ensure!( + database + .db + .execute_unprepared( + "INSERT INTO regional_ingresses (id, region, hostname, created_at, \ + updated_at) VALUES ('00000000-0000-0000-0000-000000000103', 'hk_1', \ + 'second.entry.example.com', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" + ) + .await + .is_err() + ); + let columns = query_column_shapes( + &database.db, + "SELECT column_name, udt_name, is_nullable, column_default FROM \ + information_schema.columns WHERE table_schema = current_schema() AND \ + table_name = 'regions'", + ) + .await?; + ensure!( + columns + .iter() + .any(|c| c.name == "code" && c.nullable == "NO" && c.udt_name == "text") + ); + ensure!( + columns + .iter() + .any(|c| c.name == "name" && c.nullable == "NO") + ); + Ok(()) + } + .await; + database.cleanup().await?; + result +} + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL"] +async fn postgres_domain_onboarding_migrates_and_checks_customer_dns() -> anyhow::Result<()> { + use sea_orm::{ActiveModelTrait, EntityTrait, Set}; + use serde_json::json; + + use crate::domain::{acme, certificate_settings, certificates, domain_onboarding}; + use crate::infra::database::entity::{ + HostBindingStatus, managed_certificate, project_host_binding as binding, regional_ingress, + }; + + let _guard = MIGRATION_TEST_LOCK.lock().await; + let database = + PostgresMigrationDatabase::start(&std::env::var("GRASS_TEST_DATABASE_URL")?).await?; + let result: anyhow::Result<()> = async { + let db = &database.db; + Migrator::up(db, Some(33)).await?; + let owner = Uuid::now_v7(); + let actor = Uuid::now_v7(); + let mut old = crate::test_support::certificates::binding_fixture(); + old.host = "legacy.example.org".into(); + let entry_id = Uuid::now_v7(); + db.execute_unprepared(&format!(r#" + INSERT INTO users (id, email, display_name) VALUES + ('{owner}', 'owner@example.org', 'Owner'), + ('{actor}', 'adder@example.org', 'Adding user'); + INSERT INTO teams (id, slug, name, owner_user_id) + VALUES ('{}', 'onboarding', 'Onboarding', '{owner}'); + INSERT INTO projects (id, team_id, slug, name, created_by_user_id) + VALUES ('{}', '{}', 'onboarding', 'Onboarding', '{owner}'); + INSERT INTO regions (code, name) VALUES ('eu', 'Europe'); + INSERT INTO regional_ingresses (id, region, hostname, created_at, updated_at) + VALUES ('{entry_id}', 'eu', 'entry.example.org', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP); + "#, old.team_id, old.project_id, old.team_id)).await?; + binding::ActiveModel::from(old.clone()).insert(db).await?; + db.execute_unprepared(&format!(r#" + INSERT INTO managed_certificates (id, ingress_id, hostname, issuer, generation) + VALUES ('{entry_id}', '{entry_id}', 'entry.example.org', 'letsencrypt', '{entry_id}'); + INSERT INTO managed_certificates ( + id, ingress_id, host_binding_id, hostname, issuer, generation, challenge_method + ) VALUES ( + '{0}', '{entry_id}', '{0}', 'legacy.example.org', 'letsencrypt', '{0}', 'dns01' + ); + "#, old.id)).await?; + Migrator::up(db, None).await?; + assert_migration_tracking(db, 35).await?; + ensure!( + managed_certificate::Entity::find_by_id(entry_id) + .one(db) + .await? + .is_none(), + "entry certificate must be removed" + ); + let legacy = managed_certificate::Entity::find_by_id(old.id) + .one(db) + .await? + .unwrap(); + ensure!(legacy.challenge_method == "http01" && legacy.contact_email == "owner@example.org"); + let entry_columns = query_column_shapes( + db, + "SELECT column_name, udt_name, is_nullable, column_default FROM \ + information_schema.columns WHERE table_schema = current_schema() AND \ + table_name = 'regional_ingresses'", + ) + .await?; + ensure!( + !entry_columns + .iter() + .any(|c| c.name.starts_with("certificate_") + || c.name.starts_with("dns_challenge") + || c.name == "tls_enabled" + || c.name == "acme_account") + ); + ensure!(entry_columns.iter().any(|c| c.name == "dns_checked_at" + && c.udt_name == "timestamptz" + && c.nullable == "YES" + && c.default.is_none())); + let columns = query_column_shapes( + db, + "SELECT column_name, udt_name, is_nullable, column_default FROM \ + information_schema.columns WHERE table_schema = current_schema() AND \ + table_name = 'domain_onboarding'", + ) + .await?; + for name in ["checked_at", "lease_until"] { + ensure!(columns.iter().any(|c| c.name == name + && c.udt_name == "timestamptz" + && c.nullable == "YES" + && c.default.is_none())); + } + ensure!( + columns + .iter() + .any(|c| c.name == "next_check_at" && c.nullable == "NO" && c.default.is_some()) + ); + ensure!( + columns + .iter() + .any(|c| c.name == "contact_email" && c.udt_name == "text" && c.nullable == "NO") + ); + ensure!( + object_count( + db, + "SELECT count(*) AS count FROM pg_constraint WHERE conrelid = \ + 'domain_onboarding'::regclass AND contype = 'f'" + ) + .await? + == 2 + ); + ensure!( + object_count( + db, + "SELECT count(*) AS count FROM pg_indexes WHERE schemaname = \ + current_schema() AND indexname = 'ix_domain_onboarding_due'" + ) + .await? + == 1 + ); + ensure!( + db.execute_unprepared("UPDATE managed_certificates SET host_binding_id = NULL") + .await + .is_err() + ); + ensure!( + db.execute_unprepared("UPDATE managed_certificates SET challenge_method = 'dns01'") + .await + .is_err() + ); + ensure!( + db.execute_unprepared("UPDATE domain_onboarding SET dns_status = 'invalid'") + .await + .is_err() + ); + let mut custom = old.clone(); + custom.id = Uuid::now_v7(); + custom.host = "site.example.org".into(); + custom.status = HostBindingStatus::Pending; + custom.ownership_status = "pending".into(); + binding::ActiveModel::from(custom.clone()) + .insert(db) + .await?; + domain_onboarding::create(db, &custom, actor).await?; + let contact = domain_onboarding::get(db, custom.id).await?.unwrap(); + ensure!( + contact.created_by_user_id == Some(actor) + && contact.contact_email == "adder@example.org" + ); + + let entry = regional_ingress::Entity::find_by_id(entry_id) + .one(db) + .await? + .unwrap(); + let token = + crate::domain::ingress::dns_verification_token("secret", custom.id, &custom.host); + for (address, txt, expected, active) in [ + (None, "wrong", "unresolved", false), + (Some("203.0.113.9"), token.as_str(), "mismatch", false), + (Some("203.0.113.1"), "wrong", "ready", false), + (Some("203.0.113.1"), token.as_str(), "ready", true), + ] { + let mut records = vec![ + ( + "entry.example.org", + "A", + crate::test_support::dns::answer("entry.example.org", 1, "203.0.113.1"), + ), + ( + "_grass.site.example.org", + "TXT", + crate::test_support::dns::answer( + "_grass.site.example.org", + 16, + &format!("\"{txt}\""), + ), + ), + ]; + if let Some(address) = address { + records.push(( + "site.example.org", + "A", + crate::test_support::dns::answer("site.example.org", 1, address), + )); + } + let (resolver, server) = crate::test_support::dns::fixture(records).await; + // Immediate checks and scheduled checks use the same persisted workflow. + domain_onboarding::run_check_with_resolver(db, custom.id, "secret", true, &resolver) + .await?; + server.abort(); + let check = domain_onboarding::get(db, custom.id).await?.unwrap(); + let bound = binding::Entity::find_by_id(custom.id) + .one(db) + .await? + .unwrap(); + ensure!( + check.dns_status == expected, + "unexpected DNS state: {}", + check.dns_status + ); + ensure!((bound.status == HostBindingStatus::Active) == active); + ensure!(check.lease_until.is_none()); + let interval = check.next_check_at - check.checked_at.unwrap(); + ensure!((60..=120).contains(&interval.whole_seconds())); + } + // The scheduler creates only customer-domain certificates, even without a deployment. + acme::sweep(db, "secret").await?; + let cert = managed_certificate::Entity::find_by_id(custom.id) + .one(db) + .await? + .unwrap(); + ensure!( + cert.hostname == custom.host + && cert.contact_email == "adder@example.org" + && cert.auto_renew + && cert.challenge_method == "http01" + ); + ensure!( + cert.status == "pending", + "no eligible entry nodes means no external order" + ); + ensure!( + managed_certificate::Entity::find_by_id(entry_id) + .one(db) + .await? + .is_none() + ); + let again = certificates::ensure_record(db, &entry, Some(&custom)).await?; + ensure!(again.id == cert.id && again.generation == cert.generation); + // Not-yet-due checks do not query DNS; leases also protect forced checks. + let (resolver, server) = crate::test_support::dns::fixture(vec![]).await; + let before = domain_onboarding::get(db, custom.id).await?.unwrap(); + domain_onboarding::run_check_with_resolver(db, custom.id, "secret", false, &resolver) + .await?; + ensure!(domain_onboarding::get(db, custom.id).await?.unwrap() == before); + db.execute_unprepared(&format!( + "UPDATE domain_onboarding SET lease_until = CURRENT_TIMESTAMP + INTERVAL '1 \ + minute' WHERE binding_id = '{}'", + custom.id + )) + .await?; + domain_onboarding::run_check_with_resolver(db, custom.id, "secret", true, &resolver) + .await?; + ensure!( + domain_onboarding::get(db, custom.id) + .await? + .unwrap() + .dns_status + == "ready" + ); + // Simulate a restart after an abandoned lease. The next scheduled check recovers. + db.execute_unprepared(&format!( + "UPDATE domain_onboarding SET lease_until = CURRENT_TIMESTAMP - INTERVAL '1 \ + second', next_check_at = CURRENT_TIMESTAMP - INTERVAL '1 second' WHERE \ + binding_id = '{}'", + custom.id + )) + .await?; + domain_onboarding::run_check_with_resolver(db, custom.id, "secret", false, &resolver) + .await?; + ensure!( + domain_onboarding::get(db, custom.id) + .await? + .unwrap() + .dns_status + == "entry_unavailable" + ); + server.abort(); + let settings = certificate_settings::CertificateSettings { + issuer: "zerossl".into(), + eab: json!({ + "eab_kid": "test-id", + "eab_hmac_key": "c2VjcmV0", + }), + }; + certificate_settings::save(db, &settings, "secret").await?; + ensure!(certificate_settings::issuer(db).await? == "zerossl"); + let loaded = certificate_settings::load(db, "secret").await?; + ensure!(loaded.eab == settings.eab); + let stored = crate::domain::settings::get_setting(db, "domain_https") + .await? + .unwrap(); + ensure!(!stored.value.to_string().contains("c2VjcmV0")); + let mut disabled: binding::ActiveModel = binding::Entity::find_by_id(custom.id) + .one(db) + .await? + .unwrap() + .into(); + disabled.status = Set(HostBindingStatus::Disabled); + disabled.update(db).await?; + let (resolver, server) = crate::test_support::dns::fixture(vec![]).await; + domain_onboarding::run_check_with_resolver(db, custom.id, "secret", true, &resolver) + .await?; + ensure!( + binding::Entity::find_by_id(custom.id) + .one(db) + .await? + .unwrap() + .status + == HostBindingStatus::Disabled + ); + server.abort(); + // Down/up restores the legacy shape, while reapplication produces the same new constraints. + Migrator::down(db, Some(2)).await?; + assert_migration_tracking(db, 33).await?; + Migrator::up(db, None).await?; + assert_migration_tracking(db, 35).await?; + Ok(()) + }.await; + database.cleanup().await?; + result +} diff --git a/apps/control-api/src/infra/database/migrate/tests/registration.rs b/apps/control-api/src/infra/database/migrate/tests/registration.rs new file mode 100644 index 0000000..94c77ab --- /dev/null +++ b/apps/control-api/src/infra/database/migrate/tests/registration.rs @@ -0,0 +1,193 @@ +use std::future::Future; + +use sea_orm_migration::MigratorTrait; +use tokio::sync::oneshot; + +use super::super::{MIGRATION_TEST_LOCK, Migrator, migration}; + +#[test] +fn registers_audit_foundation_migration() { + let migrations = Migrator::migrations(); + + assert_eq!(migrations.len(), 35); + assert_eq!( + migrations.get(11).expect("twelfth migration").name(), + "m20260729_000012_audit_foundation" + ); + + let sql = migration::m20260729_000012_audit_foundation::UP_SQL; + assert!(sql.contains("CREATE TYPE audit_actor_type")); + assert!(sql.contains("CREATE TYPE audit_event_visibility")); + assert!(sql.contains("ADD COLUMN request_id UUID NULL")); + assert!(sql.contains("ADD COLUMN changes JSONB NOT NULL DEFAULT '{}'")); + assert!( + sql.contains("ADD COLUMN pending_release_audit_visibility audit_event_visibility NULL") + ); + assert!(sql.contains("SET pending_release_audit_visibility = 'platform'")); + assert!(sql.contains("ck_deployments_pending_release_audit_visibility")); + assert!(sql.contains("actor_user_id IS NULL OR actor_type = 'user'")); + assert!(sql.contains("actor_node_id IS NULL OR actor_type = 'node'")); + assert!(sql.contains("actor_type NOT IN ('anonymous', 'system')")); + assert!(sql.contains("WHEN actor_user_id IS NOT NULL THEN 'user'")); + assert!(sql.contains("COALESCE(metadata ->> 'platform_admin', 'false') <> 'true'")); + assert!(sql.contains("COALESCE(metadata ->> 'completed_after_sync', 'false') <> 'true'")); + assert!(sql.contains("'team.quota_plan_overridden'")); +} + +#[test] +fn registers_team_group_review_policy_migration() { + let migrations = Migrator::migrations(); + + assert_eq!(migrations.len(), 35); + assert_eq!( + migrations.get(12).expect("thirteenth migration").name(), + "m20260729_000013_team_group_review_policy" + ); +} + +#[test] +fn registers_node_config_sync_migration() { + let migrations = Migrator::migrations(); + + assert_eq!(migrations.len(), 35); + assert_eq!( + migrations.get(13).expect("fourteenth migration").name(), + "m20260729_000014_node_config_sync" + ); +} + +#[test] +fn registers_node_deletion_queue_migration() { + let migrations = Migrator::migrations(); + + assert_eq!(migrations.len(), 35); + assert_eq!( + migrations.get(14).expect("fifteenth migration").name(), + "m20260729_000015_node_deletion_queue" + ); +} + +#[test] +fn registers_domain_review_policy_after_node_deletion_queue() { + let migrations = Migrator::migrations(); + + assert_eq!(migrations.len(), 35); + assert_eq!( + migrations.get(14).expect("fifteenth migration").name(), + "m20260729_000015_node_deletion_queue" + ); + assert_eq!( + migrations.get(15).expect("sixteenth migration").name(), + "m20260730_000016_domain_review_policy" + ); +} + +#[test] +fn registers_project_notifications_after_domain_review_policy() { + let migrations = Migrator::migrations(); + + assert_eq!(migrations.len(), 35); + assert_eq!( + migrations.get(15).expect("sixteenth migration").name(), + "m20260730_000016_domain_review_policy" + ); + assert_eq!( + migrations.get(16).expect("seventeenth migration").name(), + "m20260731_000017_project_notifications" + ); + assert_eq!( + migrations.get(17).expect("eighteenth migration").name(), + "m20260801_000018_artifact_retention" + ); + assert_eq!( + migrations.get(22).expect("twenty-third migration").name(), + "m20260804_000023_mfa_policy" + ); +} + +#[test] +fn registers_scoped_codes_after_authentication_migrations() { + let migrations = Migrator::migrations(); + + assert_eq!(migrations.len(), 35); + assert_eq!( + migrations.get(23).expect("twenty-fourth migration").name(), + "m20260806_000024_scoped_codes" + ); +} + +#[test] +fn registers_registration_allowlist_after_scoped_codes() { + let migrations = Migrator::migrations(); + + assert_eq!(migrations.len(), 35); + assert_eq!( + migrations.get(24).expect("twenty-fifth migration").name(), + "m20260806_000025_registration_allowlist" + ); +} + +#[test] +fn registers_avatar_versions_after_registration_allowlist() { + let migrations = Migrator::migrations(); + + assert_eq!(migrations.len(), 35); + assert_eq!( + migrations.get(25).expect("twenty-sixth migration").name(), + "m20260807_000026_avatars" + ); +} + +#[test] +fn registers_object_storage_after_deployment_screenshots() { + let migrations = Migrator::migrations(); + + assert_eq!(migrations.len(), 35); + assert_eq!( + migrations.get(26).expect("twenty-seventh migration").name(), + "m20260807_000027_deployment_screenshots" + ); + assert_eq!( + migrations.get(27).expect("twenty-eighth migration").name(), + "m20260808_000028_object_storage" + ); + assert_eq!( + migrations.last().expect("last migration").name(), + "m20260912_000035_user_auth_version" + ); +} + +#[tokio::test] +async fn shared_postgres_migration_lock_serializes_access() { + let (started_sender, started_receiver) = oneshot::channel(); + let (release_sender, release_receiver) = oneshot::channel(); + let first = tokio::spawn(async move { + let _guard = MIGRATION_TEST_LOCK.lock().await; + started_sender.send(()).unwrap(); + release_receiver.await.unwrap(); + }); + started_receiver.await.unwrap(); + + let (second_attempted_sender, second_attempted_receiver) = oneshot::channel(); + let (second_acquired_sender, mut second_acquired_receiver) = oneshot::channel(); + let second = tokio::spawn(async move { + let mut lock = Box::pin(MIGRATION_TEST_LOCK.lock()); + let mut attempted_sender = Some(second_attempted_sender); + let _guard = std::future::poll_fn(move |context| { + if let Some(sender) = attempted_sender.take() { + sender.send(()).unwrap(); + } + lock.as_mut().poll(context) + }) + .await; + second_acquired_sender.send(()).unwrap(); + }); + + second_attempted_receiver.await.unwrap(); + assert!(second_acquired_receiver.try_recv().is_err()); + + release_sender.send(()).unwrap(); + second.await.unwrap(); + assert!(second_acquired_receiver.await.is_ok()); + first.await.unwrap(); +} diff --git a/apps/control-api/src/infra/database/migrate/tests/revocation.rs b/apps/control-api/src/infra/database/migrate/tests/revocation.rs new file mode 100644 index 0000000..bdb168d --- /dev/null +++ b/apps/control-api/src/infra/database/migrate/tests/revocation.rs @@ -0,0 +1,354 @@ +use anyhow::{Context, ensure}; +use sea_orm::{ConnectionTrait, DatabaseBackend, Statement}; +use sea_orm_migration::MigratorTrait; +use uuid::Uuid; + +use super::super::{MIGRATION_TEST_LOCK, Migrator}; +use super::support::{ + PostgresMigrationDatabase, assert_migration_tracking, column, query_column_shapes, +}; + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL and disposable schema permission"] +async fn postgres_auth_version_shape_and_password_revocation() -> anyhow::Result<()> { + postgres_account_revocation(grass_cache::CacheStore::Moka( + grass_cache::MokaCache::connect(), + )) + .await +} + +#[tokio::test] +#[ignore = "requires GRASS_TEST_DATABASE_URL, GRASS_TEST_REDIS_URL and disposable schema permission"] +async fn postgres_redis_auth_version_shape_and_password_revocation() -> anyhow::Result<()> { + postgres_account_revocation(grass_cache::CacheStore::Redis( + grass_cache::RedisCache::connect(&std::env::var("GRASS_TEST_REDIS_URL")?).await?, + )) + .await +} + +async fn postgres_account_revocation(cache_store: grass_cache::CacheStore) -> anyhow::Result<()> { + use std::time::Duration; + + use axum::{Router, body::Body, http::Request, middleware, routing::get}; + use grass_cache::Cache; + use tower::ServiceExt; + + use crate::{ + domain::{authentication, users}, + infra::{ + config::ControlApiConfig, + database::entity::{AuthTokenKind, PlatformRole, UserStatus}, + http::{extractors::Session, middlewares::session}, + }, + state::ControlApiState, + }; + + let _guard = MIGRATION_TEST_LOCK.lock().await; + let database = + PostgresMigrationDatabase::start(&std::env::var("GRASS_TEST_DATABASE_URL")?).await?; + let result: anyhow::Result<()> = async { + let db = &database.db; + Migrator::up(db, None).await?; + assert_migration_tracking(db, 35).await?; + let shapes = query_column_shapes( + db, + "SELECT column_name, udt_name, is_nullable, column_default FROM \ + information_schema.columns WHERE table_schema = current_schema() AND \ + table_name = 'users' AND column_name = 'auth_version'", + ) + .await?; + ensure!( + shapes == vec![column("auth_version", "int8", "NO", Some("1"))], + "incorrect authentication version column shape" + ); + let constraint = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Postgres, + "SELECT pg_get_constraintdef(oid) AS definition FROM pg_constraint WHERE \ + conrelid = 'users'::regclass AND conname = 'users_auth_version_check'", + )) + .await? + .unwrap(); + ensure!( + constraint + .try_get::("", "definition")? + .contains("auth_version > 0") + ); + let user = users::create_user( + db, + users::CreateUserParams { + email: format!("revocation-{}@example.test", Uuid::now_v7()), + display_name: None, + password_hash: Some(grass_crypto::hash_password("Original-password-123!")?), + platform_role: PlatformRole::Admin, + email_verified_at: Some(time::OffsetDateTime::now_utc()), + }, + ) + .await?; + ensure!(user.auth_version == 1); + let state = ControlApiState::new(ControlApiConfig::default(), "unused.toml"); + state.database.set(db.clone()).ok().unwrap(); + state.cache.set(cache_store).ok().unwrap(); + async fn protected(_session: Session) -> &'static str { + "allowed" + } + async fn admin(_admin: crate::infra::http::extractors::PlatformAdmin) -> &'static str { + "allowed" + } + let app = Router::new() + .route("/protected", get(protected).post(protected)) + .route("/admin", get(admin)) + .nest( + "/api/v1/auth", + crate::features::api::v1::auth::login::router() + .merge(crate::features::api::v1::auth::password::reset::router()), + ) + .nest("/api/v1", crate::features::api::v1::me::password::router()) + .nest( + "/api/v1/admin", + crate::features::api::v1::admin::users::by_user_id::reset_password::router(), + ) + .layer(middleware::from_fn_with_state( + state.clone(), + session::session_middleware, + )) + .with_state(state.clone()); + let cache = state.try_cache().unwrap(); + let mut current_password = "Original-password-123!"; + for (flow, next_password) in [ + ("change", "Changed-password-123!"), + ("reset", "Reset-password-123!"), + ("admin", "Admin-reset-password-123!"), + ] { + let current = users::get_user_by_id(db, user.id).await?.unwrap(); + let first = grass_session::create_session( + cache, + user.id, + current.auth_version, + Duration::from_secs(300), + ) + .await?; + let second = grass_session::create_session( + cache, + user.id, + current.auth_version, + Duration::from_secs(300), + ) + .await?; + for sid in [&first, &second] { + ensure!( + session::validate_current_session(&state, sid, "test.active") + .await? + .is_some() + ); + } + // A refresh may read an old session before the password transaction commits. + let key = format!("session:{second}"); + let stale_refresh = cache.get(&key).await?.unwrap(); + let (uri, body) = match flow { + "change" => ( + "/api/v1/me/password".into(), + serde_json::json!({ + "current_password": current_password, + "password": next_password, + }), + ), + "reset" => { + let token = authentication::create_auth_token( + db, + user.id, + AuthTokenKind::PasswordReset, + time::Duration::hours(1), + ) + .await?; + ( + "/api/v1/auth/password/reset".into(), + serde_json::json!({ + "token": token, + "password": next_password, + }), + ) + } + _ => ( + format!("/api/v1/admin/users/{}/reset-password", user.id), + serde_json::json!({ "password": next_password }), + ), + }; + let response = app + .clone() + .oneshot( + Request::builder() + .uri(uri) + .method("POST") + .header("content-type", "application/json") + .header("cookie", format!("session_id={first}")) + .body(Body::from(body.to_string()))?, + ) + .await?; + ensure!( + response.status().is_success(), + "password flow {flow} failed with {}", + response.status() + ); + let updated = users::get_user_by_id(db, user.id).await?.unwrap(); + ensure!(updated.auth_version == current.auth_version + 1); + // Complete that delayed cache write after revocation; DB state must still win. + ensure!( + cache + .update_if_present(&key, &stale_refresh, Duration::from_secs(300)) + .await? + ); + for (sid, method, path) in [ + (&first, "GET", "/protected"), + (&second, "POST", "/protected"), + (&second, "GET", "/admin"), + ] { + let response = app + .clone() + .oneshot( + Request::builder() + .uri(path) + .method(method) + .header("cookie", format!("session_id={sid}")) + .body(Body::empty())?, + ) + .await?; + ensure!( + response.status().as_u16() == 401, + "old session survived {flow}" + ); + } + ensure!( + users::verify_user_password(db, &user.email, next_password) + .await? + .is_some() + ); + ensure!( + users::verify_user_password(db, &user.email, current_password) + .await? + .is_none() + ); + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/api/v1/auth/login") + .method("POST") + .extension(axum::extract::ConnectInfo(std::net::SocketAddr::from(( + [127, 0, 0, 1], + 12345, + )))) + .header("content-type", "application/json") + .body(Body::from( + serde_json::json!({ + "email": user.email, + "password": next_password, + }) + .to_string(), + ))?, + ) + .await?; + ensure!( + response.status().is_success(), + "new password could not log in after {flow}" + ); + let sid = response + .headers() + .get_all("set-cookie") + .iter() + .filter_map(|cookie| cookie.to_str().ok()) + .find_map(|cookie| { + cookie + .strip_prefix("session_id=") + .and_then(|value| value.split(';').next()) + }) + .context("login did not issue a session")?; + ensure!( + session::validate_current_session(&state, sid, "test.login") + .await? + .is_some() + ); + grass_session::revoke_session(cache, sid).await?; + current_password = next_password; + } + let current = users::get_user_by_id(db, user.id).await?.unwrap(); + let first = grass_session::create_session( + cache, + user.id, + current.auth_version, + Duration::from_secs(300), + ) + .await?; + let second = grass_session::create_session( + cache, + user.id, + current.auth_version, + Duration::from_secs(300), + ) + .await?; + ensure!( + session::validate_current_session(&state, &first, "test.before-disable") + .await? + .is_some() + ); + let disabled = users::update_user( + db, + current.clone(), + users::UpdateUserParams { + display_name: None, + status: Some(UserStatus::Disabled), + platform_role: None, + }, + ) + .await?; + ensure!(disabled.auth_version == current.auth_version + 1); + ensure!( + session::validate_current_session(&state, &first, "test.disabled") + .await? + .is_none() + ); + let enabled = users::update_user( + db, + disabled, + users::UpdateUserParams { + display_name: None, + status: Some(UserStatus::Active), + platform_role: None, + }, + ) + .await?; + ensure!(enabled.auth_version == current.auth_version + 1); + ensure!( + session::validate_current_session(&state, &second, "test.reenabled") + .await? + .is_none() + ); + let fresh = grass_session::create_session( + cache, + user.id, + enabled.auth_version, + Duration::from_secs(300), + ) + .await?; + ensure!( + session::validate_current_session(&state, &fresh, "test.enabled") + .await? + .is_some() + ); + db.execute_raw(Statement::from_sql_and_values( + DatabaseBackend::Postgres, + "UPDATE users SET deleted_at = CURRENT_TIMESTAMP WHERE id = $1", + [user.id.into()], + )) + .await?; + ensure!( + session::validate_current_session(&state, &fresh, "test.deleted") + .await? + .is_none() + ); + Ok(()) + } + .await; + database.cleanup().await?; + result +} diff --git a/apps/control-api/src/infra/database/migrate/tests/support.rs b/apps/control-api/src/infra/database/migrate/tests/support.rs new file mode 100644 index 0000000..502a9a9 --- /dev/null +++ b/apps/control-api/src/infra/database/migrate/tests/support.rs @@ -0,0 +1,160 @@ +use std::collections::BTreeMap; + +use anyhow::{Context, ensure}; +use sea_orm::{ConnectionTrait, Database, DatabaseBackend, DatabaseConnection, Statement}; +use sea_orm_migration::MigratorTrait; +use uuid::Uuid; + +use super::super::Migrator; + +#[derive(Debug, Eq, PartialEq)] +pub(super) struct ColumnShape { + pub(super) name: String, + pub(super) udt_name: String, + pub(super) nullable: String, + pub(super) default: Option, +} + +pub(super) struct PostgresMigrationDatabase { + pub(super) db: DatabaseConnection, + admin: DatabaseConnection, + schema: String, +} + +impl PostgresMigrationDatabase { + pub(super) async fn start(database_url: &str) -> anyhow::Result { + let admin = Database::connect(database_url).await?; + let schema = format!("gw_audit_migration_{}", Uuid::now_v7().simple()); + admin + .execute_unprepared(&format!("CREATE SCHEMA {schema}")) + .await?; + + let mut scoped_url = url::Url::parse(database_url)?; + scoped_url + .query_pairs_mut() + .append_pair("options", &format!("-csearch_path={schema}")); + let db = match Database::connect(scoped_url.as_str()).await { + Ok(db) => db, + Err(error) => { + admin + .execute_unprepared(&format!("DROP SCHEMA {schema} CASCADE")) + .await?; + return Err(error.into()); + } + }; + + Ok(Self { db, admin, schema }) + } + + pub(super) async fn cleanup(self) -> anyhow::Result<()> { + self.db.close().await?; + self.admin + .execute_unprepared(&format!("DROP SCHEMA {} CASCADE", self.schema)) + .await?; + self.admin.close().await?; + Ok(()) + } +} + +pub(super) async fn assert_migration_tracking( + db: &DatabaseConnection, + applied_count: usize, +) -> anyhow::Result<()> { + let applied = Migrator::get_applied_migrations(db).await?; + let pending = Migrator::get_pending_migrations(db).await?; + // Historical shape tests stop at their target migration. Later migrations + // remain pending even when that historical phase is fully applied. + let pending_count = Migrator::migrations().len() - applied_count; + + ensure!( + applied.len() == applied_count, + "expected {applied_count} applied migrations, found {}", + applied.len() + ); + ensure!( + pending.len() == pending_count, + "expected {pending_count} pending migrations, found {}", + pending.len() + ); + if applied_count >= 12 { + ensure!( + applied.get(11).map(|migration| migration.name()) + == Some("m20260729_000012_audit_foundation"), + "audit foundation migration was not the twelfth applied migration" + ); + } + if pending_count > 0 { + let expected = Migrator::migrations() + .get(applied_count) + .map(|migration| migration.name().to_owned()); + ensure!( + pending.first().map(|migration| migration.name()) == expected.as_deref(), + "migration tracking did not expose the next registered migration first" + ); + } + + Ok(()) +} + +pub(super) fn column( + name: &str, + udt_name: &str, + nullable: &str, + default: Option<&str>, +) -> ColumnShape { + ColumnShape { + name: name.to_owned(), + udt_name: udt_name.to_owned(), + nullable: nullable.to_owned(), + default: default.map(str::to_owned), + } +} + +pub(super) async fn query_column_shapes( + db: &DatabaseConnection, + sql: &str, +) -> anyhow::Result> { + db.query_all_raw(Statement::from_string(DatabaseBackend::Postgres, sql)) + .await? + .into_iter() + .map(|row| { + Ok(ColumnShape { + name: row.try_get::("", "column_name")?, + udt_name: row.try_get::("", "udt_name")?, + nullable: row.try_get::("", "is_nullable")?, + default: row.try_get::>("", "column_default")?, + }) + }) + .collect::, sea_orm::DbErr>>() + .map_err(Into::into) +} + +pub(super) fn ensure_index( + indexes: &BTreeMap, + name: &str, + columns: &str, + predicate: Option<&str>, +) -> anyhow::Result<()> { + let definition = indexes + .get(name) + .with_context(|| format!("missing index {name}"))?; + ensure!( + definition.contains(columns), + "index {name} has unexpected columns: {definition}" + ); + if let Some(predicate) = predicate { + ensure!( + definition.contains(predicate), + "index {name} has unexpected predicate: {definition}" + ); + } + Ok(()) +} + +pub(super) async fn object_count(db: &DatabaseConnection, sql: &str) -> anyhow::Result { + db.query_one_raw(Statement::from_string(DatabaseBackend::Postgres, sql)) + .await? + .context("count query returned no row")? + .try_get::("", "count") + .map_err(Into::into) +} diff --git a/apps/control-api/src/infra/database/migration/mod.rs b/apps/control-api/src/infra/database/migration/mod.rs index 124db0f..fea29a0 100644 --- a/apps/control-api/src/infra/database/migration/mod.rs +++ b/apps/control-api/src/infra/database/migration/mod.rs @@ -30,9 +30,6 @@ pub mod m20260908_000029_regional_routing; pub mod m20260908_000030_regional_ingress; pub mod m20260909_000031_regional_ingress_lifecycle; pub mod m20260910_000032_managed_certificates; - pub mod m20260911_000033_regions; - pub mod m20260911_000034_domain_onboarding; - pub mod m20260912_000035_user_auth_version; diff --git a/apps/control-api/src/infra/node_manager/config_file.rs b/apps/control-api/src/infra/node_manager/config_file.rs index ccfd02e..12d428d 100644 --- a/apps/control-api/src/infra/node_manager/config_file.rs +++ b/apps/control-api/src/infra/node_manager/config_file.rs @@ -169,14 +169,14 @@ fn prepare_directories(work_root: &str, artifact_cache_root: &str) -> Vec RuntimeSection { let default_build_image = "docker.io/library/node:22".to_owned(); - if let Ok(host) = std::env::var("DOCKER_HOST") { - if host.starts_with("unix://") { - return RuntimeSection { - backend: "docker-socket".to_owned(), - socket: host, - default_build_image, - }; - } + if let Ok(host) = std::env::var("DOCKER_HOST") + && host.starts_with("unix://") + { + return RuntimeSection { + backend: "docker-socket".to_owned(), + socket: host, + default_build_image, + }; } if Path::new("/var/run/docker.sock").exists() { return RuntimeSection { diff --git a/apps/control-api/src/infra/route_invalidation.rs b/apps/control-api/src/infra/route_invalidation.rs index ec13f43..a18cff5 100644 --- a/apps/control-api/src/infra/route_invalidation.rs +++ b/apps/control-api/src/infra/route_invalidation.rs @@ -32,10 +32,10 @@ async fn invalidate_at_urls( endpoint.set_query(None); endpoint.set_fragment(None); let mut request = client.post(endpoint).header("x-grass-gateway-hop", "1"); - if matches!(gateway_authentication, GatewayAuthenticationMode::Token) { - if let Some(gateway_token) = gateway_token { - request = request.header("x-grass-gateway-token", gateway_token); - } + if matches!(gateway_authentication, GatewayAuthenticationMode::Token) + && let Some(gateway_token) = gateway_token + { + request = request.header("x-grass-gateway-token", gateway_token); } let response = request .timeout(Duration::from_secs(3)) diff --git a/apps/control-api/src/infra/storage/config.rs b/apps/control-api/src/infra/storage/config.rs new file mode 100644 index 0000000..721ac86 --- /dev/null +++ b/apps/control-api/src/infra/storage/config.rs @@ -0,0 +1,190 @@ +//! Storage provider configuration and credential validation. + +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +use super::StorageError; + +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum StorageBackendKind { + #[default] + Local, + S3, + Minio, + R2, +} + +impl StorageBackendKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Local => "local", + Self::S3 => "s3", + Self::Minio => "minio", + Self::R2 => "r2", + } + } + + pub fn parse(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "local" => Some(Self::Local), + "s3" => Some(Self::S3), + "minio" => Some(Self::Minio), + "r2" => Some(Self::R2), + _ => None, + } + } + + pub fn default_region(self) -> &'static str { + match self { + Self::R2 => "auto", + Self::Local | Self::S3 | Self::Minio => "us-east-1", + } + } +} + +impl std::str::FromStr for StorageBackendKind { + type Err = StorageError; + + fn from_str(value: &str) -> Result { + Self::parse(value).ok_or_else(|| { + StorageError::InvalidConfig(format!("unsupported storage backend: {value}")) + }) + } +} + +#[derive(Debug, Clone, Deserialize, Eq, PartialEq, Serialize)] +pub struct StorageConfig { + #[serde(default)] + pub backend: StorageBackendKind, + #[serde(default = "default_local_root")] + pub local_root: String, + #[serde(default)] + pub endpoint: String, + #[serde(default = "default_region")] + pub region: String, + #[serde(default)] + pub bucket: String, + #[serde(default)] + pub prefix: String, + #[serde(default)] + pub force_path_style: bool, + #[serde(default)] + pub allow_http: bool, +} + +impl Default for StorageConfig { + fn default() -> Self { + Self { + backend: StorageBackendKind::Local, + local_root: default_local_root(), + endpoint: String::new(), + region: default_region(), + bucket: String::new(), + prefix: String::new(), + force_path_style: false, + allow_http: false, + } + } +} + +impl StorageConfig { + pub fn local(root: impl Into) -> Self { + Self { + local_root: root.into(), + ..Self::default() + } + } + + pub fn validate(&self) -> Result<(), StorageError> { + let root = self.local_root.trim(); + if root.is_empty() || !Path::new(root).is_absolute() { + return Err(StorageError::InvalidConfig( + "local_root must be a non-empty absolute path".to_owned(), + )); + } + match self.backend { + StorageBackendKind::Local => {} + StorageBackendKind::S3 | StorageBackendKind::Minio | StorageBackendKind::R2 => { + if self.bucket.trim().is_empty() { + return Err(StorageError::InvalidConfig( + "bucket is required for an S3-compatible backend".to_owned(), + )); + } + if self.region.trim().is_empty() { + return Err(StorageError::InvalidConfig( + "region is required for an S3-compatible backend".to_owned(), + )); + } + if matches!( + self.backend, + StorageBackendKind::Minio | StorageBackendKind::R2 + ) && self.endpoint.trim().is_empty() + { + return Err(StorageError::InvalidConfig(format!( + "endpoint is required for the {} backend", + self.backend.as_str() + ))); + } + if let Some(endpoint) = + (!self.endpoint.trim().is_empty()).then_some(self.endpoint.trim()) + { + let parsed = url::Url::parse(endpoint).map_err(|error| { + StorageError::InvalidConfig(format!("endpoint is invalid: {error}")) + })?; + if !matches!(parsed.scheme(), "http" | "https") { + return Err(StorageError::InvalidConfig( + "endpoint must use http or https".to_owned(), + )); + } + if parsed.scheme() == "http" && !self.allow_http { + return Err(StorageError::InvalidConfig( + "allow_http must be enabled for an http endpoint".to_owned(), + )); + } + } + } + } + + if self.prefix.split('/').any(|part| part == "..") { + return Err(StorageError::UnsafePath(self.prefix.clone())); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct StorageCredentials { + pub access_key_id: Option, + pub secret_access_key: Option, + pub session_token: Option, +} + +impl StorageCredentials { + pub fn is_configured(&self) -> bool { + self.access_key_id + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + || self + .secret_access_key + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + || self + .session_token + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + } +} + +fn default_local_root() -> String { + "/data".to_owned() +} + +fn default_region() -> String { + "us-east-1".to_owned() +} + +#[cfg(test)] +#[path = "tests/config.rs"] +mod tests; diff --git a/apps/control-api/src/infra/storage/local.rs b/apps/control-api/src/infra/storage/local.rs new file mode 100644 index 0000000..ccb0739 --- /dev/null +++ b/apps/control-api/src/infra/storage/local.rs @@ -0,0 +1,245 @@ +//! Filesystem storage with atomic writes and bounded path resolution. + +use std::path::{Path, PathBuf}; + +use async_trait::async_trait; +use futures_util::StreamExt; +use sha2::{Digest, Sha256}; +use tokio::io::AsyncWriteExt; +use tokio_util::io::ReaderStream; +use uuid::Uuid; + +use super::{ + ObjectStorage, ObjectStream, OpenedArtifact, OpenedObject, StorageError, StoredArtifact, + StoredObjectMeta, +}; + +#[derive(Clone)] +pub struct LocalStorage { + root: PathBuf, +} + +impl LocalStorage { + pub fn new(root: impl Into) -> Self { + Self { root: root.into() } + } + + pub fn artifact_relative_path(project_id: Uuid, deployment_id: Uuid) -> String { + format!("deployments/{project_id}/{deployment_id}/grass-output.zip") + } + + pub fn build_log_relative_path(project_id: Uuid, deployment_id: Uuid) -> String { + format!("deployments/{project_id}/{deployment_id}/build.log") + } + + fn resolve(&self, relative: &str) -> Result { + let relative_path = Path::new(relative); + if relative_path.is_absolute() + || relative_path.components().any(|component| { + matches!( + component, + std::path::Component::ParentDir | std::path::Component::Prefix(_) + ) + }) + { + return Err(StorageError::UnsafePath(relative.to_owned())); + } + Ok(self.root.join(relative_path)) + } + + pub async fn write_bytes( + &self, + relative_path: &str, + content: &[u8], + ) -> Result { + let final_path = self.resolve(relative_path)?; + let directory = final_path + .parent() + .ok_or_else(|| StorageError::UnsafePath(relative_path.to_owned()))?; + tokio::fs::create_dir_all(directory).await?; + let temporary_path = directory.join(format!(".write-{}.tmp", Uuid::now_v7().simple())); + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary_path) + .await?; + let result = async { + file.write_all(content).await?; + file.flush().await?; + file.sync_all().await?; + tokio::fs::rename(&temporary_path, &final_path).await?; + Ok::<(), std::io::Error>(()) + } + .await; + if let Err(error) = result { + drop(file); + let _ = tokio::fs::remove_file(&temporary_path).await; + return Err(error.into()); + } + Ok(StoredArtifact { + relative_path: relative_path.to_owned(), + size_bytes: i64::try_from(content.len()).map_err(|_| StorageError::UnsupportedSize)?, + checksum_sha256: hex::encode(Sha256::digest(content)), + }) + } + + async fn put_stream_to_key( + &self, + key: &str, + mut stream: ObjectStream, + ) -> Result<(), StorageError> { + let final_path = self.resolve(key)?; + let directory = final_path + .parent() + .ok_or_else(|| StorageError::UnsafePath(key.to_owned()))?; + tokio::fs::create_dir_all(directory).await?; + let temporary_path = directory.join(format!(".stream-{}.tmp", Uuid::now_v7().simple())); + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary_path) + .await?; + let result = async { + while let Some(chunk) = stream.next().await { + file.write_all(&chunk?).await?; + } + file.flush().await?; + file.sync_all().await?; + tokio::fs::rename(&temporary_path, &final_path).await?; + Ok::<(), StorageError>(()) + } + .await; + if let Err(error) = result { + drop(file); + let _ = tokio::fs::remove_file(&temporary_path).await; + return Err(error); + } + Ok(()) + } + + pub async fn open_artifact( + &self, + relative_path: &str, + ) -> anyhow::Result> { + let path = self.resolve(relative_path)?; + let file = match tokio::fs::File::open(path).await { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + let metadata = file.metadata().await?; + if !metadata.is_file() { + anyhow::bail!("artifact path is not a regular file"); + } + Ok(Some(OpenedArtifact { + file, + size_bytes: metadata.len(), + })) + } + + pub async fn remove(&self, relative_path: &str) -> anyhow::Result<()> { + let path = self.resolve(relative_path)?; + match tokio::fs::remove_file(path).await { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + } +} + +#[async_trait] +impl ObjectStorage for LocalStorage { + async fn put_bytes(&self, key: &str, content: &[u8]) -> Result<(), StorageError> { + self.write_bytes(key, content).await.map(|_| ()) + } + + async fn put_stream(&self, key: &str, stream: ObjectStream) -> Result<(), StorageError> { + self.put_stream_to_key(key, stream).await + } + + async fn open(&self, key: &str) -> Result, StorageError> { + let Some(opened) = self + .open_artifact(key) + .await + .map_err(|error| StorageError::Backend(error.to_string()))? + else { + return Ok(None); + }; + let stream = ReaderStream::new(opened.file) + .map(|chunk| chunk.map_err(StorageError::Io)) + .boxed(); + Ok(Some(OpenedObject { + stream, + size_bytes: opened.size_bytes, + })) + } + + async fn remove(&self, key: &str) -> Result<(), StorageError> { + self.remove(key) + .await + .map_err(|error| StorageError::Backend(error.to_string())) + } + + async fn rename(&self, from: &str, to: &str) -> Result<(), StorageError> { + let source = self.resolve(from)?; + let target = self.resolve(to)?; + if let Some(parent) = target.parent() { + tokio::fs::create_dir_all(parent).await?; + } + match tokio::fs::rename(source, target).await { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Err( + StorageError::Backend(format!("source object not found while renaming {from}")), + ), + Err(error) => Err(error.into()), + } + } + + async fn list(&self, prefix: &str) -> Result, StorageError> { + let root = self.resolve(prefix)?; + let base = self.root.clone(); + tokio::task::spawn_blocking(move || { + if !root.exists() { + return Ok(Vec::new()); + } + let mut result = Vec::new(); + for entry in walkdir::WalkDir::new(root) { + let entry = entry.map_err(|error| StorageError::Backend(error.to_string()))?; + if !entry.file_type().is_file() { + continue; + } + let key = entry + .path() + .strip_prefix(&base) + .map_err(|error| StorageError::Backend(error.to_string()))? + .to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/"); + let size_bytes = entry + .metadata() + .map_err(|error| StorageError::Backend(error.to_string()))? + .len(); + result.push(StoredObjectMeta { key, size_bytes }); + } + Ok(result) + }) + .await + .map_err(|error| StorageError::Backend(error.to_string()))? + } + + async fn probe(&self) -> Result<(), StorageError> { + let key = format!(".probe/{}", Uuid::now_v7().simple()); + self.put_bytes(&key, b"grass-storage-probe").await?; + let result = self.open(&key).await?; + ObjectStorage::remove(self, &key).await?; + if result.is_none() { + return Err(StorageError::Backend( + "storage probe could not read its object".to_owned(), + )); + } + Ok(()) + } +} + +#[cfg(test)] +#[path = "tests/local.rs"] +mod tests; diff --git a/apps/control-api/src/infra/storage/manager.rs b/apps/control-api/src/infra/storage/manager.rs new file mode 100644 index 0000000..da19057 --- /dev/null +++ b/apps/control-api/src/infra/storage/manager.rs @@ -0,0 +1,319 @@ +//! Backend switching, maintenance leases and pending artifact writes. + +use std::{ + collections::HashMap, + path::PathBuf, + sync::{ + Arc, RwLock, Weak, + atomic::{AtomicBool, Ordering}, + }, +}; + +use bytes::Bytes; +use futures_util::Stream; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use super::{ + LocalStorage, MAX_BUILD_LOG_BYTES, ObjectStorage, OpenedObject, StorageConfig, + StorageCredentials, StorageError, StoredArtifact, build_backend, + streams::{StreamStats, TrackingStream, read_object_limited}, + validate_key, +}; + +#[derive(Clone)] +struct StorageRuntime { + config: StorageConfig, + backend: Arc, +} + +#[derive(Clone)] +pub struct StorageManager { + runtime: Arc>, + write_gate: Arc>, + maintenance: Arc, + log_append_locks: Arc>>>>, +} + +pub(crate) struct StorageWriteGuard { + _guard: tokio::sync::OwnedRwLockReadGuard<()>, + backend: Arc, +} + +impl StorageWriteGuard { + pub(crate) async fn write_bytes( + &self, + key: &str, + content: &[u8], + ) -> Result { + validate_key(key)?; + self.backend.put_bytes(key, content).await?; + Ok(StoredArtifact { + relative_path: key.to_owned(), + size_bytes: i64::try_from(content.len()).map_err(|_| StorageError::UnsupportedSize)?, + checksum_sha256: hex::encode(Sha256::digest(content)), + }) + } + + pub(crate) async fn remove(&self, key: &str) -> Result<(), StorageError> { + validate_key(key)?; + self.backend.remove(key).await + } +} + +impl StorageManager { + pub fn build_log_relative_path(project_id: Uuid, deployment_id: Uuid) -> String { + LocalStorage::build_log_relative_path(project_id, deployment_id) + } + + pub fn new_local(root: impl Into) -> Self { + let root = root.into(); + let backend: Arc = Arc::new(LocalStorage::new(root.clone())); + Self::from_runtime(StorageRuntime { + config: StorageConfig::local(root.to_string_lossy()), + backend, + }) + } + + fn from_runtime(runtime: StorageRuntime) -> Self { + Self { + runtime: Arc::new(RwLock::new(runtime)), + write_gate: Arc::new(tokio::sync::RwLock::new(())), + maintenance: Arc::new(AtomicBool::new(false)), + log_append_locks: Arc::new(std::sync::Mutex::new(HashMap::new())), + } + } + + pub fn config(&self) -> StorageConfig { + self.runtime.read().unwrap().config.clone() + } + + pub fn backend(&self) -> Arc { + Arc::clone(&self.runtime.read().unwrap().backend) + } + + pub fn replace( + &self, + config: StorageConfig, + credentials: StorageCredentials, + ) -> Result<(), StorageError> { + let backend = build_backend(&config, &credentials)?; + self.replace_backend(config, backend); + Ok(()) + } + + pub fn replace_backend(&self, config: StorageConfig, backend: Arc) { + let mut runtime = self.runtime.write().unwrap(); + runtime.config = config; + runtime.backend = backend; + } + + pub fn is_maintenance(&self) -> bool { + self.maintenance.load(Ordering::Acquire) + } + + pub fn mark_maintenance(&self) { + self.maintenance.store(true, Ordering::Release); + } + + pub async fn enter_maintenance(&self) -> tokio::sync::OwnedRwLockWriteGuard<()> { + self.mark_maintenance(); + self.write_gate.clone().write_owned().await + } + + pub fn leave_maintenance(&self) { + self.maintenance.store(false, Ordering::Release); + } + + async fn write_lock(&self) -> Result, StorageError> { + if self.is_maintenance() { + return Err(StorageError::Maintenance); + } + let guard = self.write_gate.clone().read_owned().await; + if self.is_maintenance() { + drop(guard); + return Err(StorageError::Maintenance); + } + Ok(guard) + } + + pub(crate) async fn begin_write(&self) -> Result { + let guard = self.write_lock().await?; + Ok(StorageWriteGuard { + _guard: guard, + backend: self.backend(), + }) + } + + pub async fn write_bytes( + &self, + key: &str, + content: &[u8], + ) -> Result { + self.begin_write().await?.write_bytes(key, content).await + } + + pub async fn write_artifact_stream( + &self, + project_id: Uuid, + deployment_id: Uuid, + stream: S, + max_bytes: u64, + ) -> Result + where + S: Stream> + Send + 'static, + E: std::error::Error + Send + Sync + 'static, + { + let write_guard = self.write_lock().await?; + let final_key = LocalStorage::artifact_relative_path(project_id, deployment_id); + let temporary_key = format!(".pending/{}", Uuid::now_v7().simple()); + let stats = Arc::new(std::sync::Mutex::new(StreamStats::default())); + let tracked = TrackingStream::new(stream, max_bytes, Arc::clone(&stats)); + let backend = self.backend(); + if let Err(error) = backend.put_stream(&temporary_key, Box::pin(tracked)).await { + let _ = backend.remove(&temporary_key).await; + return Err(error); + } + let stats = stats.lock().unwrap().clone(); + Ok(PendingArtifact { + storage: backend, + temporary_key: Some(temporary_key), + final_key, + size_bytes: i64::try_from(stats.size_bytes) + .map_err(|_| StorageError::UnsupportedSize)?, + checksum_sha256: hex::encode(stats.hasher.finalize()), + write_guard: Some(write_guard), + }) + } + + pub async fn open_artifact(&self, key: &str) -> Result, StorageError> { + validate_key(key)?; + self.backend().open(key).await + } + + pub async fn remove(&self, key: &str) -> Result<(), StorageError> { + self.begin_write().await?.remove(key).await + } + + pub async fn append_build_log( + &self, + project_id: Uuid, + deployment_id: Uuid, + content: &str, + ) -> Result<(), StorageError> { + let lock_key = format!("{project_id}:{deployment_id}"); + let append_lock = self.log_append_lock(&lock_key); + let _append_guard = append_lock.lock().await; + let _write_guard = self.write_lock().await?; + let content_size = + u64::try_from(content.len()).map_err(|_| StorageError::UnsupportedSize)?; + if content_size > MAX_BUILD_LOG_BYTES { + return Err(StorageError::LimitExceeded { + max_bytes: MAX_BUILD_LOG_BYTES, + }); + } + let key = LocalStorage::build_log_relative_path(project_id, deployment_id); + let backend = self.backend(); + let mut current = match backend.open(&key).await? { + Some(object) => { + if object.size_bytes > MAX_BUILD_LOG_BYTES.saturating_sub(content_size) { + return Err(StorageError::LimitExceeded { + max_bytes: MAX_BUILD_LOG_BYTES, + }); + } + read_object_limited(object, MAX_BUILD_LOG_BYTES - content_size).await? + } + None => Vec::new(), + }; + if u64::try_from(current.len()) + .ok() + .and_then(|size| size.checked_add(content_size)) + .is_none_or(|size| size > MAX_BUILD_LOG_BYTES) + { + return Err(StorageError::LimitExceeded { + max_bytes: MAX_BUILD_LOG_BYTES, + }); + } + current.extend_from_slice(content.as_bytes()); + backend.put_bytes(&key, ¤t).await + } + + pub async fn read_build_log( + &self, + project_id: Uuid, + deployment_id: Uuid, + ) -> Result, StorageError> { + let key = LocalStorage::build_log_relative_path(project_id, deployment_id); + let Some(object) = self.backend().open(&key).await? else { + return Ok(None); + }; + let bytes = read_object_limited(object, MAX_BUILD_LOG_BYTES).await?; + String::from_utf8(bytes) + .map(Some) + .map_err(|error| StorageError::Backend(format!("build log is not UTF-8: {error}"))) + } + + fn log_append_lock(&self, key: &str) -> Arc> { + let mut locks = self.log_append_locks.lock().unwrap(); + locks.retain(|_, lock| lock.strong_count() > 0); + if let Some(lock) = locks.get(key).and_then(Weak::upgrade) { + return lock; + } + let lock = Arc::new(tokio::sync::Mutex::new(())); + locks.insert(key.to_owned(), Arc::downgrade(&lock)); + lock + } +} + +pub struct PendingArtifact { + storage: Arc, + temporary_key: Option, + final_key: String, + pub size_bytes: i64, + pub checksum_sha256: String, + write_guard: Option>, +} + +impl PendingArtifact { + pub async fn finalize(mut self) -> Result { + let temporary_key = self + .temporary_key + .take() + .expect("pending artifact must own a temporary object"); + if let Err(error) = self.storage.rename(&temporary_key, &self.final_key).await { + let _ = self.storage.remove(&temporary_key).await; + return Err(error); + } + self.write_guard.take(); + Ok(StoredArtifact { + relative_path: self.final_key.clone(), + size_bytes: self.size_bytes, + checksum_sha256: self.checksum_sha256.clone(), + }) + } + + pub async fn discard(mut self) { + if let Some(temporary_key) = self.temporary_key.take() { + let _ = self.storage.remove(&temporary_key).await; + } + self.write_guard.take(); + } +} + +impl Drop for PendingArtifact { + fn drop(&mut self) { + let Some(temporary_key) = self.temporary_key.take() else { + return; + }; + let storage = Arc::clone(&self.storage); + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + runtime.spawn(async move { + let _ = storage.remove(&temporary_key).await; + }); + } + } +} + +#[cfg(test)] +#[path = "tests/manager.rs"] +mod tests; diff --git a/apps/control-api/src/infra/storage/mod.rs b/apps/control-api/src/infra/storage/mod.rs index bda0e68..c235f0d 100644 --- a/apps/control-api/src/infra/storage/mod.rs +++ b/apps/control-api/src/infra/storage/mod.rs @@ -4,28 +4,25 @@ //! concrete providers are deliberately kept behind this module so adding a //! provider does not change artifact, avatar, screenshot, or log handlers. -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - pin::Pin, - sync::{ - Arc, RwLock, Weak, - atomic::{AtomicBool, Ordering}, - }, -}; +mod config; +mod local; +mod manager; +mod s3; +mod streams; +mod transfer; + +pub use config::{StorageBackendKind, StorageConfig, StorageCredentials}; +pub use local::LocalStorage; +pub(crate) use manager::StorageWriteGuard; +pub use manager::{PendingArtifact, StorageManager}; +pub use s3::S3Storage; +pub use transfer::{copy_and_verify, list_managed_backend}; + +use std::{path::Path, sync::Arc}; use async_trait::async_trait; use bytes::Bytes; -use futures_util::{Stream, StreamExt, TryStreamExt, stream::BoxStream}; -use object_store::{ - ObjectStore as ApacheObjectStore, ObjectStoreExt, PutPayload, aws::AmazonS3Builder, - buffered::BufWriter, path::Path as ObjectPath, prefix::PrefixStore, -}; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use tokio::io::AsyncWriteExt; -use tokio_util::io::ReaderStream; -use uuid::Uuid; +use futures_util::stream::BoxStream; pub type ObjectStream = BoxStream<'static, Result>; @@ -51,177 +48,6 @@ pub enum StorageError { Io(#[from] std::io::Error), } -#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum StorageBackendKind { - #[default] - Local, - S3, - Minio, - R2, -} - -impl StorageBackendKind { - pub fn as_str(self) -> &'static str { - match self { - Self::Local => "local", - Self::S3 => "s3", - Self::Minio => "minio", - Self::R2 => "r2", - } - } - - pub fn parse(value: &str) -> Option { - match value.trim().to_ascii_lowercase().as_str() { - "local" => Some(Self::Local), - "s3" => Some(Self::S3), - "minio" => Some(Self::Minio), - "r2" => Some(Self::R2), - _ => None, - } - } - - pub fn default_region(self) -> &'static str { - match self { - Self::R2 => "auto", - Self::Local | Self::S3 | Self::Minio => "us-east-1", - } - } -} - -impl std::str::FromStr for StorageBackendKind { - type Err = StorageError; - - fn from_str(value: &str) -> Result { - Self::parse(value).ok_or_else(|| { - StorageError::InvalidConfig(format!("unsupported storage backend: {value}")) - }) - } -} - -#[derive(Debug, Clone, Deserialize, Eq, PartialEq, Serialize)] -pub struct StorageConfig { - #[serde(default)] - pub backend: StorageBackendKind, - #[serde(default = "default_local_root")] - pub local_root: String, - #[serde(default)] - pub endpoint: String, - #[serde(default = "default_region")] - pub region: String, - #[serde(default)] - pub bucket: String, - #[serde(default)] - pub prefix: String, - #[serde(default)] - pub force_path_style: bool, - #[serde(default)] - pub allow_http: bool, -} - -impl Default for StorageConfig { - fn default() -> Self { - Self { - backend: StorageBackendKind::Local, - local_root: default_local_root(), - endpoint: String::new(), - region: default_region(), - bucket: String::new(), - prefix: String::new(), - force_path_style: false, - allow_http: false, - } - } -} - -impl StorageConfig { - pub fn local(root: impl Into) -> Self { - Self { - local_root: root.into(), - ..Self::default() - } - } - - pub fn validate(&self) -> Result<(), StorageError> { - let root = self.local_root.trim(); - if root.is_empty() || !Path::new(root).is_absolute() { - return Err(StorageError::InvalidConfig( - "local_root must be a non-empty absolute path".to_owned(), - )); - } - match self.backend { - StorageBackendKind::Local => {} - StorageBackendKind::S3 | StorageBackendKind::Minio | StorageBackendKind::R2 => { - if self.bucket.trim().is_empty() { - return Err(StorageError::InvalidConfig( - "bucket is required for an S3-compatible backend".to_owned(), - )); - } - if self.region.trim().is_empty() { - return Err(StorageError::InvalidConfig( - "region is required for an S3-compatible backend".to_owned(), - )); - } - if matches!( - self.backend, - StorageBackendKind::Minio | StorageBackendKind::R2 - ) && self.endpoint.trim().is_empty() - { - return Err(StorageError::InvalidConfig(format!( - "endpoint is required for the {} backend", - self.backend.as_str() - ))); - } - if let Some(endpoint) = - (!self.endpoint.trim().is_empty()).then_some(self.endpoint.trim()) - { - let parsed = url::Url::parse(endpoint).map_err(|error| { - StorageError::InvalidConfig(format!("endpoint is invalid: {error}")) - })?; - if !matches!(parsed.scheme(), "http" | "https") { - return Err(StorageError::InvalidConfig( - "endpoint must use http or https".to_owned(), - )); - } - if parsed.scheme() == "http" && !self.allow_http { - return Err(StorageError::InvalidConfig( - "allow_http must be enabled for an http endpoint".to_owned(), - )); - } - } - } - } - - if self.prefix.split('/').any(|part| part == "..") { - return Err(StorageError::UnsafePath(self.prefix.clone())); - } - Ok(()) - } -} - -#[derive(Debug, Clone, Default, Deserialize, Eq, PartialEq, Serialize)] -pub struct StorageCredentials { - pub access_key_id: Option, - pub secret_access_key: Option, - pub session_token: Option, -} - -impl StorageCredentials { - pub fn is_configured(&self) -> bool { - self.access_key_id - .as_deref() - .is_some_and(|value| !value.trim().is_empty()) - || self - .secret_access_key - .as_deref() - .is_some_and(|value| !value.trim().is_empty()) - || self - .session_token - .as_deref() - .is_some_and(|value| !value.trim().is_empty()) - } -} - #[derive(Debug, Clone, Eq, PartialEq)] pub struct StoredArtifact { pub relative_path: String, @@ -258,398 +84,6 @@ pub trait ObjectStorage: Send + Sync { async fn probe(&self) -> Result<(), StorageError>; } -#[derive(Clone)] -pub struct LocalStorage { - root: PathBuf, -} - -impl LocalStorage { - pub fn new(root: impl Into) -> Self { - Self { root: root.into() } - } - - pub fn artifact_relative_path(project_id: Uuid, deployment_id: Uuid) -> String { - format!("deployments/{project_id}/{deployment_id}/grass-output.zip") - } - - pub fn build_log_relative_path(project_id: Uuid, deployment_id: Uuid) -> String { - format!("deployments/{project_id}/{deployment_id}/build.log") - } - - fn resolve(&self, relative: &str) -> Result { - let relative_path = Path::new(relative); - if relative_path.is_absolute() - || relative_path.components().any(|component| { - matches!( - component, - std::path::Component::ParentDir | std::path::Component::Prefix(_) - ) - }) - { - return Err(StorageError::UnsafePath(relative.to_owned())); - } - Ok(self.root.join(relative_path)) - } - - pub async fn write_bytes( - &self, - relative_path: &str, - content: &[u8], - ) -> Result { - let final_path = self.resolve(relative_path)?; - let directory = final_path - .parent() - .ok_or_else(|| StorageError::UnsafePath(relative_path.to_owned()))?; - tokio::fs::create_dir_all(directory).await?; - let temporary_path = directory.join(format!(".write-{}.tmp", Uuid::now_v7().simple())); - let mut file = tokio::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&temporary_path) - .await?; - let result = async { - file.write_all(content).await?; - file.flush().await?; - file.sync_all().await?; - tokio::fs::rename(&temporary_path, &final_path).await?; - Ok::<(), std::io::Error>(()) - } - .await; - if let Err(error) = result { - drop(file); - let _ = tokio::fs::remove_file(&temporary_path).await; - return Err(error.into()); - } - Ok(StoredArtifact { - relative_path: relative_path.to_owned(), - size_bytes: i64::try_from(content.len()).map_err(|_| StorageError::UnsupportedSize)?, - checksum_sha256: hex::encode(Sha256::digest(content)), - }) - } - - async fn put_stream_to_key( - &self, - key: &str, - mut stream: ObjectStream, - ) -> Result<(), StorageError> { - let final_path = self.resolve(key)?; - let directory = final_path - .parent() - .ok_or_else(|| StorageError::UnsafePath(key.to_owned()))?; - tokio::fs::create_dir_all(directory).await?; - let temporary_path = directory.join(format!(".stream-{}.tmp", Uuid::now_v7().simple())); - let mut file = tokio::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&temporary_path) - .await?; - let result = async { - while let Some(chunk) = stream.next().await { - file.write_all(&chunk?).await?; - } - file.flush().await?; - file.sync_all().await?; - tokio::fs::rename(&temporary_path, &final_path).await?; - Ok::<(), StorageError>(()) - } - .await; - if let Err(error) = result { - drop(file); - let _ = tokio::fs::remove_file(&temporary_path).await; - return Err(error); - } - Ok(()) - } - - pub async fn open_artifact( - &self, - relative_path: &str, - ) -> anyhow::Result> { - let path = self.resolve(relative_path)?; - let file = match tokio::fs::File::open(path).await { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(error.into()), - }; - let metadata = file.metadata().await?; - if !metadata.is_file() { - anyhow::bail!("artifact path is not a regular file"); - } - Ok(Some(OpenedArtifact { - file, - size_bytes: metadata.len(), - })) - } - - pub async fn remove(&self, relative_path: &str) -> anyhow::Result<()> { - let path = self.resolve(relative_path)?; - match tokio::fs::remove_file(path).await { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error.into()), - } - } -} - -#[async_trait] -impl ObjectStorage for LocalStorage { - async fn put_bytes(&self, key: &str, content: &[u8]) -> Result<(), StorageError> { - self.write_bytes(key, content).await.map(|_| ()) - } - - async fn put_stream(&self, key: &str, stream: ObjectStream) -> Result<(), StorageError> { - self.put_stream_to_key(key, stream).await - } - - async fn open(&self, key: &str) -> Result, StorageError> { - let Some(opened) = self - .open_artifact(key) - .await - .map_err(|error| StorageError::Backend(error.to_string()))? - else { - return Ok(None); - }; - let stream = ReaderStream::new(opened.file) - .map(|chunk| chunk.map_err(StorageError::Io)) - .boxed(); - Ok(Some(OpenedObject { - stream, - size_bytes: opened.size_bytes, - })) - } - - async fn remove(&self, key: &str) -> Result<(), StorageError> { - self.remove(key) - .await - .map_err(|error| StorageError::Backend(error.to_string())) - } - - async fn rename(&self, from: &str, to: &str) -> Result<(), StorageError> { - let source = self.resolve(from)?; - let target = self.resolve(to)?; - if let Some(parent) = target.parent() { - tokio::fs::create_dir_all(parent).await?; - } - match tokio::fs::rename(source, target).await { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Err( - StorageError::Backend(format!("source object not found while renaming {from}")), - ), - Err(error) => Err(error.into()), - } - } - - async fn list(&self, prefix: &str) -> Result, StorageError> { - let root = self.resolve(prefix)?; - let base = self.root.clone(); - tokio::task::spawn_blocking(move || { - if !root.exists() { - return Ok(Vec::new()); - } - let mut result = Vec::new(); - for entry in walkdir::WalkDir::new(root) { - let entry = entry.map_err(|error| StorageError::Backend(error.to_string()))?; - if !entry.file_type().is_file() { - continue; - } - let key = entry - .path() - .strip_prefix(&base) - .map_err(|error| StorageError::Backend(error.to_string()))? - .to_string_lossy() - .replace(std::path::MAIN_SEPARATOR, "/"); - let size_bytes = entry - .metadata() - .map_err(|error| StorageError::Backend(error.to_string()))? - .len(); - result.push(StoredObjectMeta { key, size_bytes }); - } - Ok(result) - }) - .await - .map_err(|error| StorageError::Backend(error.to_string()))? - } - - async fn probe(&self) -> Result<(), StorageError> { - let key = format!(".probe/{}", Uuid::now_v7().simple()); - self.put_bytes(&key, b"grass-storage-probe").await?; - let result = self.open(&key).await?; - ObjectStorage::remove(self, &key).await?; - if result.is_none() { - return Err(StorageError::Backend( - "storage probe could not read its object".to_owned(), - )); - } - Ok(()) - } -} - -#[derive(Clone)] -pub struct S3Storage { - store: Arc, -} - -impl S3Storage { - pub fn new( - config: &StorageConfig, - credentials: &StorageCredentials, - ) -> Result { - config.validate()?; - let mut builder = AmazonS3Builder::from_env() - .with_bucket_name(config.bucket.trim()) - .with_region(config.region.trim()) - .with_allow_http(config.allow_http) - .with_virtual_hosted_style_request(!config.force_path_style); - if !config.endpoint.trim().is_empty() { - builder = builder.with_endpoint(config.endpoint.trim()); - } - if let Some(value) = credentials - .access_key_id - .as_deref() - .filter(|value| !value.trim().is_empty()) - { - builder = builder.with_access_key_id(value); - } - if let Some(value) = credentials - .secret_access_key - .as_deref() - .filter(|value| !value.trim().is_empty()) - { - builder = builder.with_secret_access_key(value); - } - if let Some(value) = credentials - .session_token - .as_deref() - .filter(|value| !value.trim().is_empty()) - { - builder = builder.with_token(value); - } - let store = builder - .build() - .map_err(|error| StorageError::Backend(error.to_string()))?; - let store: Arc = if config.prefix.trim().is_empty() { - Arc::new(store) - } else { - Arc::new(PrefixStore::new( - store, - ObjectPath::from(config.prefix.trim()), - )) - }; - Ok(Self { store }) - } - - fn path(&self, key: &str) -> Result { - validate_key(key)?; - ObjectPath::parse(key).map_err(|error| StorageError::UnsafePath(error.to_string())) - } -} - -#[async_trait] -impl ObjectStorage for S3Storage { - async fn put_bytes(&self, key: &str, content: &[u8]) -> Result<(), StorageError> { - self.store - .put( - &self.path(key)?, - PutPayload::from(Bytes::copy_from_slice(content)), - ) - .await - .map(|_| ()) - .map_err(|error| StorageError::Backend(error.to_string())) - } - - async fn put_stream(&self, key: &str, mut stream: ObjectStream) -> Result<(), StorageError> { - let mut writer = - BufWriter::with_capacity(Arc::clone(&self.store), self.path(key)?, 10 * 1024 * 1024); - while let Some(chunk) = stream.next().await { - let chunk = match chunk { - Ok(chunk) => chunk, - Err(error) => { - let _ = writer.abort().await; - return Err(error); - } - }; - match writer.put(chunk).await { - Ok(()) => {} - Err(error) => { - let _ = writer.abort().await; - return Err(StorageError::Backend(error.to_string())); - } - } - } - writer - .shutdown() - .await - .map_err(|error| StorageError::Backend(error.to_string())) - } - - async fn open(&self, key: &str) -> Result, StorageError> { - let path = self.path(key)?; - let result = match self.store.get(&path).await { - Ok(result) => result, - Err(object_store::Error::NotFound { .. }) => return Ok(None), - Err(error) => return Err(StorageError::Backend(error.to_string())), - }; - let size_bytes = result.meta.size; - let stream = result - .into_stream() - .map(|chunk| chunk.map_err(|error| StorageError::Backend(error.to_string()))) - .boxed(); - Ok(Some(OpenedObject { stream, size_bytes })) - } - - async fn remove(&self, key: &str) -> Result<(), StorageError> { - match self.store.delete(&self.path(key)?).await { - Ok(()) | Err(object_store::Error::NotFound { .. }) => Ok(()), - Err(error) => Err(StorageError::Backend(error.to_string())), - } - } - - async fn rename(&self, from: &str, to: &str) -> Result<(), StorageError> { - self.store - .rename(&self.path(from)?, &self.path(to)?) - .await - .map_err(|error| StorageError::Backend(error.to_string())) - } - - async fn list(&self, prefix: &str) -> Result, StorageError> { - let prefix = if prefix.trim().is_empty() { - None - } else { - Some(self.path(prefix)?) - }; - self.store - .list(prefix.as_ref()) - .map_ok(|meta| StoredObjectMeta { - key: meta.location.to_string(), - size_bytes: meta.size, - }) - .try_collect() - .await - .map_err(|error| StorageError::Backend(error.to_string())) - } - - async fn probe(&self) -> Result<(), StorageError> { - let key = format!(".probe/{}", Uuid::now_v7().simple()); - self.put_bytes(&key, b"grass-storage-probe").await?; - let result = self.open(&key).await?; - self.remove(&key).await?; - if result.is_none() { - return Err(StorageError::Backend( - "storage probe could not read its object".to_owned(), - )); - } - Ok(()) - } -} - -fn default_local_root() -> String { - "/data".to_owned() -} - -fn default_region() -> String { - "us-east-1".to_owned() -} - fn validate_key(key: &str) -> Result<(), StorageError> { let path = Path::new(key); if key.trim().is_empty() @@ -666,274 +100,6 @@ fn validate_key(key: &str) -> Result<(), StorageError> { Ok(()) } -#[derive(Clone)] -struct StorageRuntime { - config: StorageConfig, - backend: Arc, -} - -#[derive(Clone)] -pub struct StorageManager { - runtime: Arc>, - write_gate: Arc>, - maintenance: Arc, - log_append_locks: Arc>>>>, -} - -pub(crate) struct StorageWriteGuard { - _guard: tokio::sync::OwnedRwLockReadGuard<()>, - backend: Arc, -} - -impl StorageWriteGuard { - pub(crate) async fn write_bytes( - &self, - key: &str, - content: &[u8], - ) -> Result { - validate_key(key)?; - self.backend.put_bytes(key, content).await?; - Ok(StoredArtifact { - relative_path: key.to_owned(), - size_bytes: i64::try_from(content.len()).map_err(|_| StorageError::UnsupportedSize)?, - checksum_sha256: hex::encode(Sha256::digest(content)), - }) - } - - pub(crate) async fn remove(&self, key: &str) -> Result<(), StorageError> { - validate_key(key)?; - self.backend.remove(key).await - } -} - -impl StorageManager { - pub fn build_log_relative_path(project_id: Uuid, deployment_id: Uuid) -> String { - LocalStorage::build_log_relative_path(project_id, deployment_id) - } - - pub fn new_local(root: impl Into) -> Self { - let root = root.into(); - let backend: Arc = Arc::new(LocalStorage::new(root.clone())); - Self::from_runtime(StorageRuntime { - config: StorageConfig::local(root.to_string_lossy()), - backend, - }) - } - - fn from_runtime(runtime: StorageRuntime) -> Self { - Self { - runtime: Arc::new(RwLock::new(runtime)), - write_gate: Arc::new(tokio::sync::RwLock::new(())), - maintenance: Arc::new(AtomicBool::new(false)), - log_append_locks: Arc::new(std::sync::Mutex::new(HashMap::new())), - } - } - - pub fn config(&self) -> StorageConfig { - self.runtime.read().unwrap().config.clone() - } - - pub fn backend(&self) -> Arc { - Arc::clone(&self.runtime.read().unwrap().backend) - } - - pub fn replace( - &self, - config: StorageConfig, - credentials: StorageCredentials, - ) -> Result<(), StorageError> { - let backend = build_backend(&config, &credentials)?; - self.replace_backend(config, backend); - Ok(()) - } - - pub fn replace_backend(&self, config: StorageConfig, backend: Arc) { - let mut runtime = self.runtime.write().unwrap(); - runtime.config = config; - runtime.backend = backend; - } - - pub fn is_maintenance(&self) -> bool { - self.maintenance.load(Ordering::Acquire) - } - - pub fn mark_maintenance(&self) { - self.maintenance.store(true, Ordering::Release); - } - - pub async fn enter_maintenance(&self) -> tokio::sync::OwnedRwLockWriteGuard<()> { - self.mark_maintenance(); - self.write_gate.clone().write_owned().await - } - - pub fn leave_maintenance(&self) { - self.maintenance.store(false, Ordering::Release); - } - - async fn write_lock(&self) -> Result, StorageError> { - if self.is_maintenance() { - return Err(StorageError::Maintenance); - } - let guard = self.write_gate.clone().read_owned().await; - if self.is_maintenance() { - drop(guard); - return Err(StorageError::Maintenance); - } - Ok(guard) - } - - pub(crate) async fn begin_write(&self) -> Result { - let guard = self.write_lock().await?; - Ok(StorageWriteGuard { - _guard: guard, - backend: self.backend(), - }) - } - - pub async fn write_bytes( - &self, - key: &str, - content: &[u8], - ) -> Result { - self.begin_write().await?.write_bytes(key, content).await - } - - pub async fn write_artifact_stream( - &self, - project_id: Uuid, - deployment_id: Uuid, - stream: S, - max_bytes: u64, - ) -> Result - where - S: Stream> + Send + 'static, - E: std::error::Error + Send + Sync + 'static, - { - let write_guard = self.write_lock().await?; - let final_key = LocalStorage::artifact_relative_path(project_id, deployment_id); - let temporary_key = format!(".pending/{}", Uuid::now_v7().simple()); - let stats = Arc::new(std::sync::Mutex::new(StreamStats::default())); - let tracked = TrackingStream::new(stream, max_bytes, Arc::clone(&stats)); - let backend = self.backend(); - if let Err(error) = backend.put_stream(&temporary_key, Box::pin(tracked)).await { - let _ = backend.remove(&temporary_key).await; - return Err(error); - } - let stats = stats.lock().unwrap().clone(); - Ok(PendingArtifact { - storage: backend, - temporary_key: Some(temporary_key), - final_key, - size_bytes: i64::try_from(stats.size_bytes) - .map_err(|_| StorageError::UnsupportedSize)?, - checksum_sha256: hex::encode(stats.hasher.finalize()), - write_guard: Some(write_guard), - }) - } - - pub async fn open_artifact(&self, key: &str) -> Result, StorageError> { - validate_key(key)?; - self.backend().open(key).await - } - - pub async fn remove(&self, key: &str) -> Result<(), StorageError> { - self.begin_write().await?.remove(key).await - } - - pub async fn append_build_log( - &self, - project_id: Uuid, - deployment_id: Uuid, - content: &str, - ) -> Result<(), StorageError> { - let lock_key = format!("{project_id}:{deployment_id}"); - let append_lock = self.log_append_lock(&lock_key); - let _append_guard = append_lock.lock().await; - let _write_guard = self.write_lock().await?; - let content_size = - u64::try_from(content.len()).map_err(|_| StorageError::UnsupportedSize)?; - if content_size > MAX_BUILD_LOG_BYTES { - return Err(StorageError::LimitExceeded { - max_bytes: MAX_BUILD_LOG_BYTES, - }); - } - let key = LocalStorage::build_log_relative_path(project_id, deployment_id); - let backend = self.backend(); - let mut current = match backend.open(&key).await? { - Some(object) => { - if object.size_bytes > MAX_BUILD_LOG_BYTES.saturating_sub(content_size) { - return Err(StorageError::LimitExceeded { - max_bytes: MAX_BUILD_LOG_BYTES, - }); - } - read_object_limited(object, MAX_BUILD_LOG_BYTES - content_size).await? - } - None => Vec::new(), - }; - if u64::try_from(current.len()) - .ok() - .and_then(|size| size.checked_add(content_size)) - .is_none_or(|size| size > MAX_BUILD_LOG_BYTES) - { - return Err(StorageError::LimitExceeded { - max_bytes: MAX_BUILD_LOG_BYTES, - }); - } - current.extend_from_slice(content.as_bytes()); - backend.put_bytes(&key, ¤t).await - } - - pub async fn read_build_log( - &self, - project_id: Uuid, - deployment_id: Uuid, - ) -> Result, StorageError> { - let key = LocalStorage::build_log_relative_path(project_id, deployment_id); - let Some(object) = self.backend().open(&key).await? else { - return Ok(None); - }; - let bytes = read_object_limited(object, MAX_BUILD_LOG_BYTES).await?; - String::from_utf8(bytes) - .map(Some) - .map_err(|error| StorageError::Backend(format!("build log is not UTF-8: {error}"))) - } - - fn log_append_lock(&self, key: &str) -> Arc> { - let mut locks = self.log_append_locks.lock().unwrap(); - locks.retain(|_, lock| lock.strong_count() > 0); - if let Some(lock) = locks.get(key).and_then(Weak::upgrade) { - return lock; - } - let lock = Arc::new(tokio::sync::Mutex::new(())); - locks.insert(key.to_owned(), Arc::downgrade(&lock)); - lock - } -} - -async fn read_object_limited( - object: OpenedObject, - max_bytes: u64, -) -> Result, StorageError> { - if object.size_bytes > max_bytes { - return Err(StorageError::LimitExceeded { max_bytes }); - } - let capacity = usize::try_from(object.size_bytes).map_err(|_| StorageError::UnsupportedSize)?; - let mut bytes = Vec::with_capacity(capacity); - let mut size_bytes = 0_u64; - let mut stream = object.stream; - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - size_bytes = size_bytes - .checked_add(u64::try_from(chunk.len()).map_err(|_| StorageError::UnsupportedSize)?) - .ok_or(StorageError::UnsupportedSize)?; - if size_bytes > max_bytes { - return Err(StorageError::LimitExceeded { max_bytes }); - } - bytes.extend_from_slice(&chunk); - } - Ok(bytes) -} - pub fn build_backend( config: &StorageConfig, credentials: &StorageCredentials, @@ -946,613 +112,3 @@ pub fn build_backend( } } } - -pub async fn list_managed_backend( - backend: &Arc, -) -> Result, StorageError> { - let mut objects = backend.list("deployments/").await?; - objects.extend(backend.list("avatars/").await?); - objects.sort_by(|left, right| left.key.cmp(&right.key)); - Ok(objects) -} - -pub async fn copy_and_verify( - source: &Arc, - target: &Arc, - key: &str, -) -> Result<(u64, String), StorageError> { - let source_object = source - .open(key) - .await? - .ok_or_else(|| StorageError::Backend(format!("source object disappeared: {key}")))?; - let source_size = source_object.size_bytes; - let source_stats = Arc::new(std::sync::Mutex::new(StreamStats::default())); - let tracked = TrackingStream::new(source_object.stream, u64::MAX, Arc::clone(&source_stats)); - target.put_stream(key, Box::pin(tracked)).await?; - let source_stats = source_stats.lock().unwrap().clone(); - if source_stats.size_bytes != source_size { - return Err(StorageError::Backend(format!( - "source object size changed while copying {key}" - ))); - } - let source_checksum = hex::encode(source_stats.hasher.finalize()); - - let target_object = target.open(key).await?.ok_or_else(|| { - StorageError::Backend(format!("target object is missing after copy: {key}")) - })?; - let target_size = target_object.size_bytes; - let target_stats = Arc::new(std::sync::Mutex::new(StreamStats::default())); - let mut tracked = - TrackingStream::new(target_object.stream, u64::MAX, Arc::clone(&target_stats)); - while let Some(chunk) = tracked.next().await { - chunk?; - } - let target_stats = target_stats.lock().unwrap().clone(); - let target_checksum = hex::encode(target_stats.hasher.finalize()); - if target_size != source_size - || target_stats.size_bytes != source_size - || target_checksum != source_checksum - { - return Err(StorageError::Backend(format!( - "target verification failed for {key}" - ))); - } - Ok((source_size, source_checksum)) -} - -#[derive(Debug, Clone)] -struct StreamStats { - size_bytes: u64, - hasher: Sha256, -} - -impl Default for StreamStats { - fn default() -> Self { - Self { - size_bytes: 0, - hasher: Sha256::new(), - } - } -} - -struct TrackingStream { - inner: Pin>, - max_bytes: u64, - stats: Arc>, -} - -impl TrackingStream { - fn new(inner: S, max_bytes: u64, stats: Arc>) -> Self { - Self { - inner: Box::pin(inner), - max_bytes, - stats, - } - } -} - -impl Stream for TrackingStream -where - S: Stream>, - E: std::error::Error + Send + Sync + 'static, -{ - type Item = Result; - - fn poll_next( - mut self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - match self.inner.as_mut().poll_next(cx) { - std::task::Poll::Ready(Some(Ok(chunk))) => { - let mut stats = self.stats.lock().unwrap(); - let next_size = stats.size_bytes.saturating_add(chunk.len() as u64); - if next_size > self.max_bytes { - return std::task::Poll::Ready(Some(Err(StorageError::LimitExceeded { - max_bytes: self.max_bytes, - }))); - } - stats.size_bytes = next_size; - stats.hasher.update(&chunk); - std::task::Poll::Ready(Some(Ok(chunk))) - } - std::task::Poll::Ready(Some(Err(error))) => { - std::task::Poll::Ready(Some(Err(StorageError::Stream(error.to_string())))) - } - std::task::Poll::Ready(None) => std::task::Poll::Ready(None), - std::task::Poll::Pending => std::task::Poll::Pending, - } - } -} - -pub struct PendingArtifact { - storage: Arc, - temporary_key: Option, - final_key: String, - pub size_bytes: i64, - pub checksum_sha256: String, - write_guard: Option>, -} - -impl PendingArtifact { - pub async fn finalize(mut self) -> Result { - let temporary_key = self - .temporary_key - .take() - .expect("pending artifact must own a temporary object"); - if let Err(error) = self.storage.rename(&temporary_key, &self.final_key).await { - let _ = self.storage.remove(&temporary_key).await; - return Err(error); - } - self.write_guard.take(); - Ok(StoredArtifact { - relative_path: self.final_key.clone(), - size_bytes: self.size_bytes, - checksum_sha256: self.checksum_sha256.clone(), - }) - } - - pub async fn discard(mut self) { - if let Some(temporary_key) = self.temporary_key.take() { - let _ = self.storage.remove(&temporary_key).await; - } - self.write_guard.take(); - } -} - -impl Drop for PendingArtifact { - fn drop(&mut self) { - let Some(temporary_key) = self.temporary_key.take() else { - return; - }; - let storage = Arc::clone(&self.storage); - if let Ok(runtime) = tokio::runtime::Handle::try_current() { - runtime.spawn(async move { - let _ = storage.remove(&temporary_key).await; - }); - } - } -} - -#[cfg(test)] -mod tests { - use bytes::Bytes; - use futures_util::stream; - use object_store::{ - CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, - PutMultipartOptions, PutOptions, PutPayload, PutResult, UploadPart, memory::InMemory, - }; - - use super::*; - - #[derive(Debug)] - struct AbortTrackingStore { - inner: InMemory, - aborted: Arc, - } - - impl std::fmt::Display for AbortTrackingStore { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("abort-tracking-store") - } - } - - #[async_trait] - impl ApacheObjectStore for AbortTrackingStore { - async fn put_opts( - &self, - location: &ObjectPath, - payload: PutPayload, - options: PutOptions, - ) -> object_store::Result { - self.inner.put_opts(location, payload, options).await - } - - async fn put_multipart_opts( - &self, - location: &ObjectPath, - options: PutMultipartOptions, - ) -> object_store::Result> { - let inner = self.inner.put_multipart_opts(location, options).await?; - Ok(Box::new(AbortTrackingUpload { - inner, - aborted: Arc::clone(&self.aborted), - })) - } - - async fn get_opts( - &self, - location: &ObjectPath, - options: GetOptions, - ) -> object_store::Result { - self.inner.get_opts(location, options).await - } - - fn delete_stream( - &self, - locations: BoxStream<'static, object_store::Result>, - ) -> BoxStream<'static, object_store::Result> { - self.inner.delete_stream(locations) - } - - fn list( - &self, - prefix: Option<&ObjectPath>, - ) -> BoxStream<'static, object_store::Result> { - self.inner.list(prefix) - } - - async fn list_with_delimiter( - &self, - prefix: Option<&ObjectPath>, - ) -> object_store::Result { - self.inner.list_with_delimiter(prefix).await - } - - async fn copy_opts( - &self, - from: &ObjectPath, - to: &ObjectPath, - options: CopyOptions, - ) -> object_store::Result<()> { - self.inner.copy_opts(from, to, options).await - } - } - - #[derive(Debug)] - struct AbortTrackingUpload { - inner: Box, - aborted: Arc, - } - - #[async_trait] - impl MultipartUpload for AbortTrackingUpload { - fn put_part(&mut self, data: PutPayload) -> UploadPart { - self.inner.put_part(data) - } - - async fn complete(&mut self) -> object_store::Result { - self.inner.complete().await - } - - async fn abort(&mut self) -> object_store::Result<()> { - self.aborted.store(true, Ordering::Release); - self.inner.abort().await - } - } - - #[test] - fn storage_config_defaults_to_local_and_rejects_unsafe_paths() { - let config = StorageConfig::default(); - assert_eq!(config.backend, StorageBackendKind::Local); - assert!(config.validate().is_ok()); - assert!(validate_key("../outside").is_err()); - assert!(validate_key("/absolute").is_err()); - } - - #[test] - fn minio_and_r2_require_explicit_endpoints() { - for backend in [StorageBackendKind::Minio, StorageBackendKind::R2] { - let config = StorageConfig { - backend, - bucket: "artifacts".to_owned(), - region: "us-east-1".to_owned(), - ..StorageConfig::default() - }; - assert!( - config.validate().is_err(), - "{backend:?} accepted a missing endpoint" - ); - } - } - - #[test] - fn remote_storage_requires_an_absolute_local_node_root() { - let config = StorageConfig { - backend: StorageBackendKind::S3, - local_root: "relative-node-root".to_owned(), - endpoint: "https://s3.example.com".to_owned(), - bucket: "artifacts".to_owned(), - region: "us-east-1".to_owned(), - ..StorageConfig::default() - }; - - assert!(config.validate().is_err()); - } - - #[test] - fn provider_default_regions_preserve_backend_semantics() { - assert_eq!(StorageBackendKind::R2.default_region(), "auto"); - assert_eq!(StorageBackendKind::S3.default_region(), "us-east-1"); - assert_eq!(StorageBackendKind::Minio.default_region(), "us-east-1"); - } - - #[tokio::test] - async fn s3_stream_error_aborts_multipart_upload() { - let aborted = Arc::new(AtomicBool::new(false)); - let store: Arc = Arc::new(AbortTrackingStore { - inner: InMemory::new(), - aborted: Arc::clone(&aborted), - }); - let storage = S3Storage { store }; - let chunks = stream::iter([ - Ok(Bytes::from(vec![0; 10 * 1024 * 1024])), - Err(StorageError::Stream("injected failure".to_owned())), - ]) - .boxed(); - - let error = storage - .put_stream("multipart.bin", chunks) - .await - .unwrap_err(); - - assert!(matches!(error, StorageError::Stream(_))); - assert!(aborted.load(Ordering::Acquire)); - } - - #[tokio::test] - async fn build_log_append_rejects_content_over_the_hard_limit() { - let dir = std::env::temp_dir().join(format!("grass-storage-log-limit-{}", Uuid::now_v7())); - let manager = StorageManager::new_local(&dir); - let content = "x".repeat(16 * 1024 * 1024 + 1); - - let error = manager - .append_build_log(Uuid::now_v7(), Uuid::now_v7(), &content) - .await - .unwrap_err(); - - assert!(matches!(error, StorageError::LimitExceeded { .. })); - let _ = tokio::fs::remove_dir_all(dir).await; - } - - #[tokio::test] - async fn build_log_read_rejects_an_oversized_stored_object() { - let dir = - std::env::temp_dir().join(format!("grass-storage-log-read-limit-{}", Uuid::now_v7())); - let manager = StorageManager::new_local(&dir); - let project_id = Uuid::now_v7(); - let deployment_id = Uuid::now_v7(); - let key = LocalStorage::build_log_relative_path(project_id, deployment_id); - manager - .write_bytes(&key, &vec![b'x'; 16 * 1024 * 1024 + 1]) - .await - .unwrap(); - - let error = manager - .read_build_log(project_id, deployment_id) - .await - .unwrap_err(); - - assert!(matches!(error, StorageError::LimitExceeded { .. })); - tokio::fs::remove_dir_all(dir).await.unwrap(); - } - - #[derive(Clone)] - struct BlockingLogStorage { - blocked_key: String, - blocked_started: Arc, - release_blocked: Arc, - other_started: Arc, - } - - #[async_trait] - impl ObjectStorage for BlockingLogStorage { - async fn put_bytes(&self, _key: &str, _content: &[u8]) -> Result<(), StorageError> { - Ok(()) - } - - async fn put_stream( - &self, - _key: &str, - mut stream: ObjectStream, - ) -> Result<(), StorageError> { - while stream.next().await.transpose()?.is_some() {} - Ok(()) - } - - async fn open(&self, key: &str) -> Result, StorageError> { - if key == self.blocked_key { - self.blocked_started.notify_one(); - self.release_blocked.notified().await; - } else { - self.other_started.notify_one(); - } - Ok(None) - } - - async fn remove(&self, _key: &str) -> Result<(), StorageError> { - Ok(()) - } - - async fn rename(&self, _from: &str, _to: &str) -> Result<(), StorageError> { - Ok(()) - } - - async fn list(&self, _prefix: &str) -> Result, StorageError> { - Ok(Vec::new()) - } - - async fn probe(&self) -> Result<(), StorageError> { - Ok(()) - } - } - - #[tokio::test] - async fn build_log_appends_for_independent_deployments_do_not_share_a_lock() { - let project_id = Uuid::now_v7(); - let blocked_deployment = Uuid::now_v7(); - let other_deployment = Uuid::now_v7(); - let backend = Arc::new(BlockingLogStorage { - blocked_key: LocalStorage::build_log_relative_path(project_id, blocked_deployment), - blocked_started: Arc::new(tokio::sync::Notify::new()), - release_blocked: Arc::new(tokio::sync::Notify::new()), - other_started: Arc::new(tokio::sync::Notify::new()), - }); - let blocked_started = Arc::clone(&backend.blocked_started); - let release_blocked = Arc::clone(&backend.release_blocked); - let other_started = Arc::clone(&backend.other_started); - let manager = StorageManager::from_runtime(StorageRuntime { - config: StorageConfig::local("/tmp/grass-storage-test"), - backend, - }); - - let blocked = tokio::spawn({ - let manager = manager.clone(); - async move { - manager - .append_build_log(project_id, blocked_deployment, "blocked") - .await - } - }); - tokio::time::timeout( - std::time::Duration::from_secs(1), - blocked_started.notified(), - ) - .await - .expect("blocked deployment did not reach storage"); - - let other = tokio::spawn({ - let manager = manager.clone(); - async move { - manager - .append_build_log(project_id, other_deployment, "other") - .await - } - }); - let independent = tokio::time::timeout( - std::time::Duration::from_millis(100), - other_started.notified(), - ) - .await - .is_ok(); - - release_blocked.notify_one(); - blocked.await.unwrap().unwrap(); - other.await.unwrap().unwrap(); - assert!( - independent, - "independent deployment remained behind another log lock" - ); - } - - #[tokio::test] - async fn local_storage_round_trip_and_list() { - let dir = std::env::temp_dir().join(format!("grass-storage-test-{}", Uuid::now_v7())); - let manager = StorageManager::new_local(&dir); - let key = format!("avatars/users/{}/avatar.webp", Uuid::now_v7()); - let stored = manager.write_bytes(&key, b"webp-bytes").await.unwrap(); - assert_eq!(stored.size_bytes, 10); - let opened = manager.open_artifact(&key).await.unwrap().unwrap(); - let bytes = opened - .stream - .try_collect::>() - .await - .unwrap() - .into_iter() - .flatten() - .collect::>(); - assert_eq!(bytes, b"webp-bytes"); - assert_eq!( - list_managed_backend(&manager.backend()) - .await - .unwrap() - .len(), - 1 - ); - manager.remove(&key).await.unwrap(); - tokio::fs::remove_dir_all(dir).await.unwrap(); - } - - #[tokio::test] - async fn local_rename_rejects_a_missing_source_object() { - let dir = std::env::temp_dir().join(format!("grass-storage-rename-{}", Uuid::now_v7())); - let storage = LocalStorage::new(&dir); - - let error = ObjectStorage::rename( - &storage, - ".pending/missing", - "deployments/project/deployment/grass-output.zip", - ) - .await - .unwrap_err(); - - assert!(matches!(error, StorageError::Backend(_))); - let _ = tokio::fs::remove_dir_all(dir).await; - } - - #[tokio::test] - async fn maintenance_rejects_writes_but_keeps_reads_available() { - let dir = - std::env::temp_dir().join(format!("grass-storage-maintenance-{}", Uuid::now_v7())); - let manager = StorageManager::new_local(&dir); - let key = format!("avatars/users/{}/avatar.webp", Uuid::now_v7()); - manager.write_bytes(&key, b"existing").await.unwrap(); - - manager.mark_maintenance(); - - assert!(manager.open_artifact(&key).await.unwrap().is_some()); - assert!(matches!( - manager.write_bytes("avatars/new.webp", b"new").await, - Err(StorageError::Maintenance) - )); - assert!(matches!( - manager - .append_build_log(Uuid::now_v7(), Uuid::now_v7(), "log") - .await, - Err(StorageError::Maintenance) - )); - assert!(matches!( - manager.remove(&key).await, - Err(StorageError::Maintenance) - )); - - manager.leave_maintenance(); - manager.remove(&key).await.unwrap(); - tokio::fs::remove_dir_all(dir).await.unwrap(); - } - - #[tokio::test] - async fn streamed_artifact_is_hashed_and_limited() { - let dir = std::env::temp_dir().join(format!("grass-storage-stream-{}", Uuid::now_v7())); - let manager = StorageManager::new_local(&dir); - let chunks = stream::iter([ - Ok::<_, std::io::Error>(Bytes::from_static(b"zip-")), - Ok(Bytes::from_static(b"bytes")), - ]); - let pending = manager - .write_artifact_stream(Uuid::now_v7(), Uuid::now_v7(), chunks, 9) - .await - .unwrap(); - assert_eq!(pending.size_bytes, 9); - let stored = pending.finalize().await.unwrap(); - assert_eq!(stored.size_bytes, 9); - tokio::fs::remove_dir_all(dir).await.unwrap(); - } - - #[tokio::test] - async fn dropped_pending_artifact_removes_temporary_object() { - let dir = std::env::temp_dir().join(format!("grass-storage-drop-{}", Uuid::now_v7())); - let manager = StorageManager::new_local(&dir); - let pending = manager - .write_artifact_stream( - Uuid::now_v7(), - Uuid::now_v7(), - stream::iter([Ok::<_, std::io::Error>(Bytes::from_static(b"pending"))]), - 64, - ) - .await - .unwrap(); - let temporary_path = dir.join(pending.temporary_key.as_deref().unwrap()); - assert!(tokio::fs::try_exists(&temporary_path).await.unwrap()); - - drop(pending); - - tokio::time::timeout(std::time::Duration::from_secs(1), async { - while tokio::fs::try_exists(&temporary_path).await.unwrap() { - tokio::task::yield_now().await; - } - }) - .await - .expect("pending object cleanup timed out"); - tokio::fs::remove_dir_all(dir).await.unwrap(); - } -} diff --git a/apps/control-api/src/infra/storage/s3.rs b/apps/control-api/src/infra/storage/s3.rs new file mode 100644 index 0000000..dae7706 --- /dev/null +++ b/apps/control-api/src/infra/storage/s3.rs @@ -0,0 +1,180 @@ +//! S3-compatible adapter and multipart upload cleanup. + +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use futures_util::{StreamExt, TryStreamExt}; +use object_store::{ + ObjectStore as ApacheObjectStore, ObjectStoreExt, PutPayload, aws::AmazonS3Builder, + buffered::BufWriter, path::Path as ObjectPath, prefix::PrefixStore, +}; +use tokio::io::AsyncWriteExt; +use uuid::Uuid; + +use super::{ + ObjectStorage, ObjectStream, OpenedObject, StorageConfig, StorageCredentials, StorageError, + StoredObjectMeta, validate_key, +}; + +#[derive(Clone)] +pub struct S3Storage { + store: Arc, +} + +impl S3Storage { + pub fn new( + config: &StorageConfig, + credentials: &StorageCredentials, + ) -> Result { + config.validate()?; + let mut builder = AmazonS3Builder::from_env() + .with_bucket_name(config.bucket.trim()) + .with_region(config.region.trim()) + .with_allow_http(config.allow_http) + .with_virtual_hosted_style_request(!config.force_path_style); + if !config.endpoint.trim().is_empty() { + builder = builder.with_endpoint(config.endpoint.trim()); + } + if let Some(value) = credentials + .access_key_id + .as_deref() + .filter(|value| !value.trim().is_empty()) + { + builder = builder.with_access_key_id(value); + } + if let Some(value) = credentials + .secret_access_key + .as_deref() + .filter(|value| !value.trim().is_empty()) + { + builder = builder.with_secret_access_key(value); + } + if let Some(value) = credentials + .session_token + .as_deref() + .filter(|value| !value.trim().is_empty()) + { + builder = builder.with_token(value); + } + let store = builder + .build() + .map_err(|error| StorageError::Backend(error.to_string()))?; + let store: Arc = if config.prefix.trim().is_empty() { + Arc::new(store) + } else { + Arc::new(PrefixStore::new( + store, + ObjectPath::from(config.prefix.trim()), + )) + }; + Ok(Self { store }) + } + + fn path(&self, key: &str) -> Result { + validate_key(key)?; + ObjectPath::parse(key).map_err(|error| StorageError::UnsafePath(error.to_string())) + } +} + +#[async_trait] +impl ObjectStorage for S3Storage { + async fn put_bytes(&self, key: &str, content: &[u8]) -> Result<(), StorageError> { + self.store + .put( + &self.path(key)?, + PutPayload::from(Bytes::copy_from_slice(content)), + ) + .await + .map(|_| ()) + .map_err(|error| StorageError::Backend(error.to_string())) + } + + async fn put_stream(&self, key: &str, mut stream: ObjectStream) -> Result<(), StorageError> { + let mut writer = + BufWriter::with_capacity(Arc::clone(&self.store), self.path(key)?, 10 * 1024 * 1024); + while let Some(chunk) = stream.next().await { + let chunk = match chunk { + Ok(chunk) => chunk, + Err(error) => { + let _ = writer.abort().await; + return Err(error); + } + }; + match writer.put(chunk).await { + Ok(()) => {} + Err(error) => { + let _ = writer.abort().await; + return Err(StorageError::Backend(error.to_string())); + } + } + } + writer + .shutdown() + .await + .map_err(|error| StorageError::Backend(error.to_string())) + } + + async fn open(&self, key: &str) -> Result, StorageError> { + let path = self.path(key)?; + let result = match self.store.get(&path).await { + Ok(result) => result, + Err(object_store::Error::NotFound { .. }) => return Ok(None), + Err(error) => return Err(StorageError::Backend(error.to_string())), + }; + let size_bytes = result.meta.size; + let stream = result + .into_stream() + .map(|chunk| chunk.map_err(|error| StorageError::Backend(error.to_string()))) + .boxed(); + Ok(Some(OpenedObject { stream, size_bytes })) + } + + async fn remove(&self, key: &str) -> Result<(), StorageError> { + match self.store.delete(&self.path(key)?).await { + Ok(()) | Err(object_store::Error::NotFound { .. }) => Ok(()), + Err(error) => Err(StorageError::Backend(error.to_string())), + } + } + + async fn rename(&self, from: &str, to: &str) -> Result<(), StorageError> { + self.store + .rename(&self.path(from)?, &self.path(to)?) + .await + .map_err(|error| StorageError::Backend(error.to_string())) + } + + async fn list(&self, prefix: &str) -> Result, StorageError> { + let prefix = if prefix.trim().is_empty() { + None + } else { + Some(self.path(prefix)?) + }; + self.store + .list(prefix.as_ref()) + .map_ok(|meta| StoredObjectMeta { + key: meta.location.to_string(), + size_bytes: meta.size, + }) + .try_collect() + .await + .map_err(|error| StorageError::Backend(error.to_string())) + } + + async fn probe(&self) -> Result<(), StorageError> { + let key = format!(".probe/{}", Uuid::now_v7().simple()); + self.put_bytes(&key, b"grass-storage-probe").await?; + let result = self.open(&key).await?; + self.remove(&key).await?; + if result.is_none() { + return Err(StorageError::Backend( + "storage probe could not read its object".to_owned(), + )); + } + Ok(()) + } +} + +#[cfg(test)] +#[path = "tests/s3.rs"] +mod tests; diff --git a/apps/control-api/src/infra/storage/streams.rs b/apps/control-api/src/infra/storage/streams.rs new file mode 100644 index 0000000..7b5d585 --- /dev/null +++ b/apps/control-api/src/infra/storage/streams.rs @@ -0,0 +1,97 @@ +//! Bounded stream reads and shared size/checksum accounting. + +use std::{pin::Pin, sync::Arc}; + +use bytes::Bytes; +use futures_util::{Stream, StreamExt}; +use sha2::{Digest, Sha256}; + +use super::{OpenedObject, StorageError}; + +pub(super) async fn read_object_limited( + object: OpenedObject, + max_bytes: u64, +) -> Result, StorageError> { + if object.size_bytes > max_bytes { + return Err(StorageError::LimitExceeded { max_bytes }); + } + let capacity = usize::try_from(object.size_bytes).map_err(|_| StorageError::UnsupportedSize)?; + let mut bytes = Vec::with_capacity(capacity); + let mut size_bytes = 0_u64; + let mut stream = object.stream; + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + size_bytes = size_bytes + .checked_add(u64::try_from(chunk.len()).map_err(|_| StorageError::UnsupportedSize)?) + .ok_or(StorageError::UnsupportedSize)?; + if size_bytes > max_bytes { + return Err(StorageError::LimitExceeded { max_bytes }); + } + bytes.extend_from_slice(&chunk); + } + Ok(bytes) +} + +#[derive(Debug, Clone)] +pub(super) struct StreamStats { + pub(super) size_bytes: u64, + pub(super) hasher: Sha256, +} + +impl Default for StreamStats { + fn default() -> Self { + Self { + size_bytes: 0, + hasher: Sha256::new(), + } + } +} + +pub(super) struct TrackingStream { + inner: Pin>, + max_bytes: u64, + stats: Arc>, +} + +impl TrackingStream { + pub(super) fn new(inner: S, max_bytes: u64, stats: Arc>) -> Self { + Self { + inner: Box::pin(inner), + max_bytes, + stats, + } + } +} + +impl Stream for TrackingStream +where + S: Stream>, + E: std::error::Error + Send + Sync + 'static, +{ + type Item = Result; + + fn poll_next( + mut self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + match self.inner.as_mut().poll_next(cx) { + std::task::Poll::Ready(Some(Ok(chunk))) => { + let mut stats = self.stats.lock().unwrap(); + let next_size = stats.size_bytes.saturating_add(chunk.len() as u64); + if next_size > self.max_bytes { + return std::task::Poll::Ready(Some(Err(StorageError::LimitExceeded { + max_bytes: self.max_bytes, + }))); + } + stats.size_bytes = next_size; + stats.hasher.update(&chunk); + std::task::Poll::Ready(Some(Ok(chunk))) + } + std::task::Poll::Ready(Some(Err(error))) => { + std::task::Poll::Ready(Some(Err(StorageError::Stream(error.to_string())))) + } + std::task::Poll::Ready(None) => std::task::Poll::Ready(None), + std::task::Poll::Pending => std::task::Poll::Pending, + } + } +} diff --git a/apps/control-api/src/infra/storage/tests/config.rs b/apps/control-api/src/infra/storage/tests/config.rs new file mode 100644 index 0000000..d766e4d --- /dev/null +++ b/apps/control-api/src/infra/storage/tests/config.rs @@ -0,0 +1,48 @@ +use super::super::validate_key; +use super::*; + +#[test] +fn storage_config_defaults_to_local_and_rejects_unsafe_paths() { + let config = StorageConfig::default(); + assert_eq!(config.backend, StorageBackendKind::Local); + assert!(config.validate().is_ok()); + assert!(validate_key("../outside").is_err()); + assert!(validate_key("/absolute").is_err()); +} + +#[test] +fn minio_and_r2_require_explicit_endpoints() { + for backend in [StorageBackendKind::Minio, StorageBackendKind::R2] { + let config = StorageConfig { + backend, + bucket: "artifacts".to_owned(), + region: "us-east-1".to_owned(), + ..StorageConfig::default() + }; + assert!( + config.validate().is_err(), + "{backend:?} accepted a missing endpoint" + ); + } +} + +#[test] +fn remote_storage_requires_an_absolute_local_node_root() { + let config = StorageConfig { + backend: StorageBackendKind::S3, + local_root: "relative-node-root".to_owned(), + endpoint: "https://s3.example.com".to_owned(), + bucket: "artifacts".to_owned(), + region: "us-east-1".to_owned(), + ..StorageConfig::default() + }; + + assert!(config.validate().is_err()); +} + +#[test] +fn provider_default_regions_preserve_backend_semantics() { + assert_eq!(StorageBackendKind::R2.default_region(), "auto"); + assert_eq!(StorageBackendKind::S3.default_region(), "us-east-1"); + assert_eq!(StorageBackendKind::Minio.default_region(), "us-east-1"); +} diff --git a/apps/control-api/src/infra/storage/tests/local.rs b/apps/control-api/src/infra/storage/tests/local.rs new file mode 100644 index 0000000..3f7979f --- /dev/null +++ b/apps/control-api/src/infra/storage/tests/local.rs @@ -0,0 +1,48 @@ +use super::super::{StorageManager, list_managed_backend}; +use super::*; +use futures_util::TryStreamExt; + +#[tokio::test] +async fn local_storage_round_trip_and_list() { + let dir = std::env::temp_dir().join(format!("grass-storage-test-{}", Uuid::now_v7())); + let manager = StorageManager::new_local(&dir); + let key = format!("avatars/users/{}/avatar.webp", Uuid::now_v7()); + let stored = manager.write_bytes(&key, b"webp-bytes").await.unwrap(); + assert_eq!(stored.size_bytes, 10); + let opened = manager.open_artifact(&key).await.unwrap().unwrap(); + let bytes = opened + .stream + .try_collect::>() + .await + .unwrap() + .into_iter() + .flatten() + .collect::>(); + assert_eq!(bytes, b"webp-bytes"); + assert_eq!( + list_managed_backend(&manager.backend()) + .await + .unwrap() + .len(), + 1 + ); + manager.remove(&key).await.unwrap(); + tokio::fs::remove_dir_all(dir).await.unwrap(); +} + +#[tokio::test] +async fn local_rename_rejects_a_missing_source_object() { + let dir = std::env::temp_dir().join(format!("grass-storage-rename-{}", Uuid::now_v7())); + let storage = LocalStorage::new(&dir); + + let error = ObjectStorage::rename( + &storage, + ".pending/missing", + "deployments/project/deployment/grass-output.zip", + ) + .await + .unwrap_err(); + + assert!(matches!(error, StorageError::Backend(_))); + let _ = tokio::fs::remove_dir_all(dir).await; +} diff --git a/apps/control-api/src/infra/storage/tests/manager.rs b/apps/control-api/src/infra/storage/tests/manager.rs new file mode 100644 index 0000000..42fe4eb --- /dev/null +++ b/apps/control-api/src/infra/storage/tests/manager.rs @@ -0,0 +1,220 @@ +use super::super::{ObjectStream, StoredObjectMeta}; +use super::*; +use async_trait::async_trait; +use futures_util::{StreamExt, stream}; + +#[tokio::test] +async fn build_log_append_rejects_content_over_the_hard_limit() { + let dir = std::env::temp_dir().join(format!("grass-storage-log-limit-{}", Uuid::now_v7())); + let manager = StorageManager::new_local(&dir); + let content = "x".repeat(16 * 1024 * 1024 + 1); + + let error = manager + .append_build_log(Uuid::now_v7(), Uuid::now_v7(), &content) + .await + .unwrap_err(); + + assert!(matches!(error, StorageError::LimitExceeded { .. })); + let _ = tokio::fs::remove_dir_all(dir).await; +} + +#[tokio::test] +async fn build_log_read_rejects_an_oversized_stored_object() { + let dir = std::env::temp_dir().join(format!("grass-storage-log-read-limit-{}", Uuid::now_v7())); + let manager = StorageManager::new_local(&dir); + let project_id = Uuid::now_v7(); + let deployment_id = Uuid::now_v7(); + let key = LocalStorage::build_log_relative_path(project_id, deployment_id); + manager + .write_bytes(&key, &vec![b'x'; 16 * 1024 * 1024 + 1]) + .await + .unwrap(); + + let error = manager + .read_build_log(project_id, deployment_id) + .await + .unwrap_err(); + + assert!(matches!(error, StorageError::LimitExceeded { .. })); + tokio::fs::remove_dir_all(dir).await.unwrap(); +} + +#[derive(Clone)] +struct BlockingLogStorage { + blocked_key: String, + blocked_started: Arc, + release_blocked: Arc, + other_started: Arc, +} + +#[async_trait] +impl ObjectStorage for BlockingLogStorage { + async fn put_bytes(&self, _key: &str, _content: &[u8]) -> Result<(), StorageError> { + Ok(()) + } + + async fn put_stream(&self, _key: &str, mut stream: ObjectStream) -> Result<(), StorageError> { + while stream.next().await.transpose()?.is_some() {} + Ok(()) + } + + async fn open(&self, key: &str) -> Result, StorageError> { + if key == self.blocked_key { + self.blocked_started.notify_one(); + self.release_blocked.notified().await; + } else { + self.other_started.notify_one(); + } + Ok(None) + } + + async fn remove(&self, _key: &str) -> Result<(), StorageError> { + Ok(()) + } + + async fn rename(&self, _from: &str, _to: &str) -> Result<(), StorageError> { + Ok(()) + } + + async fn list(&self, _prefix: &str) -> Result, StorageError> { + Ok(Vec::new()) + } + + async fn probe(&self) -> Result<(), StorageError> { + Ok(()) + } +} + +#[tokio::test] +async fn build_log_appends_for_independent_deployments_do_not_share_a_lock() { + let project_id = Uuid::now_v7(); + let blocked_deployment = Uuid::now_v7(); + let other_deployment = Uuid::now_v7(); + let backend = Arc::new(BlockingLogStorage { + blocked_key: LocalStorage::build_log_relative_path(project_id, blocked_deployment), + blocked_started: Arc::new(tokio::sync::Notify::new()), + release_blocked: Arc::new(tokio::sync::Notify::new()), + other_started: Arc::new(tokio::sync::Notify::new()), + }); + let blocked_started = Arc::clone(&backend.blocked_started); + let release_blocked = Arc::clone(&backend.release_blocked); + let other_started = Arc::clone(&backend.other_started); + let manager = StorageManager::from_runtime(StorageRuntime { + config: StorageConfig::local("/tmp/grass-storage-test"), + backend, + }); + + let blocked = tokio::spawn({ + let manager = manager.clone(); + async move { + manager + .append_build_log(project_id, blocked_deployment, "blocked") + .await + } + }); + tokio::time::timeout( + std::time::Duration::from_secs(1), + blocked_started.notified(), + ) + .await + .expect("blocked deployment did not reach storage"); + + let other = tokio::spawn({ + let manager = manager.clone(); + async move { + manager + .append_build_log(project_id, other_deployment, "other") + .await + } + }); + let independent = tokio::time::timeout( + std::time::Duration::from_millis(100), + other_started.notified(), + ) + .await + .is_ok(); + + release_blocked.notify_one(); + blocked.await.unwrap().unwrap(); + other.await.unwrap().unwrap(); + assert!( + independent, + "independent deployment remained behind another log lock" + ); +} + +#[tokio::test] +async fn maintenance_rejects_writes_but_keeps_reads_available() { + let dir = std::env::temp_dir().join(format!("grass-storage-maintenance-{}", Uuid::now_v7())); + let manager = StorageManager::new_local(&dir); + let key = format!("avatars/users/{}/avatar.webp", Uuid::now_v7()); + manager.write_bytes(&key, b"existing").await.unwrap(); + + manager.mark_maintenance(); + + assert!(manager.open_artifact(&key).await.unwrap().is_some()); + assert!(matches!( + manager.write_bytes("avatars/new.webp", b"new").await, + Err(StorageError::Maintenance) + )); + assert!(matches!( + manager + .append_build_log(Uuid::now_v7(), Uuid::now_v7(), "log") + .await, + Err(StorageError::Maintenance) + )); + assert!(matches!( + manager.remove(&key).await, + Err(StorageError::Maintenance) + )); + + manager.leave_maintenance(); + manager.remove(&key).await.unwrap(); + tokio::fs::remove_dir_all(dir).await.unwrap(); +} + +#[tokio::test] +async fn streamed_artifact_is_hashed_and_limited() { + let dir = std::env::temp_dir().join(format!("grass-storage-stream-{}", Uuid::now_v7())); + let manager = StorageManager::new_local(&dir); + let chunks = stream::iter([ + Ok::<_, std::io::Error>(Bytes::from_static(b"zip-")), + Ok(Bytes::from_static(b"bytes")), + ]); + let pending = manager + .write_artifact_stream(Uuid::now_v7(), Uuid::now_v7(), chunks, 9) + .await + .unwrap(); + assert_eq!(pending.size_bytes, 9); + let stored = pending.finalize().await.unwrap(); + assert_eq!(stored.size_bytes, 9); + tokio::fs::remove_dir_all(dir).await.unwrap(); +} + +#[tokio::test] +async fn dropped_pending_artifact_removes_temporary_object() { + let dir = std::env::temp_dir().join(format!("grass-storage-drop-{}", Uuid::now_v7())); + let manager = StorageManager::new_local(&dir); + let pending = manager + .write_artifact_stream( + Uuid::now_v7(), + Uuid::now_v7(), + stream::iter([Ok::<_, std::io::Error>(Bytes::from_static(b"pending"))]), + 64, + ) + .await + .unwrap(); + let temporary_path = dir.join(pending.temporary_key.as_deref().unwrap()); + assert!(tokio::fs::try_exists(&temporary_path).await.unwrap()); + + drop(pending); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while tokio::fs::try_exists(&temporary_path).await.unwrap() { + tokio::task::yield_now().await; + } + }) + .await + .expect("pending object cleanup timed out"); + tokio::fs::remove_dir_all(dir).await.unwrap(); +} diff --git a/apps/control-api/src/infra/storage/tests/s3.rs b/apps/control-api/src/infra/storage/tests/s3.rs new file mode 100644 index 0000000..2e2ba38 --- /dev/null +++ b/apps/control-api/src/infra/storage/tests/s3.rs @@ -0,0 +1,125 @@ +use super::*; +use futures_util::{stream, stream::BoxStream}; +use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, + PutMultipartOptions, PutOptions, PutResult, UploadPart, memory::InMemory, +}; +use std::sync::atomic::{AtomicBool, Ordering}; +#[derive(Debug)] +struct AbortTrackingStore { + inner: InMemory, + aborted: Arc, +} + +impl std::fmt::Display for AbortTrackingStore { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("abort-tracking-store") + } +} + +#[async_trait] +impl ApacheObjectStore for AbortTrackingStore { + async fn put_opts( + &self, + location: &ObjectPath, + payload: PutPayload, + options: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(location, payload, options).await + } + + async fn put_multipart_opts( + &self, + location: &ObjectPath, + options: PutMultipartOptions, + ) -> object_store::Result> { + let inner = self.inner.put_multipart_opts(location, options).await?; + Ok(Box::new(AbortTrackingUpload { + inner, + aborted: Arc::clone(&self.aborted), + })) + } + + async fn get_opts( + &self, + location: &ObjectPath, + options: GetOptions, + ) -> object_store::Result { + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(locations) + } + + fn list( + &self, + prefix: Option<&ObjectPath>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter( + &self, + prefix: Option<&ObjectPath>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts( + &self, + from: &ObjectPath, + to: &ObjectPath, + options: CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +#[derive(Debug)] +struct AbortTrackingUpload { + inner: Box, + aborted: Arc, +} + +#[async_trait] +impl MultipartUpload for AbortTrackingUpload { + fn put_part(&mut self, data: PutPayload) -> UploadPart { + self.inner.put_part(data) + } + + async fn complete(&mut self) -> object_store::Result { + self.inner.complete().await + } + + async fn abort(&mut self) -> object_store::Result<()> { + self.aborted.store(true, Ordering::Release); + self.inner.abort().await + } +} + +#[tokio::test] +async fn s3_stream_error_aborts_multipart_upload() { + let aborted = Arc::new(AtomicBool::new(false)); + let store: Arc = Arc::new(AbortTrackingStore { + inner: InMemory::new(), + aborted: Arc::clone(&aborted), + }); + let storage = S3Storage { store }; + let chunks = stream::iter([ + Ok(Bytes::from(vec![0; 10 * 1024 * 1024])), + Err(StorageError::Stream("injected failure".to_owned())), + ]) + .boxed(); + + let error = storage + .put_stream("multipart.bin", chunks) + .await + .unwrap_err(); + + assert!(matches!(error, StorageError::Stream(_))); + assert!(aborted.load(Ordering::Acquire)); +} diff --git a/apps/control-api/src/infra/storage/transfer.rs b/apps/control-api/src/infra/storage/transfer.rs new file mode 100644 index 0000000..4083aee --- /dev/null +++ b/apps/control-api/src/infra/storage/transfer.rs @@ -0,0 +1,64 @@ +//! Object enumeration and verified backend-to-backend copying. + +use std::sync::Arc; + +use futures_util::StreamExt; +use sha2::Digest; + +use super::{ + ObjectStorage, StorageError, StoredObjectMeta, + streams::{StreamStats, TrackingStream}, +}; + +pub async fn list_managed_backend( + backend: &Arc, +) -> Result, StorageError> { + let mut objects = backend.list("deployments/").await?; + objects.extend(backend.list("avatars/").await?); + objects.sort_by(|left, right| left.key.cmp(&right.key)); + Ok(objects) +} + +pub async fn copy_and_verify( + source: &Arc, + target: &Arc, + key: &str, +) -> Result<(u64, String), StorageError> { + let source_object = source + .open(key) + .await? + .ok_or_else(|| StorageError::Backend(format!("source object disappeared: {key}")))?; + let source_size = source_object.size_bytes; + let source_stats = Arc::new(std::sync::Mutex::new(StreamStats::default())); + let tracked = TrackingStream::new(source_object.stream, u64::MAX, Arc::clone(&source_stats)); + target.put_stream(key, Box::pin(tracked)).await?; + let source_stats = source_stats.lock().unwrap().clone(); + if source_stats.size_bytes != source_size { + return Err(StorageError::Backend(format!( + "source object size changed while copying {key}" + ))); + } + let source_checksum = hex::encode(source_stats.hasher.finalize()); + + let target_object = target.open(key).await?.ok_or_else(|| { + StorageError::Backend(format!("target object is missing after copy: {key}")) + })?; + let target_size = target_object.size_bytes; + let target_stats = Arc::new(std::sync::Mutex::new(StreamStats::default())); + let mut tracked = + TrackingStream::new(target_object.stream, u64::MAX, Arc::clone(&target_stats)); + while let Some(chunk) = tracked.next().await { + chunk?; + } + let target_stats = target_stats.lock().unwrap().clone(); + let target_checksum = hex::encode(target_stats.hasher.finalize()); + if target_size != source_size + || target_stats.size_bytes != source_size + || target_checksum != source_checksum + { + return Err(StorageError::Backend(format!( + "target verification failed for {key}" + ))); + } + Ok((source_size, source_checksum)) +} diff --git a/apps/node/src/build/realtime.rs b/apps/node/src/build/realtime.rs index 633df9f..8eb37fb 100644 --- a/apps/node/src/build/realtime.rs +++ b/apps/node/src/build/realtime.rs @@ -54,23 +54,22 @@ async fn forward(url: String, token: String, mut receiver: mpsc::Receiver } budget.entry(0)?; budget.names(bytes)?; - if kind.is_pax_local_extensions() { - if let Some(fields) = entry.pax_extensions()? { - for field in fields { - let field = field?; - let key = field.key().map_err(io::Error::other)?; - if key.starts_with("GNU.sparse") { - return Err(io::Error::other("sparse export metadata is unsupported")); - } - if key == "size" { - pax_size = Some( - field - .value() - .map_err(io::Error::other)? - .parse::() - .map_err(io::Error::other)?, - ); - } + if kind.is_pax_local_extensions() + && let Some(fields) = entry.pax_extensions()? + { + for field in fields { + let field = field?; + let key = field.key().map_err(io::Error::other)?; + if key.starts_with("GNU.sparse") { + return Err(io::Error::other("sparse export metadata is unsupported")); + } + if key == "size" { + pax_size = Some( + field + .value() + .map_err(io::Error::other)? + .parse::() + .map_err(io::Error::other)?, + ); } } } diff --git a/apps/node/src/serve/local.rs b/apps/node/src/serve/local.rs new file mode 100644 index 0000000..6f1d263 --- /dev/null +++ b/apps/node/src/serve/local.rs @@ -0,0 +1,241 @@ +//! Resolve staged deployments and deliver their static or SSR output. + +use std::{ + net::SocketAddr, + path::{Path, PathBuf}, + sync::Arc, +}; + +use axum::{ + extract::Request, + http::{StatusCode, header}, + response::Response, +}; +use grass_node_protocol::{ServeAccess, ServeRoute}; +use tracing::warn; +use uuid::Uuid; + +use super::{ + ServeState, normalize_public_path, + paths::resolve_not_found_file, + preview::{ + callback_code, handle_preview_callback, is_preview_callback, preview_cookie_value, + request_destination, require_preview_access, strip_preview_cookie_header, + }, + proxy::forward_to_ssr, + resolve_static_file, + response::error_page, + routing::{GATEWAY_HOP_HEADER, GATEWAY_TOKEN_HEADER, GatewayOrigin}, + static_files, sync, +}; +use crate::output::manifest; + +#[derive(Clone)] +pub(super) enum ResolvedTarget { + Static { + static_dir: PathBuf, + spa_fallback: bool, + not_found: Option, + }, + Ssr { + deployment_id: Uuid, + deployment_dir: PathBuf, + server: manifest::ServerSection, + }, +} + +pub(super) async fn serve_local( + state: Arc, + route: ServeRoute, + client_addr: SocketAddr, + origin: GatewayOrigin, + mut request: Request, +) -> Response { + request.headers_mut().remove(GATEWAY_TOKEN_HEADER); + request.headers_mut().remove(GATEWAY_HOP_HEADER); + + let target = match resolve_deployment(&state, route.deployment_id).await { + Ok(target) => target, + Err(error) => { + warn!(operation = "node.serve.resolve_host", %error, host = %route.host, "local deployment resolution failed"); + return error_page( + StatusCode::BAD_GATEWAY, + "The assigned deployment is not ready on this Serve Node.", + ); + } + }; + + let requires_preview_access = matches!(route.access, ServeAccess::TeamOrPlatformAdmin); + if requires_preview_access { + if is_preview_callback(request.uri().path()) { + let code = callback_code(&request); + return handle_preview_callback(&state, &route.host, code).await; + } + let destination = request_destination( + request + .uri() + .path_and_query() + .map(|value| value.as_str()) + .unwrap_or("/"), + ); + let grant = request + .headers() + .get(header::COOKIE) + .and_then(|value| value.to_str().ok()) + .and_then(preview_cookie_value) + .map(str::to_owned); + if let Err(response) = require_preview_access(&state, &route.host, destination, grant).await + { + return response; + } + } + + match target { + ResolvedTarget::Static { + static_dir, + spa_fallback, + not_found, + } => { + let method = request.method().clone(); + let range = request.headers().get(header::RANGE).cloned(); + let Some(segments) = normalize_public_path(request.uri().path()) else { + return error_page( + StatusCode::BAD_REQUEST, + "The requested path is not allowed.", + ); + }; + + match resolve_static_file(&static_dir, &segments, spa_fallback) { + Some(file) => serve_file(&file, StatusCode::OK, &method, range.as_ref()).await, + None => { + if let Some(not_found_file) = + resolve_not_found_file(&static_dir, not_found.as_deref()) + { + return serve_file( + ¬_found_file, + StatusCode::NOT_FOUND, + &method, + range.as_ref(), + ) + .await; + } + error_page(StatusCode::NOT_FOUND, "This page could not be found.") + } + } + } + ResolvedTarget::Ssr { + deployment_id, + deployment_dir, + server, + } => { + if requires_preview_access { + strip_preview_cookie_header(request.headers_mut()); + } + let upstream = match state + .ssr + .upstream_for(deployment_id, &deployment_dir, &server, route.resources) + .await + { + Ok(upstream) => upstream, + Err(error) => { + warn!( + operation = "node.serve.ssr_start", + %error, + deployment_id = %deployment_id, + "ssr service unavailable" + ); + return error_page( + StatusCode::BAD_GATEWAY, + "The application server failed to start.", + ); + } + }; + match forward_to_ssr(&state.proxy, &upstream, client_addr, origin, request).await { + Ok(response) => response, + Err(error) => { + // A connect failure means the container died or lost its + // address; drop it so the next request restarts it. + if error.is_connect() { + state.ssr.invalidate(deployment_id).await; + } + warn!( + operation = "node.serve.ssr_proxy", + %error, + deployment_id = %deployment_id, + "ssr proxy request failed" + ); + error_page( + StatusCode::BAD_GATEWAY, + "The application server could not be reached.", + ) + } + } + } + } +} + +async fn serve_file( + path: &Path, + status: StatusCode, + method: &axum::http::Method, + range: Option<&axum::http::HeaderValue>, +) -> Response { + match static_files::serve_file(path, method, range, status).await { + Ok(response) => response, + Err(_) => error_page(StatusCode::NOT_FOUND, "This page could not be found."), + } +} + +async fn resolve_deployment( + state: &ServeState, + deployment_id: Uuid, +) -> anyhow::Result { + if let Some(target) = state.targets.lock().await.get(&deployment_id).cloned() { + return Ok(target); + } + let target = ensure_artifact(state, deployment_id).await?; + state + .targets + .lock() + .await + .insert(deployment_id, target.clone()); + Ok(target) +} + +/// Loads the already staged deployment artifact and returns the serve target +/// described by its manifest. +async fn ensure_artifact( + state: &ServeState, + deployment_id: Uuid, +) -> anyhow::Result { + let deployment_dir = sync::staged_artifact_path(&state.cache_root, deployment_id)?; + let manifest_path = deployment_dir.join("output.toml"); + + let manifest_content = tokio::fs::read_to_string(&manifest_path).await?; + let manifest = manifest::parse_manifest(&manifest_content) + .map_err(|error| anyhow::anyhow!("invalid output manifest: {error}"))?; + manifest::validate_manifest(&manifest, &deployment_dir) + .map_err(|error| anyhow::anyhow!("invalid output manifest: {error}"))?; + + if manifest.runtime.kind == "ssr" { + let server = manifest + .server + .ok_or_else(|| anyhow::anyhow!("ssr manifest has no server section"))?; + return Ok(ResolvedTarget::Ssr { + deployment_id, + deployment_dir, + server, + }); + } + + let static_section = manifest + .static_site + .ok_or_else(|| anyhow::anyhow!("manifest has no static section"))?; + + Ok(ResolvedTarget::Static { + static_dir: deployment_dir.join(static_section.directory), + spa_fallback: static_section.spa_fallback, + not_found: (!static_section.not_found.trim().is_empty()) + .then(|| static_section.not_found.trim().to_owned()), + }) +} diff --git a/apps/node/src/serve/mod.rs b/apps/node/src/serve/mod.rs index ee3b3c8..da39cd3 100644 --- a/apps/node/src/serve/mod.rs +++ b/apps/node/src/serve/mod.rs @@ -13,54 +13,44 @@ pub mod static_files; pub mod sync; pub mod tls; -use std::collections::HashMap; -use std::net::SocketAddr; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::sync::atomic::Ordering; -use std::time::{Duration, Instant}; +mod local; +mod paths; +mod preview; +mod proxy; +mod response; +mod routing; + +pub use paths::{normalize_public_path, resolve_static_file}; + +use std::{ + collections::HashMap, + net::SocketAddr, + path::PathBuf, + sync::{Arc, atomic::Ordering}, + time::{Duration, Instant}, +}; use axum::{ Json, - body::Body, - extract::{ConnectInfo, Request, State}, - http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header}, + extract::{Request, State}, + http::{HeaderMap, HeaderValue, StatusCode, header}, middleware::{self, Next}, response::{IntoResponse, Response}, routing::{any, get, post}, }; -use grass_node_protocol::{GatewayAuthenticationMode, ServeAccess, ServeRoute}; +use grass_node_protocol::GatewayAuthenticationMode; use serde::Deserialize; -use sha2::{Digest, Sha256}; -use subtle::ConstantTimeEq; use tokio::sync::Mutex; use tracing::{info, warn}; use uuid::Uuid; -use crate::{ - client::{ControlApiClient, PreviewAuthError}, - config::NodeConfig, - output::manifest, +use crate::{client::ControlApiClient, config::NodeConfig}; +use local::ResolvedTarget; +use routing::{ + GatewayOrigin, PEER_PROXY_PREFIX, ROUTE_INVALIDATION_PATH, gateway_origin, handle_peer_proxy, + host_from_headers, route_public_request, }; -const SECURE_PREVIEW_COOKIE: &str = "__Host-gw_preview_access"; -const INSECURE_PREVIEW_COOKIE: &str = "gw_preview_access"; -const PREVIEW_CALLBACK_PATH: &str = "/.grass/auth/callback"; - -#[derive(Clone)] -enum ResolvedTarget { - Static { - static_dir: PathBuf, - spa_fallback: bool, - not_found: Option, - }, - Ssr { - deployment_id: Uuid, - deployment_dir: PathBuf, - server: manifest::ServerSection, - }, -} - pub struct ServeState { client: ControlApiClient, node_id: Uuid, @@ -305,1758 +295,8 @@ pub fn spawn(state: Arc, config: &NodeConfig) -> tokio::task::JoinHa }) } -/// Normalizes a public request path into safe relative segments. Returns -/// `None` for anything that tries to escape the static root. -pub fn normalize_public_path(path: &str) -> Option> { - if path.contains('\0') || path.contains('\\') { - return None; - } - // Percent-decode so encoded traversal (%2e%2e) cannot slip through. - let decoded = percent_decode(path)?; - if decoded.contains('\0') || decoded.contains('\\') { - return None; - } - - let mut segments = Vec::new(); - for segment in decoded.split('/') { - match segment { - "" | "." => continue, - ".." => return None, - segment => segments.push(segment.to_owned()), - } - } - Some(segments) -} - -fn percent_decode(input: &str) -> Option { - let bytes = input.as_bytes(); - let mut output = Vec::with_capacity(bytes.len()); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'%' => { - let high = bytes.get(index + 1)?; - let low = bytes.get(index + 2)?; - let value = - (char::from(*high).to_digit(16)? * 16 + char::from(*low).to_digit(16)?) as u8; - output.push(value); - index += 3; - } - byte => { - output.push(byte); - index += 1; - } - } - } - String::from_utf8(output).ok() -} - -/// Resolves a normalized request path against the static directory: -/// directories serve their `index.html`, missing paths fall back to the SPA -/// index when enabled. -pub fn resolve_static_file( - static_dir: &Path, - segments: &[String], - spa_fallback: bool, -) -> Option { - let mut candidate = static_dir.to_path_buf(); - for segment in segments { - candidate.push(segment); - } - - if candidate.is_dir() { - candidate.push("index.html"); - } - if candidate.is_file() { - return Some(candidate); - } - - // Pretty URLs: /about → /about.html when present. - if let Some(last) = segments.last() - && !last.contains('.') - { - let mut with_extension = static_dir.to_path_buf(); - for segment in &segments[..segments.len() - 1] { - with_extension.push(segment); - } - with_extension.push(format!("{last}.html")); - if with_extension.is_file() { - return Some(with_extension); - } - } - - if spa_fallback { - let index = static_dir.join("index.html"); - if index.is_file() { - return Some(index); - } - } - None -} - -fn resolve_not_found_file(static_dir: &Path, configured: Option<&str>) -> Option { - if let Some(configured) = configured { - let mut candidate = static_dir.to_path_buf(); - for segment in configured.trim_start_matches('/').split('/') { - candidate.push(segment); - } - if candidate.is_file() { - return Some(candidate); - } - } - - let root_404 = static_dir.join("404.html"); - root_404.is_file().then_some(root_404) -} - -const GATEWAY_TOKEN_HEADER: &str = "x-grass-gateway-token"; -const GATEWAY_HOP_HEADER: &str = "x-grass-gateway-hop"; -const PEER_PROXY_PREFIX: &str = "/_grass/internal/proxy"; -const ROUTE_INVALIDATION_PATH: &str = "/_grass/internal/routes/invalidate"; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum GatewayOrigin { - External, - Authenticated, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum RouteAction { - Local, - Proxy, -} - -fn gateway_origin( - headers: &HeaderMap, - expected_token: &str, - authentication: GatewayAuthenticationMode, -) -> Result { - let token = headers.get(GATEWAY_TOKEN_HEADER); - let hop = headers.get(GATEWAY_HOP_HEADER); - if matches!(authentication, GatewayAuthenticationMode::None) { - return match (token, hop) { - (None, None) => Ok(GatewayOrigin::External), - (None, Some(hop)) if hop.to_str().ok() == Some("1") => Ok(GatewayOrigin::Authenticated), - (None, Some(_)) => Err("invalid gateway hop"), - (Some(_), _) => Err("gateway token is not accepted in none mode"), - }; - } - match (token, hop) { - (None, None) => Ok(GatewayOrigin::External), - (Some(token), Some(hop)) => { - let token = token.to_str().map_err(|_| "invalid gateway token")?; - let hop = hop.to_str().map_err(|_| "invalid gateway hop")?; - let valid_token: bool = token.as_bytes().ct_eq(expected_token.as_bytes()).into(); - if expected_token.is_empty() || !valid_token { - return Err("invalid gateway token"); - } - if hop != "1" { - return Err("invalid gateway hop"); - } - Ok(GatewayOrigin::Authenticated) - } - _ => Err("incomplete gateway authentication"), - } -} - -fn route_action( - local_node_id: Uuid, - target_node_id: Uuid, - origin: GatewayOrigin, -) -> Result { - if local_node_id == target_node_id { - return Ok(RouteAction::Local); - } - if matches!(origin, GatewayOrigin::Authenticated) { - return Err("gateway request cannot be proxied more than once"); - } - Ok(RouteAction::Proxy) -} - -fn host_from_headers(headers: &HeaderMap) -> Option { - if headers.get_all(header::HOST).iter().count() != 1 { - return None; - } - let raw = headers.get(header::HOST)?.to_str().ok()?; - let without_port = raw.rsplit_once(':').map_or(raw, |(host, port)| { - if port.chars().all(|character| character.is_ascii_digit()) { - host - } else { - raw - } - }); - grass_validator::normalize_host(without_port).ok() -} - -fn is_preview_callback(path: &str) -> bool { - path == PREVIEW_CALLBACK_PATH -} - -fn request_destination(path_and_query: &str) -> String { - if path_and_query.is_empty() { - "/".to_owned() - } else { - path_and_query.to_owned() - } -} - -fn preview_cookie_value(cookie_header: &str) -> Option<&str> { - [SECURE_PREVIEW_COOKIE, INSECURE_PREVIEW_COOKIE] - .into_iter() - .find_map(|expected| { - cookie_header.split(';').find_map(|pair| { - let (name, value) = pair.trim().split_once('=')?; - (name == expected).then_some(value) - }) - }) -} - -fn strip_preview_cookie(cookie_header: &str) -> Option { - let cookies = cookie_header - .split(';') - .filter_map(|pair| { - let pair = pair.trim(); - let (name, _) = pair.split_once('=')?; - (![SECURE_PREVIEW_COOKIE, INSECURE_PREVIEW_COOKIE].contains(&name)).then_some(pair) - }) - .collect::>(); - (!cookies.is_empty()).then(|| cookies.join("; ")) -} - -fn preview_access_cookie(grant: &str, max_age_seconds: u64, secure: bool) -> String { - let (name, secure_attribute) = if secure { - (SECURE_PREVIEW_COOKIE, "; Secure") - } else { - (INSECURE_PREVIEW_COOKIE, "") - }; - format!( - "{name}={grant}; Path=/; Max-Age={max_age_seconds}{secure_attribute}; HttpOnly; SameSite=Lax" - ) -} - -fn clear_preview_cookies() -> Vec { - vec![ - format!("{SECURE_PREVIEW_COOKIE}=; Path=/; Max-Age=0; Secure; HttpOnly; SameSite=Lax"), - format!("{INSECURE_PREVIEW_COOKIE}=; Path=/; Max-Age=0; HttpOnly; SameSite=Lax"), - ] -} - -fn preview_cache_key(host: &str, grant: &str) -> String { - let mut digest = Sha256::new(); - digest.update(host.as_bytes()); - digest.update([0]); - digest.update(grant.as_bytes()); - hex::encode(digest.finalize()) -} - -fn callback_code(request: &Request) -> Option { - let mut codes = request - .uri() - .query() - .into_iter() - .flat_map(|query| url::form_urlencoded::parse(query.as_bytes())) - .filter(|(key, _)| key == "code") - .map(|(_, value)| value.into_owned()); - let code = codes.next().filter(|code| !code.is_empty())?; - codes.next().is_none().then_some(code) -} - -fn redirect_response(location: &str, cookies: Vec) -> Response { - let Ok(location) = HeaderValue::from_str(location) else { - return error_page( - StatusCode::BAD_GATEWAY, - "The control plane returned an invalid authorization redirect.", - ); - }; - let mut response = Response::builder() - .status(StatusCode::FOUND) - .header(header::LOCATION, location) - .header(header::CACHE_CONTROL, "no-store") - .header(header::REFERRER_POLICY, "no-referrer"); - for cookie in cookies { - let Ok(cookie) = HeaderValue::from_str(&cookie) else { - return error_page( - StatusCode::BAD_GATEWAY, - "The control plane returned an invalid preview grant.", - ); - }; - response = response.header(header::SET_COOKIE, cookie); - } - response - .body(Body::empty()) - .unwrap_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response()) -} - -async fn begin_preview_authorization( - state: &ServeState, - host: &str, - return_to: &str, - clear_cookie: bool, -) -> Response { - match state - .client - .start_preview_authorization(host, return_to) - .await - { - Ok(started) => redirect_response( - &started.authorization_url, - clear_cookie.then(clear_preview_cookies).unwrap_or_default(), - ), - Err(error) => { - warn!( - operation = "node.serve.preview_authorize", - %error, - host = %host, - "preview authorization could not start" - ); - error_page( - StatusCode::BAD_GATEWAY, - "The control plane could not authorize this preview.", - ) - } - } -} - -async fn handle_preview_callback(state: &ServeState, host: &str, code: Option) -> Response { - let Some(code) = code else { - return begin_preview_authorization(state, host, "/", true).await; - }; - match state.client.exchange_preview_code(host, &code).await { - Ok(exchanged) => redirect_response( - &exchanged.return_to, - vec![preview_access_cookie( - &exchanged.grant, - exchanged.max_age_seconds.min(12 * 60 * 60), - exchanged.cookie_secure, - )], - ), - Err(PreviewAuthError::Unauthorized) => { - begin_preview_authorization(state, host, "/", true).await - } - Err(PreviewAuthError::Forbidden) => error_page( - StatusCode::FORBIDDEN, - "Your account is not a member of the team that owns this preview.", - ), - Err(PreviewAuthError::Infrastructure(error)) => { - warn!( - operation = "node.serve.preview_exchange", - %error, - host = %host, - "preview callback exchange failed" - ); - error_page( - StatusCode::BAD_GATEWAY, - "The control plane could not complete preview authorization.", - ) - } - } -} - -#[allow(clippy::result_large_err)] -async fn require_preview_access( - state: &ServeState, - host: &str, - destination: String, - grant: Option, -) -> Result<(), Response> { - let Some(grant) = grant else { - return Err(begin_preview_authorization(state, host, &destination, false).await); - }; - - let cache_key = preview_cache_key(host, &grant); - { - let mut grants = state.preview_grants.lock().await; - if grants - .get(&cache_key) - .is_some_and(|expires_at| *expires_at > Instant::now()) - { - return Ok(()); - } - grants.remove(&cache_key); - } - - match state.client.verify_preview_grant(host, &grant).await { - Ok(verification) if verification.allowed => { - let mut grants = state.preview_grants.lock().await; - let now = Instant::now(); - grants.retain(|_, expires_at| *expires_at > now); - grants.insert(cache_key, now + state.preview_access_ttl); - Ok(()) - } - Ok(_) | Err(PreviewAuthError::Forbidden) => Err(error_page( - StatusCode::FORBIDDEN, - "Your account is not a member of the team that owns this preview.", - )), - Err(PreviewAuthError::Unauthorized) => { - Err(begin_preview_authorization(state, host, &destination, true).await) - } - Err(PreviewAuthError::Infrastructure(error)) => { - warn!( - operation = "node.serve.preview_verify", - %error, - host = %host, - "preview grant verification failed" - ); - Err(error_page( - StatusCode::BAD_GATEWAY, - "The control plane could not verify preview access.", - )) - } - } -} - -fn strip_preview_cookie_header(headers: &mut HeaderMap) { - let filtered = headers - .get(header::COOKIE) - .and_then(|value| value.to_str().ok()) - .and_then(strip_preview_cookie); - match filtered.and_then(|value| HeaderValue::from_str(&value).ok()) { - Some(value) => { - headers.insert(header::COOKIE, value); - } - None => { - headers.remove(header::COOKIE); - } - } -} - -fn strip_peer_proxy_prefix(request: &mut Request) -> Result<(), &'static str> { - let path_and_query = request - .uri() - .path_and_query() - .map(|value| value.as_str()) - .unwrap_or("/"); - let suffix = path_and_query - .strip_prefix(PEER_PROXY_PREFIX) - .ok_or("missing peer proxy prefix")?; - let restored = match suffix.chars().next() { - None => "/".to_owned(), - Some('/') => suffix.to_owned(), - Some('?') => format!("/{suffix}"), - Some(_) => return Err("invalid peer proxy path"), - }; - *request.uri_mut() = restored.parse().map_err(|_| "invalid peer proxy URI")?; - Ok(()) -} - -async fn route_public_request( - State(state): State>, - ConnectInfo(client_addr): ConnectInfo, - request: Request, -) -> Response { - let origin = match gateway_origin( - request.headers(), - state.gateway_token.as_deref().unwrap_or_default(), - state.gateway_authentication, - ) { - Ok(origin) => origin, - Err(_) => { - return error_page( - StatusCode::FORBIDDEN, - "This gateway request is not authorized.", - ); - } - }; - let Some(host) = host_from_headers(request.headers()) else { - return error_page( - StatusCode::BAD_REQUEST, - "This request has no valid Host header.", - ); - }; - - let Some(route) = state.routes.lookup(&host).await else { - return error_page( - StatusCode::NOT_FOUND, - "This host is not bound to any active deployment.", - ); - }; - if normalize_public_path(request.uri().path()).is_none() { - return error_page( - StatusCode::BAD_REQUEST, - "The requested path is not allowed.", - ); - } - match route_action(state.node_id, route.target_node_id, origin) { - Ok(RouteAction::Proxy) => { - return match forward_to_gateway( - &state.proxy, - &route.target_base_url, - state.gateway_token.as_deref().unwrap_or_default(), - route.gateway_authentication, - client_addr, - request, - ) - .await - { - Ok(response) => response, - Err(error) => { - warn!( - operation = "node.serve.gateway", - %error, - host = %host, - target_node_id = %route.target_node_id, - "Serve gateway proxy failed" - ); - error_page( - StatusCode::BAD_GATEWAY, - "The assigned Serve Node could not be reached.", - ) - } - }; - } - Ok(RouteAction::Local) => {} - Err(_) => { - return error_page( - StatusCode::BAD_GATEWAY, - "The gateway route snapshot points to another Serve Node.", - ); - } - } - - serve_local(state, route, client_addr, origin, request).await -} - -async fn handle_peer_proxy( - State(state): State>, - ConnectInfo(client_addr): ConnectInfo, - mut request: Request, -) -> Response { - if !matches!( - gateway_origin( - request.headers(), - state.gateway_token.as_deref().unwrap_or_default(), - state.gateway_authentication, - ), - Ok(GatewayOrigin::Authenticated) - ) { - return error_page( - StatusCode::FORBIDDEN, - "This gateway request is not authorized.", - ); - } - if strip_peer_proxy_prefix(&mut request).is_err() { - return error_page(StatusCode::BAD_REQUEST, "This gateway path is invalid."); - } - let Some(host) = host_from_headers(request.headers()) else { - return error_page( - StatusCode::BAD_REQUEST, - "This request has no valid Host header.", - ); - }; - let Some(route) = state.routes.lookup(&host).await else { - return error_page( - StatusCode::BAD_GATEWAY, - "The gateway route snapshot no longer contains this Host.", - ); - }; - if !matches!( - route_action( - state.node_id, - route.target_node_id, - GatewayOrigin::Authenticated, - ), - Ok(RouteAction::Local) - ) { - return error_page( - StatusCode::BAD_GATEWAY, - "The gateway route snapshot points to another Serve Node.", - ); - } - - serve_local( - state, - route, - client_addr, - GatewayOrigin::Authenticated, - request, - ) - .await -} - -async fn serve_local( - state: Arc, - route: ServeRoute, - client_addr: SocketAddr, - origin: GatewayOrigin, - mut request: Request, -) -> Response { - request.headers_mut().remove(GATEWAY_TOKEN_HEADER); - request.headers_mut().remove(GATEWAY_HOP_HEADER); - - let target = match resolve_deployment(&state, route.deployment_id).await { - Ok(target) => target, - Err(error) => { - warn!(operation = "node.serve.resolve_host", %error, host = %route.host, "local deployment resolution failed"); - return error_page( - StatusCode::BAD_GATEWAY, - "The assigned deployment is not ready on this Serve Node.", - ); - } - }; - - let requires_preview_access = matches!(route.access, ServeAccess::TeamOrPlatformAdmin); - if requires_preview_access { - if is_preview_callback(request.uri().path()) { - let code = callback_code(&request); - return handle_preview_callback(&state, &route.host, code).await; - } - let destination = request_destination( - request - .uri() - .path_and_query() - .map(|value| value.as_str()) - .unwrap_or("/"), - ); - let grant = request - .headers() - .get(header::COOKIE) - .and_then(|value| value.to_str().ok()) - .and_then(preview_cookie_value) - .map(str::to_owned); - if let Err(response) = require_preview_access(&state, &route.host, destination, grant).await - { - return response; - } - } - - match target { - ResolvedTarget::Static { - static_dir, - spa_fallback, - not_found, - } => { - let method = request.method().clone(); - let range = request.headers().get(header::RANGE).cloned(); - let Some(segments) = normalize_public_path(request.uri().path()) else { - return error_page( - StatusCode::BAD_REQUEST, - "The requested path is not allowed.", - ); - }; - - match resolve_static_file(&static_dir, &segments, spa_fallback) { - Some(file) => serve_file(&file, StatusCode::OK, &method, range.as_ref()).await, - None => { - if let Some(not_found_file) = - resolve_not_found_file(&static_dir, not_found.as_deref()) - { - return serve_file( - ¬_found_file, - StatusCode::NOT_FOUND, - &method, - range.as_ref(), - ) - .await; - } - error_page(StatusCode::NOT_FOUND, "This page could not be found.") - } - } - } - ResolvedTarget::Ssr { - deployment_id, - deployment_dir, - server, - } => { - if requires_preview_access { - strip_preview_cookie_header(request.headers_mut()); - } - let upstream = match state - .ssr - .upstream_for(deployment_id, &deployment_dir, &server, route.resources) - .await - { - Ok(upstream) => upstream, - Err(error) => { - warn!( - operation = "node.serve.ssr_start", - %error, - deployment_id = %deployment_id, - "ssr service unavailable" - ); - return error_page( - StatusCode::BAD_GATEWAY, - "The application server failed to start.", - ); - } - }; - match forward_to_ssr(&state.proxy, &upstream, client_addr, origin, request).await { - Ok(response) => response, - Err(error) => { - // A connect failure means the container died or lost its - // address; drop it so the next request restarts it. - if error.is_connect() { - state.ssr.invalidate(deployment_id).await; - } - warn!( - operation = "node.serve.ssr_proxy", - %error, - deployment_id = %deployment_id, - "ssr proxy request failed" - ); - error_page( - StatusCode::BAD_GATEWAY, - "The application server could not be reached.", - ) - } - } - } - } -} - -/// Hop-by-hop headers never forwarded in either direction. -fn is_hop_by_hop(name: &HeaderName) -> bool { - matches!( - name.as_str(), - "connection" - | "keep-alive" - | "proxy-authenticate" - | "proxy-authorization" - | "te" - | "trailer" - | "transfer-encoding" - | "upgrade" - ) -} - -fn is_gateway_internal(name: &HeaderName) -> bool { - matches!(name.as_str(), GATEWAY_TOKEN_HEADER | GATEWAY_HOP_HEADER) -} - -fn is_forwarded_metadata(name: &HeaderName) -> bool { - matches!( - name.as_str(), - "x-forwarded-for" | "x-forwarded-host" | "x-forwarded-proto" - ) -} - -/// Streams the request to the SSR upstream and the response back, keeping -/// end-to-end headers and adding the standard forwarding metadata. -async fn forward_to_ssr( - proxy: &reqwest::Client, - upstream: &str, - client_addr: SocketAddr, - origin: GatewayOrigin, - request: Request, -) -> Result { - let (parts, body) = request.into_parts(); - let path_and_query = parts - .uri - .path_and_query() - .map(|value| value.as_str()) - .unwrap_or("/"); - let url = format!("http://{upstream}{path_and_query}"); - - let mut builder = proxy.request(parts.method.clone(), url); - for (name, value) in &parts.headers { - if is_hop_by_hop(name) - || is_gateway_internal(name) - || matches!(origin, GatewayOrigin::External) && is_forwarded_metadata(name) - { - continue; - } - builder = builder.header(name, value); - } - if matches!(origin, GatewayOrigin::External) { - builder = builder - .header( - "x-forwarded-proto", - if parts.extensions.get::().is_some() { - "https" - } else { - "http" - }, - ) - .header("x-forwarded-for", client_addr.ip().to_string()); - if let Some(host) = parts.headers.get(header::HOST) { - builder = builder.header("x-forwarded-host", host); - } - } - - let upstream_response = builder - .body(reqwest::Body::wrap_stream(body.into_data_stream())) - .send() - .await?; - - let mut response = Response::builder().status(upstream_response.status()); - for (name, value) in upstream_response.headers() { - if is_hop_by_hop(name) || is_gateway_internal(name) { - continue; - } - response = response.header(name, value); - } - Ok(response - .body(Body::from_stream(upstream_response.bytes_stream())) - .unwrap_or_else(|_| StatusCode::BAD_GATEWAY.into_response())) -} - -async fn forward_to_gateway( - proxy: &reqwest::Client, - target_base_url: &str, - gateway_token: &str, - gateway_authentication: GatewayAuthenticationMode, - client_addr: SocketAddr, - request: Request, -) -> anyhow::Result { - if matches!(gateway_authentication, GatewayAuthenticationMode::Token) - && gateway_token.is_empty() - { - anyhow::bail!("destination gateway requires an outbound credential"); - } - let (parts, body) = request.into_parts(); - // URL parsers resolve literal and encoded dot segments. Validate before - // adding credentials so a public path cannot escape the peer endpoint. - if normalize_public_path(parts.uri.path()).is_none() { - anyhow::bail!("invalid gateway request path"); - } - let mut url = url::Url::parse(target_base_url) - .map_err(|error| anyhow::anyhow!("invalid target Serve Node URL: {error}"))?; - url.set_path(&format!("{PEER_PROXY_PREFIX}{}", parts.uri.path())); - url.set_query(parts.uri.query()); - url.set_fragment(None); - if !url.path().starts_with(&format!("{PEER_PROXY_PREFIX}/")) { - anyhow::bail!("gateway request escaped the peer endpoint"); - } - - let mut builder = proxy.request(parts.method, url); - for (name, value) in &parts.headers { - if is_hop_by_hop(name) || is_gateway_internal(name) || is_forwarded_metadata(name) { - continue; - } - builder = builder.header(name, value); - } - builder = builder.header(GATEWAY_HOP_HEADER, "1"); - if matches!(gateway_authentication, GatewayAuthenticationMode::Token) { - builder = builder.header(GATEWAY_TOKEN_HEADER, gateway_token); - } - builder = builder - .header("x-forwarded-for", client_addr.ip().to_string()) - .header( - "x-forwarded-proto", - if parts.extensions.get::().is_some() { - "https" - } else { - "http" - }, - ); - if let Some(host) = parts.headers.get(header::HOST) { - builder = builder.header("x-forwarded-host", host); - } - let upstream = builder - .body(reqwest::Body::wrap_stream(body.into_data_stream())) - .send() - .await?; - - let mut response = Response::builder().status(upstream.status()); - for (name, value) in upstream.headers() { - if !is_hop_by_hop(name) && !is_gateway_internal(name) { - response = response.header(name, value); - } - } - Ok(response - .body(Body::from_stream(upstream.bytes_stream())) - .unwrap_or_else(|_| StatusCode::BAD_GATEWAY.into_response())) -} - -async fn serve_file( - path: &Path, - status: StatusCode, - method: &axum::http::Method, - range: Option<&axum::http::HeaderValue>, -) -> Response { - match static_files::serve_file(path, method, range, status).await { - Ok(response) => response, - Err(_) => error_page(StatusCode::NOT_FOUND, "This page could not be found."), - } -} - -fn error_page(status: StatusCode, message: &str) -> Response { - let body = format!( - "{code}\ - \ -

{code}

{message}

grass-worker node

", - code = status.as_u16(), - ); - Response::builder() - .status(status) - .header(header::CONTENT_TYPE, "text/html; charset=utf-8") - .header(header::CACHE_CONTROL, "no-store") - .header(header::REFERRER_POLICY, "no-referrer") - .body(Body::from(body)) - .unwrap_or_else(|_| status.into_response()) -} - -async fn resolve_deployment( - state: &ServeState, - deployment_id: Uuid, -) -> anyhow::Result { - if let Some(target) = state.targets.lock().await.get(&deployment_id).cloned() { - return Ok(target); - } - let target = ensure_artifact(state, deployment_id).await?; - state - .targets - .lock() - .await - .insert(deployment_id, target.clone()); - Ok(target) -} - -/// Loads the already staged deployment artifact and returns the serve target -/// described by its manifest. -async fn ensure_artifact( - state: &ServeState, - deployment_id: Uuid, -) -> anyhow::Result { - let deployment_dir = sync::staged_artifact_path(&state.cache_root, deployment_id)?; - let manifest_path = deployment_dir.join("output.toml"); - - let manifest_content = tokio::fs::read_to_string(&manifest_path).await?; - let manifest = manifest::parse_manifest(&manifest_content) - .map_err(|error| anyhow::anyhow!("invalid output manifest: {error}"))?; - manifest::validate_manifest(&manifest, &deployment_dir) - .map_err(|error| anyhow::anyhow!("invalid output manifest: {error}"))?; - - if manifest.runtime.kind == "ssr" { - let server = manifest - .server - .ok_or_else(|| anyhow::anyhow!("ssr manifest has no server section"))?; - return Ok(ResolvedTarget::Ssr { - deployment_id, - deployment_dir, - server, - }); - } - - let static_section = manifest - .static_site - .ok_or_else(|| anyhow::anyhow!("manifest has no static section"))?; - - Ok(ResolvedTarget::Static { - static_dir: deployment_dir.join(static_section.directory), - spa_fallback: static_section.spa_fallback, - not_found: (!static_section.not_found.trim().is_empty()) - .then(|| static_section.not_found.trim().to_owned()), - }) -} - #[cfg(test)] mod release_smoke; #[cfg(test)] -mod tests { - use super::*; - - fn static_site(spa: bool) -> PathBuf { - let dir = - std::env::temp_dir().join(format!("grass-serve-{}", uuid::Uuid::now_v7().simple())); - std::fs::create_dir_all(dir.join("assets")).unwrap(); - std::fs::create_dir_all(dir.join("docs")).unwrap(); - std::fs::write(dir.join("index.html"), "index").unwrap(); - std::fs::write(dir.join("about.html"), "about").unwrap(); - std::fs::write(dir.join("assets/app.js"), "js").unwrap(); - std::fs::write(dir.join("docs/index.html"), "docs").unwrap(); - let _ = spa; - dir - } - - #[test] - fn public_paths_are_normalized_and_traversal_is_rejected() { - assert_eq!(normalize_public_path("/"), Some(vec![])); - assert_eq!( - normalize_public_path("/assets/app.js"), - Some(vec!["assets".to_owned(), "app.js".to_owned()]) - ); - assert_eq!( - normalize_public_path("/a/./b"), - Some(vec!["a".to_owned(), "b".to_owned()]) - ); - assert_eq!(normalize_public_path("/../etc/passwd"), None); - assert_eq!(normalize_public_path("/a/../../etc"), None); - assert_eq!(normalize_public_path("/%2e%2e/secret"), None); - assert_eq!(normalize_public_path("/a%2F..%2F..%2Fetc"), None); - assert_eq!(normalize_public_path("/back\\slash"), None); - } - - #[test] - fn static_resolution_serves_index_pretty_urls_and_spa_fallback() { - let dir = static_site(true); - - // Root and directory index. - assert_eq!( - resolve_static_file(&dir, &[], false).unwrap(), - dir.join("index.html") - ); - assert_eq!( - resolve_static_file(&dir, &["docs".to_owned()], false).unwrap(), - dir.join("docs/index.html") - ); - - // Direct file and pretty URL. - assert_eq!( - resolve_static_file(&dir, &["assets".to_owned(), "app.js".to_owned()], false).unwrap(), - dir.join("assets/app.js") - ); - assert_eq!( - resolve_static_file(&dir, &["about".to_owned()], false).unwrap(), - dir.join("about.html") - ); - - // SPA fallback on unknown routes only when enabled. - assert_eq!( - resolve_static_file(&dir, &["missing".to_owned()], true).unwrap(), - dir.join("index.html") - ); - assert_eq!( - resolve_static_file(&dir, &["missing".to_owned()], false), - None - ); - - std::fs::remove_dir_all(dir).unwrap(); - } - - #[test] - fn missing_static_paths_select_custom_then_root_404() { - let dir = static_site(false); - std::fs::create_dir_all(dir.join("errors")).unwrap(); - std::fs::write(dir.join("errors/not-found.html"), "custom").unwrap(); - std::fs::write(dir.join("404.html"), "root").unwrap(); - - assert_eq!( - resolve_not_found_file(&dir, Some("errors/not-found.html")), - Some(dir.join("errors/not-found.html")) - ); - assert_eq!( - resolve_not_found_file(&dir, Some("missing.html")), - Some(dir.join("404.html")) - ); - - std::fs::remove_file(dir.join("404.html")).unwrap(); - assert_eq!(resolve_not_found_file(&dir, None), None); - std::fs::remove_dir_all(dir).unwrap(); - } - - #[test] - fn preview_cookie_contract_and_ssr_filtering_are_host_scoped() { - assert_eq!( - preview_access_cookie("opaque", 43_200, true), - "__Host-gw_preview_access=opaque; Path=/; Max-Age=43200; Secure; HttpOnly; SameSite=Lax" - ); - assert_eq!( - preview_access_cookie("opaque", 43_200, false), - "gw_preview_access=opaque; Path=/; Max-Age=43200; HttpOnly; SameSite=Lax" - ); - assert_eq!( - preview_cookie_value( - "app=1; __Host-gw_preview_access=secure; gw_preview_access=plain; theme=dark" - ), - Some("secure") - ); - assert_eq!( - preview_cookie_value("app=1; gw_preview_access=plain; theme=dark"), - Some("plain") - ); - assert_eq!( - strip_preview_cookie( - "app=1; __Host-gw_preview_access=secure; gw_preview_access=plain; theme=dark" - ), - Some("app=1; theme=dark".to_owned()) - ); - assert_eq!( - strip_preview_cookie("__Host-gw_preview_access=opaque"), - None - ); - assert_eq!( - clear_preview_cookies(), - vec![ - "__Host-gw_preview_access=; Path=/; Max-Age=0; Secure; HttpOnly; SameSite=Lax" - .to_owned(), - "gw_preview_access=; Path=/; Max-Age=0; HttpOnly; SameSite=Lax".to_owned(), - ] - ); - } - - #[test] - fn preview_callback_is_reserved_and_destinations_keep_the_query() { - assert!(is_preview_callback("/.grass/auth/callback")); - assert!(!is_preview_callback("/.grass/auth/callback/child")); - assert_eq!(request_destination("/docs?q=1"), "/docs?q=1"); - assert_eq!(request_destination(""), "/"); - } - - #[test] - fn platform_error_pages_do_not_send_authorization_urls_as_referrers() { - let response = error_page(StatusCode::BAD_GATEWAY, "unavailable"); - assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); - assert_eq!(response.headers()[header::REFERRER_POLICY], "no-referrer"); - } - - #[test] - fn preview_redirect_can_clear_secure_and_http_development_cookies() { - let response = redirect_response("/", clear_preview_cookies()); - assert_eq!( - response - .headers() - .get_all(header::SET_COOKIE) - .iter() - .count(), - 2 - ); - } - - #[test] - fn host_header_parsing_strips_ports_and_normalizes() { - let mut headers = HeaderMap::new(); - headers.insert(header::HOST, "Demo.Grass.Test:8080".parse().unwrap()); - assert_eq!( - host_from_headers(&headers).as_deref(), - Some("demo.grass.test") - ); - - headers.insert(header::HOST, "demo.grass.test".parse().unwrap()); - assert_eq!( - host_from_headers(&headers).as_deref(), - Some("demo.grass.test") - ); - - headers.insert(header::HOST, "..".parse().unwrap()); - assert_eq!(host_from_headers(&headers), None); - } - - #[test] - fn gateway_hops_authenticate_and_never_reproxy() { - let token = "shared-gateway-token"; - let mut headers = HeaderMap::new(); - let external = gateway_origin(&headers, token, GatewayAuthenticationMode::Token).unwrap(); - assert_eq!(external, GatewayOrigin::External); - assert_eq!( - route_action(Uuid::nil(), Uuid::now_v7(), external).unwrap(), - RouteAction::Proxy - ); - - headers.insert("x-grass-gateway-token", token.parse().unwrap()); - headers.insert("x-grass-gateway-hop", "1".parse().unwrap()); - let authenticated = - gateway_origin(&headers, token, GatewayAuthenticationMode::Token).unwrap(); - assert_eq!(authenticated, GatewayOrigin::Authenticated); - assert!(route_action(Uuid::nil(), Uuid::now_v7(), authenticated).is_err()); - - headers.insert("x-grass-gateway-token", "wrong-token".parse().unwrap()); - assert!(gateway_origin(&headers, token, GatewayAuthenticationMode::Token).is_err()); - headers.insert("x-grass-gateway-token", token.parse().unwrap()); - headers.insert("x-grass-gateway-hop", "2".parse().unwrap()); - assert!(gateway_origin(&headers, token, GatewayAuthenticationMode::Token).is_err()); - - headers.remove(GATEWAY_TOKEN_HEADER); - headers.insert(GATEWAY_HOP_HEADER, "1".parse().unwrap()); - assert_eq!( - gateway_origin(&headers, token, GatewayAuthenticationMode::None).unwrap(), - GatewayOrigin::Authenticated - ); - headers.insert(GATEWAY_HOP_HEADER, "2".parse().unwrap()); - assert!(gateway_origin(&headers, token, GatewayAuthenticationMode::None).is_err()); - headers.insert(GATEWAY_TOKEN_HEADER, token.parse().unwrap()); - assert!(gateway_origin(&headers, token, GatewayAuthenticationMode::None).is_err()); - } - - #[test] - fn peer_proxy_prefix_is_removed_without_changing_path_or_query() { - let mut request = Request::builder() - .uri("/_grass/internal/proxy/submit/item?preview=1") - .body(Body::empty()) - .unwrap(); - - strip_peer_proxy_prefix(&mut request).unwrap(); - - assert_eq!( - request.uri().path_and_query().unwrap().as_str(), - "/submit/item?preview=1" - ); - } - - #[tokio::test] - async fn gateway_proxy_preserves_request_and_adds_single_hop_auth() { - let app = axum::Router::new().fallback(|request: Request| async move { - assert_eq!(request.method(), axum::http::Method::POST); - assert_eq!( - request.uri().path_and_query().unwrap().as_str(), - "/_grass/internal/proxy/submit/%2Fitem?preview=1" - ); - assert_eq!(request.headers()[header::HOST], "app.example.com"); - assert_eq!(request.headers()[header::AUTHORIZATION], "Bearer app-token"); - assert_eq!( - request.headers()[GATEWAY_TOKEN_HEADER], - "shared-gateway-token" - ); - assert_eq!(request.headers()[GATEWAY_HOP_HEADER], "1"); - assert_eq!(request.headers()["x-forwarded-for"], "192.0.2.10"); - assert_eq!(request.headers()["x-forwarded-host"], "app.example.com"); - assert_eq!(request.headers()["x-forwarded-proto"], "http"); - let body = axum::body::to_bytes(request.into_body(), 1024) - .await - .unwrap(); - assert_eq!(body, "payload"); - Response::new(Body::from("proxied")) - }); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - let proxy = reqwest::Client::new(); - let request = Request::builder() - .method("POST") - .uri("/submit/%2Fitem?preview=1") - .header(header::HOST, "app.example.com") - .header(header::AUTHORIZATION, "Bearer app-token") - .header("x-forwarded-for", "203.0.113.99") - .header("x-forwarded-host", "spoofed.example.com") - .header("x-forwarded-proto", "https") - .body(Body::from("payload")) - .unwrap(); - - let response = forward_to_gateway( - &proxy, - &format!("http://{address}"), - "shared-gateway-token", - GatewayAuthenticationMode::Token, - "192.0.2.10:43123".parse().unwrap(), - request, - ) - .await - .unwrap(); - - assert_eq!(response.status(), StatusCode::OK); - let body = axum::body::to_bytes(response.into_body(), 1024) - .await - .unwrap(); - assert_eq!(body, "proxied"); - server.abort(); - } - - #[tokio::test] - async fn ssr_proxy_sanitizes_external_headers_and_preserves_gateway_metadata() { - let app = axum::Router::new().fallback(|request: Request| async move { - let headers = request.headers(); - assert_eq!(headers[header::AUTHORIZATION], "Bearer app-token"); - assert!(!headers.contains_key(GATEWAY_TOKEN_HEADER)); - assert!(!headers.contains_key(GATEWAY_HOP_HEADER)); - match request.uri().path() { - "/external" => { - assert_eq!(headers["x-forwarded-for"], "192.0.2.20"); - assert_eq!(headers["x-forwarded-host"], "app.example.com"); - assert_eq!(headers["x-forwarded-proto"], "http"); - } - "/peer" => { - assert_eq!(headers["x-forwarded-for"], "198.51.100.40"); - assert_eq!(headers["x-forwarded-host"], "app.example.com"); - assert_eq!(headers["x-forwarded-proto"], "https"); - } - path => panic!("unexpected SSR test path: {path}"), - } - Response::new(Body::empty()) - }); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - let proxy = reqwest::Client::new(); - - let external = Request::builder() - .uri("/external") - .header(header::HOST, "app.example.com") - .header(header::AUTHORIZATION, "Bearer app-token") - .header(GATEWAY_TOKEN_HEADER, "must-not-leak") - .header(GATEWAY_HOP_HEADER, "1") - .header("x-forwarded-for", "203.0.113.99") - .header("x-forwarded-host", "spoofed.example.com") - .header("x-forwarded-proto", "https") - .body(Body::empty()) - .unwrap(); - forward_to_ssr( - &proxy, - &address.to_string(), - "192.0.2.20:41234".parse().unwrap(), - GatewayOrigin::External, - external, - ) - .await - .unwrap(); - - let peer = Request::builder() - .uri("/peer") - .header(header::HOST, "app.example.com") - .header(header::AUTHORIZATION, "Bearer app-token") - .header(GATEWAY_TOKEN_HEADER, "must-not-leak") - .header(GATEWAY_HOP_HEADER, "1") - .header("x-forwarded-for", "198.51.100.40") - .header("x-forwarded-host", "app.example.com") - .header("x-forwarded-proto", "https") - .body(Body::empty()) - .unwrap(); - forward_to_ssr( - &proxy, - &address.to_string(), - "127.0.0.1:51234".parse().unwrap(), - GatewayOrigin::Authenticated, - peer, - ) - .await - .unwrap(); - - server.abort(); - } - - #[tokio::test] - async fn route_invalidation_is_authenticated_and_removes_cached_access_before_acknowledging() { - let authority = axum::Router::new().fallback(|| async { "stale deployment" }); - let authority_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let authority_address = authority_listener.local_addr().unwrap(); - let authority_server = - tokio::spawn(async move { axum::serve(authority_listener, authority).await.unwrap() }); - - let config = NodeConfig::default(); - let node_id = Uuid::now_v7(); - let deployment_id = Uuid::now_v7(); - let routes = Arc::new(routes::RouteTable::default()); - routes - .apply(grass_node_protocol::RouteSnapshotResponse { - revision: "before-withdrawal".to_owned(), - routes: vec![ServeRoute { - host: "app.example.com".to_owned(), - region: "default".to_owned(), - deployment_id, - target_node_id: Uuid::now_v7(), - target_base_url: format!("http://{authority_address}"), - gateway_authentication: Default::default(), - resources: grass_node_protocol::ServeResources { - cpu_millicores: 50, - memory_mb: 64, - disk_mb: 256, - }, - access: ServeAccess::Public, - }], - }) - .await - .unwrap(); - let ssr = Arc::new(ssr::SsrManager::new(None, node_id, &config)); - let state = Arc::new(ServeState::new( - ControlApiClient::new(&format!("http://{authority_address}"), "node-token").unwrap(), - node_id, - Some("shared-gateway-token".to_owned()), - routes, - &config, - ssr, - )); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { - axum::serve( - listener, - serve_router(state).into_make_service_with_connect_info::(), - ) - .await - .unwrap() - }); - - let client = reqwest::Client::new(); - let rejected = client - .post(format!( - "http://{address}/_grass/internal/routes/invalidate" - )) - .header(GATEWAY_TOKEN_HEADER, "wrong-token") - .json(&serde_json::json!({ "deployment_id": deployment_id })) - .send() - .await - .unwrap(); - assert_eq!(rejected.status(), StatusCode::FORBIDDEN); - - let stale = client - .get(format!("http://{address}/")) - .header(header::HOST, "app.example.com") - .send() - .await - .unwrap(); - assert_eq!(stale.status(), StatusCode::OK); - - let invalidation = client - .post(format!( - "http://{address}/_grass/internal/routes/invalidate" - )) - .header(GATEWAY_TOKEN_HEADER, "shared-gateway-token") - .header(GATEWAY_HOP_HEADER, "1") - .json(&serde_json::json!({ "deployment_id": deployment_id })) - .send() - .await - .unwrap(); - assert_eq!(invalidation.status(), StatusCode::OK); - - let response = client - .get(format!("http://{address}/")) - .header(header::HOST, "app.example.com") - .send() - .await - .unwrap(); - - assert_eq!(response.status(), StatusCode::NOT_FOUND); - server.abort(); - authority_server.abort(); - } - - #[tokio::test] - async fn peer_endpoint_requires_gateway_auth_before_route_lookup() { - let config = NodeConfig::default(); - let routes = Arc::new(routes::RouteTable::default()); - let ssr = Arc::new(ssr::SsrManager::new(None, Uuid::now_v7(), &config)); - let state = Arc::new(ServeState::new( - ControlApiClient::new("http://127.0.0.1:9", "node-token").unwrap(), - Uuid::now_v7(), - Some("shared-gateway-token".to_owned()), - routes, - &config, - ssr, - )); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { - axum::serve( - listener, - serve_router(state).into_make_service_with_connect_info::(), - ) - .await - .unwrap() - }); - let client = reqwest::Client::new(); - let endpoint = format!("http://{address}{PEER_PROXY_PREFIX}/path"); - - let missing = client - .get(&endpoint) - .header(header::HOST, "app.example.com") - .send() - .await - .unwrap(); - assert_eq!(missing.status(), StatusCode::FORBIDDEN); - - let wrong = client - .get(&endpoint) - .header(header::HOST, "app.example.com") - .header(GATEWAY_TOKEN_HEADER, "wrong-token") - .header(GATEWAY_HOP_HEADER, "1") - .send() - .await - .unwrap(); - assert_eq!(wrong.status(), StatusCode::FORBIDDEN); - - let authorized = client - .get(&endpoint) - .header(header::HOST, "app.example.com") - .header(GATEWAY_TOKEN_HEADER, "shared-gateway-token") - .header(GATEWAY_HOP_HEADER, "1") - .send() - .await - .unwrap(); - assert_eq!(authorized.status(), StatusCode::BAD_GATEWAY); - - server.abort(); - } - - #[tokio::test] - async fn mixed_gateway_modes_deliver_bound_hosts_and_reject_second_hops() { - use grass_node_protocol::{RouteSnapshotResponse, ServeResources}; - for source_mode in [ - GatewayAuthenticationMode::Token, - GatewayAuthenticationMode::None, - ] { - for target_mode in [ - GatewayAuthenticationMode::Token, - GatewayAuthenticationMode::None, - ] { - let directory = tempfile::tempdir().unwrap(); - tokio::fs::write(directory.path().join("index.html"), "regional site") - .await - .unwrap(); - let destination_id = Uuid::now_v7(); - let deployment_id = Uuid::now_v7(); - let destination_listener = - tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let destination_url = - format!("http://{}", destination_listener.local_addr().unwrap()); - let route = ServeRoute { - host: "app.example.com".to_owned(), - region: "eu-west".to_owned(), - deployment_id, - target_node_id: destination_id, - target_base_url: destination_url.clone(), - gateway_authentication: target_mode, - resources: ServeResources { - cpu_millicores: 50, - memory_mb: 64, - disk_mb: 256, - }, - access: ServeAccess::Public, - }; - let make_state = |node_id, mode| { - let mut config = NodeConfig::default(); - config.security.gateway_authentication = mode; - Arc::new(ServeState::new( - ControlApiClient::new("http://127.0.0.1:9", "node-token").unwrap(), - node_id, - Some("shared-gateway-token".to_owned()), - Arc::new(routes::RouteTable::default()), - &config, - Arc::new(ssr::SsrManager::new(None, node_id, &config)), - )) - }; - let destination = make_state(destination_id, target_mode); - destination - .routes - .apply(RouteSnapshotResponse { - revision: "target".to_owned(), - routes: vec![route.clone()], - }) - .await - .unwrap(); - destination.targets.lock().await.insert( - deployment_id, - ResolvedTarget::Static { - static_dir: directory.path().to_owned(), - spa_fallback: false, - not_found: None, - }, - ); - let destination_server = tokio::spawn(async move { - axum::serve( - destination_listener, - serve_router(destination) - .into_make_service_with_connect_info::(), - ) - .await - .unwrap(); - }); - let source = make_state(Uuid::now_v7(), source_mode); - source - .routes - .apply(RouteSnapshotResponse { - revision: "source".to_owned(), - routes: vec![route], - }) - .await - .unwrap(); - let source_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let source_url = format!("http://{}", source_listener.local_addr().unwrap()); - let source_server = tokio::spawn(async move { - axum::serve( - source_listener, - serve_router(source).into_make_service_with_connect_info::(), - ) - .await - .unwrap(); - }); - let client = reqwest::Client::new(); - let response = client - .get(&source_url) - .header(header::HOST, "app.example.com") - .send() - .await - .unwrap(); - assert_eq!( - response.status(), - StatusCode::OK, - "{source_mode:?} -> {target_mode:?}" - ); - assert_eq!(response.text().await.unwrap(), "regional site"); - let unbound = client - .get(&source_url) - .header(header::HOST, "unbound.example.com") - .send() - .await - .unwrap(); - assert_eq!(unbound.status(), StatusCode::NOT_FOUND); - let mut repeated = client - .get(format!("{source_url}{PEER_PROXY_PREFIX}/")) - .header(header::HOST, "app.example.com") - .header(GATEWAY_HOP_HEADER, "1"); - if source_mode == GatewayAuthenticationMode::Token { - repeated = repeated.header(GATEWAY_TOKEN_HEADER, "shared-gateway-token"); - } - assert_eq!( - repeated.send().await.unwrap().status(), - StatusCode::BAD_GATEWAY - ); - source_server.abort(); - destination_server.abort(); - } - } - } - - #[tokio::test] - async fn missing_outbound_token_fails_before_contacting_destination() { - let request = Request::builder() - .header(header::HOST, "app.example.com") - .body(Body::empty()) - .unwrap(); - let error = forward_to_gateway( - &reqwest::Client::new(), - "http://127.0.0.1:9", - "", - GatewayAuthenticationMode::Token, - "127.0.0.1:1234".parse().unwrap(), - request, - ) - .await - .unwrap_err(); - assert_eq!( - error.to_string(), - "destination gateway requires an outbound credential" - ); - let mut headers = HeaderMap::new(); - headers.insert(GATEWAY_TOKEN_HEADER, "".parse().unwrap()); - headers.insert(GATEWAY_HOP_HEADER, "1".parse().unwrap()); - assert!(gateway_origin(&headers, "", GatewayAuthenticationMode::Token).is_err()); - } - - #[tokio::test] - async fn gateway_redirects_and_traversal_never_send_credentials_to_other_endpoints() { - use std::sync::atomic::{AtomicUsize, Ordering}; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - let captured = Arc::new(AtomicUsize::new(0)); - let capture_router = axum::Router::new().fallback({ - let captured = captured.clone(); - move || { - let captured = captured.clone(); - async move { - captured.fetch_add(1, Ordering::SeqCst); - StatusCode::OK - } - } - }); - let capture_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let capture_url = format!("http://{}/capture", capture_listener.local_addr().unwrap()); - let capture_server = tokio::spawn(async move { - axum::serve(capture_listener, capture_router).await.unwrap(); - }); - - let peer_requests = Arc::new(AtomicUsize::new(0)); - let peer_router = axum::Router::new().fallback({ - let destination = capture_url.clone(); - let peer_requests = peer_requests.clone(); - move |request: Request| { - let destination = destination.clone(); - let peer_requests = peer_requests.clone(); - async move { - peer_requests.fetch_add(1, Ordering::SeqCst); - assert_eq!( - request.headers()[GATEWAY_TOKEN_HEADER], - "shared-gateway-token" - ); - assert_eq!(request.headers()[GATEWAY_HOP_HEADER], "1"); - assert_eq!(request.headers()[header::HOST], "app.example.com"); - assert!(request.uri().path().starts_with(PEER_PROXY_PREFIX)); - (StatusCode::FOUND, [(header::LOCATION, destination)]) - } - } - }); - let peer_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let peer_url = format!("http://{}", peer_listener.local_addr().unwrap()); - let peer_server = tokio::spawn(async move { - axum::serve(peer_listener, peer_router).await.unwrap(); - }); - - let mut config = NodeConfig::default(); - config.security.gateway_authentication = GatewayAuthenticationMode::None; - let node_id = Uuid::now_v7(); - let routes = Arc::new(routes::RouteTable::default()); - routes - .apply(grass_node_protocol::RouteSnapshotResponse { - revision: "security".to_owned(), - routes: vec![ServeRoute { - host: "app.example.com".to_owned(), - region: "default".to_owned(), - deployment_id: Uuid::now_v7(), - target_node_id: Uuid::now_v7(), - target_base_url: peer_url.clone(), - gateway_authentication: GatewayAuthenticationMode::Token, - resources: grass_node_protocol::ServeResources { - cpu_millicores: 50, - memory_mb: 64, - disk_mb: 256, - }, - access: ServeAccess::Public, - }], - }) - .await - .unwrap(); - let state = Arc::new(ServeState::new( - ControlApiClient::new("http://127.0.0.1:9", "node-token").unwrap(), - node_id, - Some("shared-gateway-token".to_owned()), - routes, - &config, - Arc::new(ssr::SsrManager::new(None, node_id, &config)), - )); - let source_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let source_address = source_listener.local_addr().unwrap(); - let router = serve_router(state.clone()); - let source_server = tokio::spawn(async move { - axum::serve( - source_listener, - router.into_make_service_with_connect_info::(), - ) - .await - .unwrap(); - }); - let browser = reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .unwrap(); - let response = browser - .get(format!("http://{source_address}/redirect")) - .header(header::HOST, "app.example.com") - .send() - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::FOUND); - assert_eq!(response.headers()[header::LOCATION], capture_url); - assert_eq!(peer_requests.load(Ordering::SeqCst), 1); - assert_eq!(captured.load(Ordering::SeqCst), 0); - - for path in [ - "/../../../_grass/internal/routes/invalidate", - "/%2e%2e/%2e%2e/%2e%2e/_grass/internal/routes/invalidate", - "/.%2E/.%2E/.%2E/_grass/internal/routes/invalidate", - "/%5c../%5c../_grass/internal/routes/invalidate", - ] { - // Send raw HTTP to preserve the malicious path instead of the - // test HTTP client's own URL parser normalizing it in advance. - let mut stream = tokio::net::TcpStream::connect(source_address) - .await - .unwrap(); - stream.write_all(format!("POST {path} HTTP/1.1\r\nHost: app.example.com\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").as_bytes()).await.unwrap(); - let mut response = Vec::new(); - stream.read_to_end(&mut response).await.unwrap(); - assert!( - response.starts_with(b"HTTP/1.1 400"), - "{path}: {}", - String::from_utf8_lossy(&response) - ); - let request = Request::builder() - .uri(path) - .header(header::HOST, "app.example.com") - .body(Body::empty()) - .unwrap(); - let error = forward_to_gateway( - &state.proxy, - &peer_url, - "shared-gateway-token", - GatewayAuthenticationMode::Token, - source_address, - request, - ) - .await - .unwrap_err(); - assert_eq!(error.to_string(), "invalid gateway request path"); - } - assert_eq!(peer_requests.load(Ordering::SeqCst), 1); - assert_eq!(captured.load(Ordering::SeqCst), 0); - source_server.abort(); - peer_server.abort(); - capture_server.abort(); - } -} +mod tests; diff --git a/apps/node/src/serve/paths.rs b/apps/node/src/serve/paths.rs new file mode 100644 index 0000000..d9a9c14 --- /dev/null +++ b/apps/node/src/serve/paths.rs @@ -0,0 +1,110 @@ +//! Public path normalization and static-file selection. + +use std::path::{Path, PathBuf}; + +/// Normalizes a public request path into safe relative segments. Returns +/// `None` for anything that tries to escape the static root. +pub fn normalize_public_path(path: &str) -> Option> { + if path.contains('\0') || path.contains('\\') { + return None; + } + // Percent-decode so encoded traversal (%2e%2e) cannot slip through. + let decoded = percent_decode(path)?; + if decoded.contains('\0') || decoded.contains('\\') { + return None; + } + + let mut segments = Vec::new(); + for segment in decoded.split('/') { + match segment { + "" | "." => continue, + ".." => return None, + segment => segments.push(segment.to_owned()), + } + } + Some(segments) +} + +fn percent_decode(input: &str) -> Option { + let bytes = input.as_bytes(); + let mut output = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + match bytes[index] { + b'%' => { + let high = bytes.get(index + 1)?; + let low = bytes.get(index + 2)?; + let value = + (char::from(*high).to_digit(16)? * 16 + char::from(*low).to_digit(16)?) as u8; + output.push(value); + index += 3; + } + byte => { + output.push(byte); + index += 1; + } + } + } + String::from_utf8(output).ok() +} + +/// Resolves a normalized request path against the static directory: +/// directories serve their `index.html`, missing paths fall back to the SPA +/// index when enabled. +pub fn resolve_static_file( + static_dir: &Path, + segments: &[String], + spa_fallback: bool, +) -> Option { + let mut candidate = static_dir.to_path_buf(); + for segment in segments { + candidate.push(segment); + } + + if candidate.is_dir() { + candidate.push("index.html"); + } + if candidate.is_file() { + return Some(candidate); + } + + // Pretty URLs: /about → /about.html when present. + if let Some(last) = segments.last() + && !last.contains('.') + { + let mut with_extension = static_dir.to_path_buf(); + for segment in &segments[..segments.len() - 1] { + with_extension.push(segment); + } + with_extension.push(format!("{last}.html")); + if with_extension.is_file() { + return Some(with_extension); + } + } + + if spa_fallback { + let index = static_dir.join("index.html"); + if index.is_file() { + return Some(index); + } + } + None +} + +pub(super) fn resolve_not_found_file( + static_dir: &Path, + configured: Option<&str>, +) -> Option { + if let Some(configured) = configured { + let mut candidate = static_dir.to_path_buf(); + for segment in configured.trim_start_matches('/').split('/') { + candidate.push(segment); + } + if candidate.is_file() { + return Some(candidate); + } + } + + let root_404 = static_dir.join("404.html"); + root_404.is_file().then_some(root_404) +} diff --git a/apps/node/src/serve/preview.rs b/apps/node/src/serve/preview.rs new file mode 100644 index 0000000..afd215a --- /dev/null +++ b/apps/node/src/serve/preview.rs @@ -0,0 +1,255 @@ +//! Host-scoped preview grants, authorization callbacks and cookie handling. + +use std::time::Instant; + +use axum::{ + body::Body, + extract::Request, + http::{HeaderMap, HeaderValue, StatusCode, header}, + response::{IntoResponse, Response}, +}; +use sha2::{Digest, Sha256}; +use tracing::warn; + +use super::{ServeState, response::error_page}; +use crate::client::PreviewAuthError; + +pub(super) const SECURE_PREVIEW_COOKIE: &str = "__Host-gw_preview_access"; +pub(super) const INSECURE_PREVIEW_COOKIE: &str = "gw_preview_access"; +pub(super) const PREVIEW_CALLBACK_PATH: &str = "/.grass/auth/callback"; + +pub(super) fn is_preview_callback(path: &str) -> bool { + path == PREVIEW_CALLBACK_PATH +} + +pub(super) fn request_destination(path_and_query: &str) -> String { + if path_and_query.is_empty() { + "/".to_owned() + } else { + path_and_query.to_owned() + } +} + +pub(super) fn preview_cookie_value(cookie_header: &str) -> Option<&str> { + [SECURE_PREVIEW_COOKIE, INSECURE_PREVIEW_COOKIE] + .into_iter() + .find_map(|expected| { + cookie_header.split(';').find_map(|pair| { + let (name, value) = pair.trim().split_once('=')?; + (name == expected).then_some(value) + }) + }) +} + +pub(super) fn strip_preview_cookie(cookie_header: &str) -> Option { + let cookies = cookie_header + .split(';') + .filter_map(|pair| { + let pair = pair.trim(); + let (name, _) = pair.split_once('=')?; + (![SECURE_PREVIEW_COOKIE, INSECURE_PREVIEW_COOKIE].contains(&name)).then_some(pair) + }) + .collect::>(); + (!cookies.is_empty()).then(|| cookies.join("; ")) +} + +pub(super) fn preview_access_cookie(grant: &str, max_age_seconds: u64, secure: bool) -> String { + let (name, secure_attribute) = if secure { + (SECURE_PREVIEW_COOKIE, "; Secure") + } else { + (INSECURE_PREVIEW_COOKIE, "") + }; + format!( + "{name}={grant}; Path=/; Max-Age={max_age_seconds}{secure_attribute}; HttpOnly; SameSite=Lax" + ) +} + +pub(super) fn clear_preview_cookies() -> Vec { + vec![ + format!("{SECURE_PREVIEW_COOKIE}=; Path=/; Max-Age=0; Secure; HttpOnly; SameSite=Lax"), + format!("{INSECURE_PREVIEW_COOKIE}=; Path=/; Max-Age=0; HttpOnly; SameSite=Lax"), + ] +} + +pub(super) fn preview_cache_key(host: &str, grant: &str) -> String { + let mut digest = Sha256::new(); + digest.update(host.as_bytes()); + digest.update([0]); + digest.update(grant.as_bytes()); + hex::encode(digest.finalize()) +} + +pub(super) fn callback_code(request: &Request) -> Option { + let mut codes = request + .uri() + .query() + .into_iter() + .flat_map(|query| url::form_urlencoded::parse(query.as_bytes())) + .filter(|(key, _)| key == "code") + .map(|(_, value)| value.into_owned()); + let code = codes.next().filter(|code| !code.is_empty())?; + codes.next().is_none().then_some(code) +} + +pub(super) fn redirect_response(location: &str, cookies: Vec) -> Response { + let Ok(location) = HeaderValue::from_str(location) else { + return error_page( + StatusCode::BAD_GATEWAY, + "The control plane returned an invalid authorization redirect.", + ); + }; + let mut response = Response::builder() + .status(StatusCode::FOUND) + .header(header::LOCATION, location) + .header(header::CACHE_CONTROL, "no-store") + .header(header::REFERRER_POLICY, "no-referrer"); + for cookie in cookies { + let Ok(cookie) = HeaderValue::from_str(&cookie) else { + return error_page( + StatusCode::BAD_GATEWAY, + "The control plane returned an invalid preview grant.", + ); + }; + response = response.header(header::SET_COOKIE, cookie); + } + response + .body(Body::empty()) + .unwrap_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response()) +} + +async fn begin_preview_authorization( + state: &ServeState, + host: &str, + return_to: &str, + clear_cookie: bool, +) -> Response { + match state + .client + .start_preview_authorization(host, return_to) + .await + { + Ok(started) => redirect_response( + &started.authorization_url, + clear_cookie.then(clear_preview_cookies).unwrap_or_default(), + ), + Err(error) => { + warn!( + operation = "node.serve.preview_authorize", + %error, + host = %host, + "preview authorization could not start" + ); + error_page( + StatusCode::BAD_GATEWAY, + "The control plane could not authorize this preview.", + ) + } + } +} + +pub(super) async fn handle_preview_callback( + state: &ServeState, + host: &str, + code: Option, +) -> Response { + let Some(code) = code else { + return begin_preview_authorization(state, host, "/", true).await; + }; + match state.client.exchange_preview_code(host, &code).await { + Ok(exchanged) => redirect_response( + &exchanged.return_to, + vec![preview_access_cookie( + &exchanged.grant, + exchanged.max_age_seconds.min(12 * 60 * 60), + exchanged.cookie_secure, + )], + ), + Err(PreviewAuthError::Unauthorized) => { + begin_preview_authorization(state, host, "/", true).await + } + Err(PreviewAuthError::Forbidden) => error_page( + StatusCode::FORBIDDEN, + "Your account is not a member of the team that owns this preview.", + ), + Err(PreviewAuthError::Infrastructure(error)) => { + warn!( + operation = "node.serve.preview_exchange", + %error, + host = %host, + "preview callback exchange failed" + ); + error_page( + StatusCode::BAD_GATEWAY, + "The control plane could not complete preview authorization.", + ) + } + } +} + +#[allow(clippy::result_large_err)] +pub(super) async fn require_preview_access( + state: &ServeState, + host: &str, + destination: String, + grant: Option, +) -> Result<(), Response> { + let Some(grant) = grant else { + return Err(begin_preview_authorization(state, host, &destination, false).await); + }; + + let cache_key = preview_cache_key(host, &grant); + { + let mut grants = state.preview_grants.lock().await; + if grants + .get(&cache_key) + .is_some_and(|expires_at| *expires_at > Instant::now()) + { + return Ok(()); + } + grants.remove(&cache_key); + } + + match state.client.verify_preview_grant(host, &grant).await { + Ok(verification) if verification.allowed => { + let mut grants = state.preview_grants.lock().await; + let now = Instant::now(); + grants.retain(|_, expires_at| *expires_at > now); + grants.insert(cache_key, now + state.preview_access_ttl); + Ok(()) + } + Ok(_) | Err(PreviewAuthError::Forbidden) => Err(error_page( + StatusCode::FORBIDDEN, + "Your account is not a member of the team that owns this preview.", + )), + Err(PreviewAuthError::Unauthorized) => { + Err(begin_preview_authorization(state, host, &destination, true).await) + } + Err(PreviewAuthError::Infrastructure(error)) => { + warn!( + operation = "node.serve.preview_verify", + %error, + host = %host, + "preview grant verification failed" + ); + Err(error_page( + StatusCode::BAD_GATEWAY, + "The control plane could not verify preview access.", + )) + } + } +} + +pub(super) fn strip_preview_cookie_header(headers: &mut HeaderMap) { + let filtered = headers + .get(header::COOKIE) + .and_then(|value| value.to_str().ok()) + .and_then(strip_preview_cookie); + match filtered.and_then(|value| HeaderValue::from_str(&value).ok()) { + Some(value) => { + headers.insert(header::COOKIE, value); + } + None => { + headers.remove(header::COOKIE); + } + } +} diff --git a/apps/node/src/serve/proxy.rs b/apps/node/src/serve/proxy.rs new file mode 100644 index 0000000..8401c91 --- /dev/null +++ b/apps/node/src/serve/proxy.rs @@ -0,0 +1,171 @@ +//! Streaming peer and SSR forwarding with boundary-specific header filtering. + +use std::net::SocketAddr; + +use axum::{ + body::Body, + extract::Request, + http::{HeaderName, StatusCode, header}, + response::{IntoResponse, Response}, +}; +use grass_node_protocol::GatewayAuthenticationMode; + +use super::{ + normalize_public_path, + routing::{GATEWAY_HOP_HEADER, GATEWAY_TOKEN_HEADER, GatewayOrigin, PEER_PROXY_PREFIX}, + tls, +}; + +/// Hop-by-hop headers never forwarded in either direction. +fn is_hop_by_hop(name: &HeaderName) -> bool { + matches!( + name.as_str(), + "connection" + | "keep-alive" + | "proxy-authenticate" + | "proxy-authorization" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + ) +} + +fn is_gateway_internal(name: &HeaderName) -> bool { + matches!(name.as_str(), GATEWAY_TOKEN_HEADER | GATEWAY_HOP_HEADER) +} + +fn is_forwarded_metadata(name: &HeaderName) -> bool { + matches!( + name.as_str(), + "x-forwarded-for" | "x-forwarded-host" | "x-forwarded-proto" + ) +} + +/// Streams the request to the SSR upstream and the response back, keeping +/// end-to-end headers and adding the standard forwarding metadata. +pub(super) async fn forward_to_ssr( + proxy: &reqwest::Client, + upstream: &str, + client_addr: SocketAddr, + origin: GatewayOrigin, + request: Request, +) -> Result { + let (parts, body) = request.into_parts(); + let path_and_query = parts + .uri + .path_and_query() + .map(|value| value.as_str()) + .unwrap_or("/"); + let url = format!("http://{upstream}{path_and_query}"); + + let mut builder = proxy.request(parts.method.clone(), url); + for (name, value) in &parts.headers { + if is_hop_by_hop(name) + || is_gateway_internal(name) + || matches!(origin, GatewayOrigin::External) && is_forwarded_metadata(name) + { + continue; + } + builder = builder.header(name, value); + } + if matches!(origin, GatewayOrigin::External) { + builder = builder + .header( + "x-forwarded-proto", + if parts.extensions.get::().is_some() { + "https" + } else { + "http" + }, + ) + .header("x-forwarded-for", client_addr.ip().to_string()); + if let Some(host) = parts.headers.get(header::HOST) { + builder = builder.header("x-forwarded-host", host); + } + } + + let upstream_response = builder + .body(reqwest::Body::wrap_stream(body.into_data_stream())) + .send() + .await?; + + let mut response = Response::builder().status(upstream_response.status()); + for (name, value) in upstream_response.headers() { + if is_hop_by_hop(name) || is_gateway_internal(name) { + continue; + } + response = response.header(name, value); + } + Ok(response + .body(Body::from_stream(upstream_response.bytes_stream())) + .unwrap_or_else(|_| StatusCode::BAD_GATEWAY.into_response())) +} + +pub(super) async fn forward_to_gateway( + proxy: &reqwest::Client, + target_base_url: &str, + gateway_token: &str, + gateway_authentication: GatewayAuthenticationMode, + client_addr: SocketAddr, + request: Request, +) -> anyhow::Result { + if matches!(gateway_authentication, GatewayAuthenticationMode::Token) + && gateway_token.is_empty() + { + anyhow::bail!("destination gateway requires an outbound credential"); + } + let (parts, body) = request.into_parts(); + // URL parsers resolve literal and encoded dot segments. Validate before + // adding credentials so a public path cannot escape the peer endpoint. + if normalize_public_path(parts.uri.path()).is_none() { + anyhow::bail!("invalid gateway request path"); + } + let mut url = url::Url::parse(target_base_url) + .map_err(|error| anyhow::anyhow!("invalid target Serve Node URL: {error}"))?; + url.set_path(&format!("{PEER_PROXY_PREFIX}{}", parts.uri.path())); + url.set_query(parts.uri.query()); + url.set_fragment(None); + if !url.path().starts_with(&format!("{PEER_PROXY_PREFIX}/")) { + anyhow::bail!("gateway request escaped the peer endpoint"); + } + + let mut builder = proxy.request(parts.method, url); + for (name, value) in &parts.headers { + if is_hop_by_hop(name) || is_gateway_internal(name) || is_forwarded_metadata(name) { + continue; + } + builder = builder.header(name, value); + } + builder = builder.header(GATEWAY_HOP_HEADER, "1"); + if matches!(gateway_authentication, GatewayAuthenticationMode::Token) { + builder = builder.header(GATEWAY_TOKEN_HEADER, gateway_token); + } + builder = builder + .header("x-forwarded-for", client_addr.ip().to_string()) + .header( + "x-forwarded-proto", + if parts.extensions.get::().is_some() { + "https" + } else { + "http" + }, + ); + if let Some(host) = parts.headers.get(header::HOST) { + builder = builder.header("x-forwarded-host", host); + } + let upstream = builder + .body(reqwest::Body::wrap_stream(body.into_data_stream())) + .send() + .await?; + + let mut response = Response::builder().status(upstream.status()); + for (name, value) in upstream.headers() { + if !is_hop_by_hop(name) && !is_gateway_internal(name) { + response = response.header(name, value); + } + } + Ok(response + .body(Body::from_stream(upstream.bytes_stream())) + .unwrap_or_else(|_| StatusCode::BAD_GATEWAY.into_response())) +} diff --git a/apps/node/src/serve/response.rs b/apps/node/src/serve/response.rs new file mode 100644 index 0000000..29b743b --- /dev/null +++ b/apps/node/src/serve/response.rs @@ -0,0 +1,23 @@ +//! Shared public error responses. + +use axum::{ + body::Body, + http::{StatusCode, header}, + response::{IntoResponse, Response}, +}; + +pub(super) fn error_page(status: StatusCode, message: &str) -> Response { + let body = format!( + "{code}\ + \ +

{code}

{message}

grass-worker node

", + code = status.as_u16(), + ); + Response::builder() + .status(status) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CACHE_CONTROL, "no-store") + .header(header::REFERRER_POLICY, "no-referrer") + .body(Body::from(body)) + .unwrap_or_else(|_| status.into_response()) +} diff --git a/apps/node/src/serve/routing.rs b/apps/node/src/serve/routing.rs new file mode 100644 index 0000000..628530d --- /dev/null +++ b/apps/node/src/serve/routing.rs @@ -0,0 +1,250 @@ +//! Host binding, gateway authentication and single-hop route dispatch. + +use std::{net::SocketAddr, sync::Arc}; + +use axum::{ + extract::{ConnectInfo, Request, State}, + http::{HeaderMap, StatusCode, header}, + response::Response, +}; +use grass_node_protocol::GatewayAuthenticationMode; +use subtle::ConstantTimeEq; +use tracing::warn; +use uuid::Uuid; + +use super::{ + ServeState, local::serve_local, normalize_public_path, proxy::forward_to_gateway, + response::error_page, +}; + +pub(super) const GATEWAY_TOKEN_HEADER: &str = "x-grass-gateway-token"; +pub(super) const GATEWAY_HOP_HEADER: &str = "x-grass-gateway-hop"; +pub(super) const PEER_PROXY_PREFIX: &str = "/_grass/internal/proxy"; +pub(super) const ROUTE_INVALIDATION_PATH: &str = "/_grass/internal/routes/invalidate"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum GatewayOrigin { + External, + Authenticated, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum RouteAction { + Local, + Proxy, +} + +pub(super) fn gateway_origin( + headers: &HeaderMap, + expected_token: &str, + authentication: GatewayAuthenticationMode, +) -> Result { + let token = headers.get(GATEWAY_TOKEN_HEADER); + let hop = headers.get(GATEWAY_HOP_HEADER); + if matches!(authentication, GatewayAuthenticationMode::None) { + return match (token, hop) { + (None, None) => Ok(GatewayOrigin::External), + (None, Some(hop)) if hop.to_str().ok() == Some("1") => Ok(GatewayOrigin::Authenticated), + (None, Some(_)) => Err("invalid gateway hop"), + (Some(_), _) => Err("gateway token is not accepted in none mode"), + }; + } + match (token, hop) { + (None, None) => Ok(GatewayOrigin::External), + (Some(token), Some(hop)) => { + let token = token.to_str().map_err(|_| "invalid gateway token")?; + let hop = hop.to_str().map_err(|_| "invalid gateway hop")?; + let valid_token: bool = token.as_bytes().ct_eq(expected_token.as_bytes()).into(); + if expected_token.is_empty() || !valid_token { + return Err("invalid gateway token"); + } + if hop != "1" { + return Err("invalid gateway hop"); + } + Ok(GatewayOrigin::Authenticated) + } + _ => Err("incomplete gateway authentication"), + } +} + +pub(super) fn route_action( + local_node_id: Uuid, + target_node_id: Uuid, + origin: GatewayOrigin, +) -> Result { + if local_node_id == target_node_id { + return Ok(RouteAction::Local); + } + if matches!(origin, GatewayOrigin::Authenticated) { + return Err("gateway request cannot be proxied more than once"); + } + Ok(RouteAction::Proxy) +} + +pub(super) fn host_from_headers(headers: &HeaderMap) -> Option { + if headers.get_all(header::HOST).iter().count() != 1 { + return None; + } + let raw = headers.get(header::HOST)?.to_str().ok()?; + let without_port = raw.rsplit_once(':').map_or(raw, |(host, port)| { + if port.chars().all(|character| character.is_ascii_digit()) { + host + } else { + raw + } + }); + grass_validator::normalize_host(without_port).ok() +} + +pub(super) fn strip_peer_proxy_prefix(request: &mut Request) -> Result<(), &'static str> { + let path_and_query = request + .uri() + .path_and_query() + .map(|value| value.as_str()) + .unwrap_or("/"); + let suffix = path_and_query + .strip_prefix(PEER_PROXY_PREFIX) + .ok_or("missing peer proxy prefix")?; + let restored = match suffix.chars().next() { + None => "/".to_owned(), + Some('/') => suffix.to_owned(), + Some('?') => format!("/{suffix}"), + Some(_) => return Err("invalid peer proxy path"), + }; + *request.uri_mut() = restored.parse().map_err(|_| "invalid peer proxy URI")?; + Ok(()) +} + +pub(super) async fn route_public_request( + State(state): State>, + ConnectInfo(client_addr): ConnectInfo, + request: Request, +) -> Response { + let origin = match gateway_origin( + request.headers(), + state.gateway_token.as_deref().unwrap_or_default(), + state.gateway_authentication, + ) { + Ok(origin) => origin, + Err(_) => { + return error_page( + StatusCode::FORBIDDEN, + "This gateway request is not authorized.", + ); + } + }; + let Some(host) = host_from_headers(request.headers()) else { + return error_page( + StatusCode::BAD_REQUEST, + "This request has no valid Host header.", + ); + }; + + let Some(route) = state.routes.lookup(&host).await else { + return error_page( + StatusCode::NOT_FOUND, + "This host is not bound to any active deployment.", + ); + }; + if normalize_public_path(request.uri().path()).is_none() { + return error_page( + StatusCode::BAD_REQUEST, + "The requested path is not allowed.", + ); + } + match route_action(state.node_id, route.target_node_id, origin) { + Ok(RouteAction::Proxy) => { + return match forward_to_gateway( + &state.proxy, + &route.target_base_url, + state.gateway_token.as_deref().unwrap_or_default(), + route.gateway_authentication, + client_addr, + request, + ) + .await + { + Ok(response) => response, + Err(error) => { + warn!( + operation = "node.serve.gateway", + %error, + host = %host, + target_node_id = %route.target_node_id, + "Serve gateway proxy failed" + ); + error_page( + StatusCode::BAD_GATEWAY, + "The assigned Serve Node could not be reached.", + ) + } + }; + } + Ok(RouteAction::Local) => {} + Err(_) => { + return error_page( + StatusCode::BAD_GATEWAY, + "The gateway route snapshot points to another Serve Node.", + ); + } + } + + serve_local(state, route, client_addr, origin, request).await +} + +pub(super) async fn handle_peer_proxy( + State(state): State>, + ConnectInfo(client_addr): ConnectInfo, + mut request: Request, +) -> Response { + if !matches!( + gateway_origin( + request.headers(), + state.gateway_token.as_deref().unwrap_or_default(), + state.gateway_authentication, + ), + Ok(GatewayOrigin::Authenticated) + ) { + return error_page( + StatusCode::FORBIDDEN, + "This gateway request is not authorized.", + ); + } + if strip_peer_proxy_prefix(&mut request).is_err() { + return error_page(StatusCode::BAD_REQUEST, "This gateway path is invalid."); + } + let Some(host) = host_from_headers(request.headers()) else { + return error_page( + StatusCode::BAD_REQUEST, + "This request has no valid Host header.", + ); + }; + let Some(route) = state.routes.lookup(&host).await else { + return error_page( + StatusCode::BAD_GATEWAY, + "The gateway route snapshot no longer contains this Host.", + ); + }; + if !matches!( + route_action( + state.node_id, + route.target_node_id, + GatewayOrigin::Authenticated, + ), + Ok(RouteAction::Local) + ) { + return error_page( + StatusCode::BAD_GATEWAY, + "The gateway route snapshot points to another Serve Node.", + ); + } + + serve_local( + state, + route, + client_addr, + GatewayOrigin::Authenticated, + request, + ) + .await +} diff --git a/apps/node/src/serve/tests.rs b/apps/node/src/serve/tests.rs new file mode 100644 index 0000000..4b44e68 --- /dev/null +++ b/apps/node/src/serve/tests.rs @@ -0,0 +1,822 @@ +use super::*; +use super::{ + paths::resolve_not_found_file, + preview::*, + proxy::{forward_to_gateway, forward_to_ssr}, + response::error_page, + routing::{ + GATEWAY_HOP_HEADER, GATEWAY_TOKEN_HEADER, RouteAction, route_action, + strip_peer_proxy_prefix, + }, +}; +use axum::body::Body; +use grass_node_protocol::{ServeAccess, ServeRoute}; + +fn static_site(spa: bool) -> PathBuf { + let dir = std::env::temp_dir().join(format!("grass-serve-{}", uuid::Uuid::now_v7().simple())); + std::fs::create_dir_all(dir.join("assets")).unwrap(); + std::fs::create_dir_all(dir.join("docs")).unwrap(); + std::fs::write(dir.join("index.html"), "index").unwrap(); + std::fs::write(dir.join("about.html"), "about").unwrap(); + std::fs::write(dir.join("assets/app.js"), "js").unwrap(); + std::fs::write(dir.join("docs/index.html"), "docs").unwrap(); + let _ = spa; + dir +} + +#[test] +fn public_paths_are_normalized_and_traversal_is_rejected() { + assert_eq!(normalize_public_path("/"), Some(vec![])); + assert_eq!( + normalize_public_path("/assets/app.js"), + Some(vec!["assets".to_owned(), "app.js".to_owned()]) + ); + assert_eq!( + normalize_public_path("/a/./b"), + Some(vec!["a".to_owned(), "b".to_owned()]) + ); + assert_eq!(normalize_public_path("/../etc/passwd"), None); + assert_eq!(normalize_public_path("/a/../../etc"), None); + assert_eq!(normalize_public_path("/%2e%2e/secret"), None); + assert_eq!(normalize_public_path("/a%2F..%2F..%2Fetc"), None); + assert_eq!(normalize_public_path("/back\\slash"), None); +} + +#[test] +fn static_resolution_serves_index_pretty_urls_and_spa_fallback() { + let dir = static_site(true); + + // Root and directory index. + assert_eq!( + resolve_static_file(&dir, &[], false).unwrap(), + dir.join("index.html") + ); + assert_eq!( + resolve_static_file(&dir, &["docs".to_owned()], false).unwrap(), + dir.join("docs/index.html") + ); + + // Direct file and pretty URL. + assert_eq!( + resolve_static_file(&dir, &["assets".to_owned(), "app.js".to_owned()], false).unwrap(), + dir.join("assets/app.js") + ); + assert_eq!( + resolve_static_file(&dir, &["about".to_owned()], false).unwrap(), + dir.join("about.html") + ); + + // SPA fallback on unknown routes only when enabled. + assert_eq!( + resolve_static_file(&dir, &["missing".to_owned()], true).unwrap(), + dir.join("index.html") + ); + assert_eq!( + resolve_static_file(&dir, &["missing".to_owned()], false), + None + ); + + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn missing_static_paths_select_custom_then_root_404() { + let dir = static_site(false); + std::fs::create_dir_all(dir.join("errors")).unwrap(); + std::fs::write(dir.join("errors/not-found.html"), "custom").unwrap(); + std::fs::write(dir.join("404.html"), "root").unwrap(); + + assert_eq!( + resolve_not_found_file(&dir, Some("errors/not-found.html")), + Some(dir.join("errors/not-found.html")) + ); + assert_eq!( + resolve_not_found_file(&dir, Some("missing.html")), + Some(dir.join("404.html")) + ); + + std::fs::remove_file(dir.join("404.html")).unwrap(); + assert_eq!(resolve_not_found_file(&dir, None), None); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn preview_cookie_contract_and_ssr_filtering_are_host_scoped() { + assert_eq!( + preview_access_cookie("opaque", 43_200, true), + "__Host-gw_preview_access=opaque; Path=/; Max-Age=43200; Secure; HttpOnly; SameSite=Lax" + ); + assert_eq!( + preview_access_cookie("opaque", 43_200, false), + "gw_preview_access=opaque; Path=/; Max-Age=43200; HttpOnly; SameSite=Lax" + ); + assert_eq!( + preview_cookie_value( + "app=1; __Host-gw_preview_access=secure; gw_preview_access=plain; theme=dark" + ), + Some("secure") + ); + assert_eq!( + preview_cookie_value("app=1; gw_preview_access=plain; theme=dark"), + Some("plain") + ); + assert_eq!( + strip_preview_cookie( + "app=1; __Host-gw_preview_access=secure; gw_preview_access=plain; theme=dark" + ), + Some("app=1; theme=dark".to_owned()) + ); + assert_eq!( + strip_preview_cookie("__Host-gw_preview_access=opaque"), + None + ); + assert_eq!( + clear_preview_cookies(), + vec![ + "__Host-gw_preview_access=; Path=/; Max-Age=0; Secure; HttpOnly; SameSite=Lax" + .to_owned(), + "gw_preview_access=; Path=/; Max-Age=0; HttpOnly; SameSite=Lax".to_owned(), + ] + ); +} + +#[test] +fn preview_callback_is_reserved_and_destinations_keep_the_query() { + assert!(is_preview_callback("/.grass/auth/callback")); + assert!(!is_preview_callback("/.grass/auth/callback/child")); + assert_eq!(request_destination("/docs?q=1"), "/docs?q=1"); + assert_eq!(request_destination(""), "/"); +} + +#[test] +fn platform_error_pages_do_not_send_authorization_urls_as_referrers() { + let response = error_page(StatusCode::BAD_GATEWAY, "unavailable"); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + assert_eq!(response.headers()[header::REFERRER_POLICY], "no-referrer"); +} + +#[test] +fn preview_redirect_can_clear_secure_and_http_development_cookies() { + let response = redirect_response("/", clear_preview_cookies()); + assert_eq!( + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .count(), + 2 + ); +} + +#[test] +fn host_header_parsing_strips_ports_and_normalizes() { + let mut headers = HeaderMap::new(); + headers.insert(header::HOST, "Demo.Grass.Test:8080".parse().unwrap()); + assert_eq!( + host_from_headers(&headers).as_deref(), + Some("demo.grass.test") + ); + + headers.insert(header::HOST, "demo.grass.test".parse().unwrap()); + assert_eq!( + host_from_headers(&headers).as_deref(), + Some("demo.grass.test") + ); + + headers.insert(header::HOST, "..".parse().unwrap()); + assert_eq!(host_from_headers(&headers), None); +} + +#[test] +fn gateway_hops_authenticate_and_never_reproxy() { + let token = "shared-gateway-token"; + let mut headers = HeaderMap::new(); + let external = gateway_origin(&headers, token, GatewayAuthenticationMode::Token).unwrap(); + assert_eq!(external, GatewayOrigin::External); + assert_eq!( + route_action(Uuid::nil(), Uuid::now_v7(), external).unwrap(), + RouteAction::Proxy + ); + + headers.insert("x-grass-gateway-token", token.parse().unwrap()); + headers.insert("x-grass-gateway-hop", "1".parse().unwrap()); + let authenticated = gateway_origin(&headers, token, GatewayAuthenticationMode::Token).unwrap(); + assert_eq!(authenticated, GatewayOrigin::Authenticated); + assert!(route_action(Uuid::nil(), Uuid::now_v7(), authenticated).is_err()); + + headers.insert("x-grass-gateway-token", "wrong-token".parse().unwrap()); + assert!(gateway_origin(&headers, token, GatewayAuthenticationMode::Token).is_err()); + headers.insert("x-grass-gateway-token", token.parse().unwrap()); + headers.insert("x-grass-gateway-hop", "2".parse().unwrap()); + assert!(gateway_origin(&headers, token, GatewayAuthenticationMode::Token).is_err()); + + headers.remove(GATEWAY_TOKEN_HEADER); + headers.insert(GATEWAY_HOP_HEADER, "1".parse().unwrap()); + assert_eq!( + gateway_origin(&headers, token, GatewayAuthenticationMode::None).unwrap(), + GatewayOrigin::Authenticated + ); + headers.insert(GATEWAY_HOP_HEADER, "2".parse().unwrap()); + assert!(gateway_origin(&headers, token, GatewayAuthenticationMode::None).is_err()); + headers.insert(GATEWAY_TOKEN_HEADER, token.parse().unwrap()); + assert!(gateway_origin(&headers, token, GatewayAuthenticationMode::None).is_err()); +} + +#[test] +fn peer_proxy_prefix_is_removed_without_changing_path_or_query() { + let mut request = Request::builder() + .uri("/_grass/internal/proxy/submit/item?preview=1") + .body(Body::empty()) + .unwrap(); + + strip_peer_proxy_prefix(&mut request).unwrap(); + + assert_eq!( + request.uri().path_and_query().unwrap().as_str(), + "/submit/item?preview=1" + ); +} + +#[tokio::test] +async fn gateway_proxy_preserves_request_and_adds_single_hop_auth() { + let app = axum::Router::new().fallback(|request: Request| async move { + assert_eq!(request.method(), axum::http::Method::POST); + assert_eq!( + request.uri().path_and_query().unwrap().as_str(), + "/_grass/internal/proxy/submit/%2Fitem?preview=1" + ); + assert_eq!(request.headers()[header::HOST], "app.example.com"); + assert_eq!(request.headers()[header::AUTHORIZATION], "Bearer app-token"); + assert_eq!( + request.headers()[GATEWAY_TOKEN_HEADER], + "shared-gateway-token" + ); + assert_eq!(request.headers()[GATEWAY_HOP_HEADER], "1"); + assert_eq!(request.headers()["x-forwarded-for"], "192.0.2.10"); + assert_eq!(request.headers()["x-forwarded-host"], "app.example.com"); + assert_eq!(request.headers()["x-forwarded-proto"], "http"); + let body = axum::body::to_bytes(request.into_body(), 1024) + .await + .unwrap(); + assert_eq!(body, "payload"); + Response::new(Body::from("proxied")) + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let proxy = reqwest::Client::new(); + let request = Request::builder() + .method("POST") + .uri("/submit/%2Fitem?preview=1") + .header(header::HOST, "app.example.com") + .header(header::AUTHORIZATION, "Bearer app-token") + .header("x-forwarded-for", "203.0.113.99") + .header("x-forwarded-host", "spoofed.example.com") + .header("x-forwarded-proto", "https") + .body(Body::from("payload")) + .unwrap(); + + let response = forward_to_gateway( + &proxy, + &format!("http://{address}"), + "shared-gateway-token", + GatewayAuthenticationMode::Token, + "192.0.2.10:43123".parse().unwrap(), + request, + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(); + assert_eq!(body, "proxied"); + server.abort(); +} + +#[tokio::test] +async fn ssr_proxy_sanitizes_external_headers_and_preserves_gateway_metadata() { + let app = axum::Router::new().fallback(|request: Request| async move { + let headers = request.headers(); + assert_eq!(headers[header::AUTHORIZATION], "Bearer app-token"); + assert!(!headers.contains_key(GATEWAY_TOKEN_HEADER)); + assert!(!headers.contains_key(GATEWAY_HOP_HEADER)); + match request.uri().path() { + "/external" => { + assert_eq!(headers["x-forwarded-for"], "192.0.2.20"); + assert_eq!(headers["x-forwarded-host"], "app.example.com"); + assert_eq!(headers["x-forwarded-proto"], "http"); + } + "/peer" => { + assert_eq!(headers["x-forwarded-for"], "198.51.100.40"); + assert_eq!(headers["x-forwarded-host"], "app.example.com"); + assert_eq!(headers["x-forwarded-proto"], "https"); + } + path => panic!("unexpected SSR test path: {path}"), + } + Response::new(Body::empty()) + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let proxy = reqwest::Client::new(); + + let external = Request::builder() + .uri("/external") + .header(header::HOST, "app.example.com") + .header(header::AUTHORIZATION, "Bearer app-token") + .header(GATEWAY_TOKEN_HEADER, "must-not-leak") + .header(GATEWAY_HOP_HEADER, "1") + .header("x-forwarded-for", "203.0.113.99") + .header("x-forwarded-host", "spoofed.example.com") + .header("x-forwarded-proto", "https") + .body(Body::empty()) + .unwrap(); + forward_to_ssr( + &proxy, + &address.to_string(), + "192.0.2.20:41234".parse().unwrap(), + GatewayOrigin::External, + external, + ) + .await + .unwrap(); + + let peer = Request::builder() + .uri("/peer") + .header(header::HOST, "app.example.com") + .header(header::AUTHORIZATION, "Bearer app-token") + .header(GATEWAY_TOKEN_HEADER, "must-not-leak") + .header(GATEWAY_HOP_HEADER, "1") + .header("x-forwarded-for", "198.51.100.40") + .header("x-forwarded-host", "app.example.com") + .header("x-forwarded-proto", "https") + .body(Body::empty()) + .unwrap(); + forward_to_ssr( + &proxy, + &address.to_string(), + "127.0.0.1:51234".parse().unwrap(), + GatewayOrigin::Authenticated, + peer, + ) + .await + .unwrap(); + + server.abort(); +} + +#[tokio::test] +async fn route_invalidation_is_authenticated_and_removes_cached_access_before_acknowledging() { + let authority = axum::Router::new().fallback(|| async { "stale deployment" }); + let authority_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let authority_address = authority_listener.local_addr().unwrap(); + let authority_server = + tokio::spawn(async move { axum::serve(authority_listener, authority).await.unwrap() }); + + let config = NodeConfig::default(); + let node_id = Uuid::now_v7(); + let deployment_id = Uuid::now_v7(); + let routes = Arc::new(routes::RouteTable::default()); + routes + .apply(grass_node_protocol::RouteSnapshotResponse { + revision: "before-withdrawal".to_owned(), + routes: vec![ServeRoute { + host: "app.example.com".to_owned(), + region: "default".to_owned(), + deployment_id, + target_node_id: Uuid::now_v7(), + target_base_url: format!("http://{authority_address}"), + gateway_authentication: Default::default(), + resources: grass_node_protocol::ServeResources { + cpu_millicores: 50, + memory_mb: 64, + disk_mb: 256, + }, + access: ServeAccess::Public, + }], + }) + .await + .unwrap(); + let ssr = Arc::new(ssr::SsrManager::new(None, node_id, &config)); + let state = Arc::new(ServeState::new( + ControlApiClient::new(&format!("http://{authority_address}"), "node-token").unwrap(), + node_id, + Some("shared-gateway-token".to_owned()), + routes, + &config, + ssr, + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve( + listener, + serve_router(state).into_make_service_with_connect_info::(), + ) + .await + .unwrap() + }); + + let client = reqwest::Client::new(); + let rejected = client + .post(format!( + "http://{address}/_grass/internal/routes/invalidate" + )) + .header(GATEWAY_TOKEN_HEADER, "wrong-token") + .json(&serde_json::json!({ "deployment_id": deployment_id })) + .send() + .await + .unwrap(); + assert_eq!(rejected.status(), StatusCode::FORBIDDEN); + + let stale = client + .get(format!("http://{address}/")) + .header(header::HOST, "app.example.com") + .send() + .await + .unwrap(); + assert_eq!(stale.status(), StatusCode::OK); + + let invalidation = client + .post(format!( + "http://{address}/_grass/internal/routes/invalidate" + )) + .header(GATEWAY_TOKEN_HEADER, "shared-gateway-token") + .header(GATEWAY_HOP_HEADER, "1") + .json(&serde_json::json!({ "deployment_id": deployment_id })) + .send() + .await + .unwrap(); + assert_eq!(invalidation.status(), StatusCode::OK); + + let response = client + .get(format!("http://{address}/")) + .header(header::HOST, "app.example.com") + .send() + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + server.abort(); + authority_server.abort(); +} + +#[tokio::test] +async fn peer_endpoint_requires_gateway_auth_before_route_lookup() { + let config = NodeConfig::default(); + let routes = Arc::new(routes::RouteTable::default()); + let ssr = Arc::new(ssr::SsrManager::new(None, Uuid::now_v7(), &config)); + let state = Arc::new(ServeState::new( + ControlApiClient::new("http://127.0.0.1:9", "node-token").unwrap(), + Uuid::now_v7(), + Some("shared-gateway-token".to_owned()), + routes, + &config, + ssr, + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve( + listener, + serve_router(state).into_make_service_with_connect_info::(), + ) + .await + .unwrap() + }); + let client = reqwest::Client::new(); + let endpoint = format!("http://{address}{PEER_PROXY_PREFIX}/path"); + + let missing = client + .get(&endpoint) + .header(header::HOST, "app.example.com") + .send() + .await + .unwrap(); + assert_eq!(missing.status(), StatusCode::FORBIDDEN); + + let wrong = client + .get(&endpoint) + .header(header::HOST, "app.example.com") + .header(GATEWAY_TOKEN_HEADER, "wrong-token") + .header(GATEWAY_HOP_HEADER, "1") + .send() + .await + .unwrap(); + assert_eq!(wrong.status(), StatusCode::FORBIDDEN); + + let authorized = client + .get(&endpoint) + .header(header::HOST, "app.example.com") + .header(GATEWAY_TOKEN_HEADER, "shared-gateway-token") + .header(GATEWAY_HOP_HEADER, "1") + .send() + .await + .unwrap(); + assert_eq!(authorized.status(), StatusCode::BAD_GATEWAY); + + server.abort(); +} + +#[tokio::test] +async fn mixed_gateway_modes_deliver_bound_hosts_and_reject_second_hops() { + use grass_node_protocol::{RouteSnapshotResponse, ServeResources}; + for source_mode in [ + GatewayAuthenticationMode::Token, + GatewayAuthenticationMode::None, + ] { + for target_mode in [ + GatewayAuthenticationMode::Token, + GatewayAuthenticationMode::None, + ] { + let directory = tempfile::tempdir().unwrap(); + tokio::fs::write(directory.path().join("index.html"), "regional site") + .await + .unwrap(); + let destination_id = Uuid::now_v7(); + let deployment_id = Uuid::now_v7(); + let destination_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let destination_url = format!("http://{}", destination_listener.local_addr().unwrap()); + let route = ServeRoute { + host: "app.example.com".to_owned(), + region: "eu-west".to_owned(), + deployment_id, + target_node_id: destination_id, + target_base_url: destination_url.clone(), + gateway_authentication: target_mode, + resources: ServeResources { + cpu_millicores: 50, + memory_mb: 64, + disk_mb: 256, + }, + access: ServeAccess::Public, + }; + let make_state = |node_id, mode| { + let mut config = NodeConfig::default(); + config.security.gateway_authentication = mode; + Arc::new(ServeState::new( + ControlApiClient::new("http://127.0.0.1:9", "node-token").unwrap(), + node_id, + Some("shared-gateway-token".to_owned()), + Arc::new(routes::RouteTable::default()), + &config, + Arc::new(ssr::SsrManager::new(None, node_id, &config)), + )) + }; + let destination = make_state(destination_id, target_mode); + destination + .routes + .apply(RouteSnapshotResponse { + revision: "target".to_owned(), + routes: vec![route.clone()], + }) + .await + .unwrap(); + destination.targets.lock().await.insert( + deployment_id, + ResolvedTarget::Static { + static_dir: directory.path().to_owned(), + spa_fallback: false, + not_found: None, + }, + ); + let destination_server = tokio::spawn(async move { + axum::serve( + destination_listener, + serve_router(destination).into_make_service_with_connect_info::(), + ) + .await + .unwrap(); + }); + let source = make_state(Uuid::now_v7(), source_mode); + source + .routes + .apply(RouteSnapshotResponse { + revision: "source".to_owned(), + routes: vec![route], + }) + .await + .unwrap(); + let source_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let source_url = format!("http://{}", source_listener.local_addr().unwrap()); + let source_server = tokio::spawn(async move { + axum::serve( + source_listener, + serve_router(source).into_make_service_with_connect_info::(), + ) + .await + .unwrap(); + }); + let client = reqwest::Client::new(); + let response = client + .get(&source_url) + .header(header::HOST, "app.example.com") + .send() + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::OK, + "{source_mode:?} -> {target_mode:?}" + ); + assert_eq!(response.text().await.unwrap(), "regional site"); + let unbound = client + .get(&source_url) + .header(header::HOST, "unbound.example.com") + .send() + .await + .unwrap(); + assert_eq!(unbound.status(), StatusCode::NOT_FOUND); + let mut repeated = client + .get(format!("{source_url}{PEER_PROXY_PREFIX}/")) + .header(header::HOST, "app.example.com") + .header(GATEWAY_HOP_HEADER, "1"); + if source_mode == GatewayAuthenticationMode::Token { + repeated = repeated.header(GATEWAY_TOKEN_HEADER, "shared-gateway-token"); + } + assert_eq!( + repeated.send().await.unwrap().status(), + StatusCode::BAD_GATEWAY + ); + source_server.abort(); + destination_server.abort(); + } + } +} + +#[tokio::test] +async fn missing_outbound_token_fails_before_contacting_destination() { + let request = Request::builder() + .header(header::HOST, "app.example.com") + .body(Body::empty()) + .unwrap(); + let error = forward_to_gateway( + &reqwest::Client::new(), + "http://127.0.0.1:9", + "", + GatewayAuthenticationMode::Token, + "127.0.0.1:1234".parse().unwrap(), + request, + ) + .await + .unwrap_err(); + assert_eq!( + error.to_string(), + "destination gateway requires an outbound credential" + ); + let mut headers = HeaderMap::new(); + headers.insert(GATEWAY_TOKEN_HEADER, "".parse().unwrap()); + headers.insert(GATEWAY_HOP_HEADER, "1".parse().unwrap()); + assert!(gateway_origin(&headers, "", GatewayAuthenticationMode::Token).is_err()); +} + +#[tokio::test] +async fn gateway_redirects_and_traversal_never_send_credentials_to_other_endpoints() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let captured = Arc::new(AtomicUsize::new(0)); + let capture_router = axum::Router::new().fallback({ + let captured = captured.clone(); + move || { + let captured = captured.clone(); + async move { + captured.fetch_add(1, Ordering::SeqCst); + StatusCode::OK + } + } + }); + let capture_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let capture_url = format!("http://{}/capture", capture_listener.local_addr().unwrap()); + let capture_server = tokio::spawn(async move { + axum::serve(capture_listener, capture_router).await.unwrap(); + }); + + let peer_requests = Arc::new(AtomicUsize::new(0)); + let peer_router = axum::Router::new().fallback({ + let destination = capture_url.clone(); + let peer_requests = peer_requests.clone(); + move |request: Request| { + let destination = destination.clone(); + let peer_requests = peer_requests.clone(); + async move { + peer_requests.fetch_add(1, Ordering::SeqCst); + assert_eq!( + request.headers()[GATEWAY_TOKEN_HEADER], + "shared-gateway-token" + ); + assert_eq!(request.headers()[GATEWAY_HOP_HEADER], "1"); + assert_eq!(request.headers()[header::HOST], "app.example.com"); + assert!(request.uri().path().starts_with(PEER_PROXY_PREFIX)); + (StatusCode::FOUND, [(header::LOCATION, destination)]) + } + } + }); + let peer_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let peer_url = format!("http://{}", peer_listener.local_addr().unwrap()); + let peer_server = tokio::spawn(async move { + axum::serve(peer_listener, peer_router).await.unwrap(); + }); + + let mut config = NodeConfig::default(); + config.security.gateway_authentication = GatewayAuthenticationMode::None; + let node_id = Uuid::now_v7(); + let routes = Arc::new(routes::RouteTable::default()); + routes + .apply(grass_node_protocol::RouteSnapshotResponse { + revision: "security".to_owned(), + routes: vec![ServeRoute { + host: "app.example.com".to_owned(), + region: "default".to_owned(), + deployment_id: Uuid::now_v7(), + target_node_id: Uuid::now_v7(), + target_base_url: peer_url.clone(), + gateway_authentication: GatewayAuthenticationMode::Token, + resources: grass_node_protocol::ServeResources { + cpu_millicores: 50, + memory_mb: 64, + disk_mb: 256, + }, + access: ServeAccess::Public, + }], + }) + .await + .unwrap(); + let state = Arc::new(ServeState::new( + ControlApiClient::new("http://127.0.0.1:9", "node-token").unwrap(), + node_id, + Some("shared-gateway-token".to_owned()), + routes, + &config, + Arc::new(ssr::SsrManager::new(None, node_id, &config)), + )); + let source_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let source_address = source_listener.local_addr().unwrap(); + let router = serve_router(state.clone()); + let source_server = tokio::spawn(async move { + axum::serve( + source_listener, + router.into_make_service_with_connect_info::(), + ) + .await + .unwrap(); + }); + let browser = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(); + let response = browser + .get(format!("http://{source_address}/redirect")) + .header(header::HOST, "app.example.com") + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::FOUND); + assert_eq!(response.headers()[header::LOCATION], capture_url); + assert_eq!(peer_requests.load(Ordering::SeqCst), 1); + assert_eq!(captured.load(Ordering::SeqCst), 0); + + for path in [ + "/../../../_grass/internal/routes/invalidate", + "/%2e%2e/%2e%2e/%2e%2e/_grass/internal/routes/invalidate", + "/.%2E/.%2E/.%2E/_grass/internal/routes/invalidate", + "/%5c../%5c../_grass/internal/routes/invalidate", + ] { + // Send raw HTTP to preserve the malicious path instead of the + // test HTTP client's own URL parser normalizing it in advance. + let mut stream = tokio::net::TcpStream::connect(source_address) + .await + .unwrap(); + stream.write_all(format!("POST {path} HTTP/1.1\r\nHost: app.example.com\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").as_bytes()).await.unwrap(); + let mut response = Vec::new(); + stream.read_to_end(&mut response).await.unwrap(); + assert!( + response.starts_with(b"HTTP/1.1 400"), + "{path}: {}", + String::from_utf8_lossy(&response) + ); + let request = Request::builder() + .uri(path) + .header(header::HOST, "app.example.com") + .body(Body::empty()) + .unwrap(); + let error = forward_to_gateway( + &state.proxy, + &peer_url, + "shared-gateway-token", + GatewayAuthenticationMode::Token, + source_address, + request, + ) + .await + .unwrap_err(); + assert_eq!(error.to_string(), "invalid gateway request path"); + } + assert_eq!(peer_requests.load(Ordering::SeqCst), 1); + assert_eq!(captured.load(Ordering::SeqCst), 0); + source_server.abort(); + peer_server.abort(); + capture_server.abort(); +} diff --git a/crates/assets/build.rs b/crates/assets/build.rs index e1ed635..04f8ff5 100644 --- a/crates/assets/build.rs +++ b/crates/assets/build.rs @@ -1,29 +1,53 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; fn main() { - let profile = std::env::var("PROFILE").unwrap_or_default(); - let is_release = profile == "release"; + let manifest = PathBuf::from( + std::env::var_os("CARGO_MANIFEST_DIR").expect("Cargo provides CARGO_MANIFEST_DIR"), + ); + let dist = manifest.join("../../apps/console/dist"); + let output = PathBuf::from(std::env::var_os("OUT_DIR").expect("Cargo provides OUT_DIR")); + let public = output.join("public"); - let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")); - let dist = workspace_root.join("../../apps/console/dist"); - let public = workspace_root.join("assets/public"); + println!("cargo:rerun-if-changed=build.rs"); + // Watching the directory includes additions, removals and nested assets. + println!("cargo:rerun-if-changed={}", dist.display()); - let _ = std::fs::remove_dir_all(&public); - std::fs::create_dir_all(&public).unwrap(); + let is_release = std::env::var("PROFILE").is_ok_and(|profile| profile == "release"); + if is_release { + let index = dist.join("index.html"); + assert!( + index + .metadata() + .is_ok_and(|meta| meta.is_file() && meta.len() > 0), + "Console release assets are missing: run `just build console` or `just release` first" + ); + } + + if public.exists() { + std::fs::remove_dir_all(&public).expect("old generated assets can be removed"); + } + std::fs::create_dir_all(&public).expect("generated asset directory can be created"); - if is_release && dist.is_dir() { - copy_dir(&dist, &public).unwrap(); + if is_release { + copy_dir(&dist, &public).expect("Console release assets can be copied"); } else { std::fs::write( public.join("index.html"), - if is_release { - "

Console not built. Run just build console first.

" - } else { - "

Frontend served by Vite dev server in debug mode.

" - }, + "

Frontend served by Vite dev server in debug mode.

", ) - .unwrap(); + .expect("development placeholder can be written"); } + + // Generate the literal path without adding path-interpolation dependencies. + // OUT_DIR belongs to this Cargo target/profile and never modifies crate sources. + std::fs::write( + output.join("embedded.rs"), + format!( + "#[derive(rust_embed::RustEmbed)]\n#[folder = {:?}]\npub struct ConsoleAssets;\n", + public + ), + ) + .expect("embedded asset declaration can be written"); } fn copy_dir(src: &Path, dst: &Path) -> std::io::Result<()> { diff --git a/crates/assets/src/lib.rs b/crates/assets/src/lib.rs index c02c9f5..a131a2b 100644 --- a/crates/assets/src/lib.rs +++ b/crates/assets/src/lib.rs @@ -1,13 +1,9 @@ //! grass-assets — embedded Console build assets. //! -//! Embeds `public/` at compile time via `rust-embed`. -//! The build pipeline copies `apps/console/dist/` here before compilation. +//! Release builds embed `apps/console/dist/` via a profile-local `OUT_DIR` copy. +//! Development builds use a placeholder and a separate Vite development server. -use rust_embed::RustEmbed; - -#[derive(RustEmbed)] -#[folder = "assets/public/"] -pub struct ConsoleAssets; +include!(concat!(env!("OUT_DIR"), "/embedded.rs")); pub fn get(path: &str) -> Option { ConsoleAssets::get(path) diff --git a/crates/cache/src/redis_backend.rs b/crates/cache/src/redis_backend.rs index 22d6b8b..9bf34b7 100644 --- a/crates/cache/src/redis_backend.rs +++ b/crates/cache/src/redis_backend.rs @@ -274,9 +274,10 @@ mod tests { use super::super::Cache; use super::*; - async fn test_cache() -> Option { - let url = std::env::var("GRASS_TEST_REDIS_URL").ok()?; - Some(RedisCache::connect(&url).await.unwrap()) + async fn test_cache() -> RedisCache { + let url = std::env::var("GRASS_TEST_REDIS_URL") + .expect("GRASS_TEST_REDIS_URL is required for this ignored test"); + RedisCache::connect(&url).await.unwrap() } fn unique_key(suffix: &str) -> String { @@ -288,10 +289,9 @@ mod tests { } #[tokio::test] + #[ignore = "requires GRASS_TEST_REDIS_URL pointing to a disposable Redis service"] async fn conditional_update_does_not_recreate_deleted_value() { - let Some(cache) = test_cache().await else { - return; - }; + let cache = test_cache().await; let key = unique_key("conditional"); cache .set(&key, "old", Duration::from_secs(60)) @@ -309,10 +309,9 @@ mod tests { } #[tokio::test] + #[ignore = "requires GRASS_TEST_REDIS_URL pointing to a disposable Redis service"] async fn take_returns_a_value_to_only_one_concurrent_caller() { - let Some(cache) = test_cache().await else { - return; - }; + let cache = test_cache().await; let key = unique_key("take"); cache .set(&key, "value", Duration::from_secs(60)) @@ -334,10 +333,9 @@ mod tests { } #[tokio::test] + #[ignore = "requires GRASS_TEST_REDIS_URL pointing to a disposable Redis service"] async fn token_bucket_is_atomic_in_redis() { - let Some(cache) = test_cache().await else { - return; - }; + let cache = test_cache().await; let key = unique_key("rate-limit"); assert!( diff --git a/docs/agents/control-api-conventions.md b/docs/agents/control-api-conventions.md index 9f91357..d55cf36 100644 --- a/docs/agents/control-api-conventions.md +++ b/docs/agents/control-api-conventions.md @@ -46,6 +46,6 @@ Manually expand dense SQL, JSON, and mock closures where rustfmt cannot express ## Validation and tracking -Protect route paths/methods, authentication middleware, response contracts, lifecycle transactions, audit isolation/redaction, and preview grants with relevant tests. PostgreSQL suites that create disposable schemas are explicitly ignored in ordinary runs. Running them requires `GRASS_TEST_DATABASE_URL` and authorization for disposable schema creation and cleanup; Redis cases additionally require `GRASS_TEST_REDIS_URL`. They fail when explicitly selected without that configuration. CI explicitly runs the delivery and Node deletion PostgreSQL suites against its disposable database, preserving their coverage after the move to ignored tests. Do not count ignored or unconfigured database cases as executed regressions. +Protect route paths/methods, authentication middleware, response contracts, lifecycle transactions, audit isolation/redaction, and preview grants with relevant tests. PostgreSQL suites that create disposable schemas are explicitly ignored in ordinary runs. Running them requires `GRASS_TEST_DATABASE_URL` and authorization for disposable schema creation and cleanup; Redis cases additionally require `GRASS_TEST_REDIS_URL`. They fail when explicitly selected without that configuration. CI runs the complete PostgreSQL and Redis suite through `just test-services` against disposable services, including delivery, Node deletion, migration shape/rollback, authentication revocation and cache regressions. The Chromium screenshot case is excluded from this service suite. Do not count ignored or unconfigured database cases as executed regressions. Use the repository Just commands and Vite+ for Console checks. Each commit requires a successful `just quality`. Follow the issue, worktree, review, PR, merge, and TODO cleanup gates in `AGENTS.md`. diff --git a/docs/release-quality.md b/docs/release-quality.md new file mode 100644 index 0000000..bc3da96 --- /dev/null +++ b/docs/release-quality.md @@ -0,0 +1,52 @@ +# Release quality checks + +## Console types + +Use `just check console` (or `vp check` in `apps/console`) for formatting, linting and type checking. Vite+ enables its type-aware checker through `lint.options.typeAware` and `lint.options.typeCheck`. Type errors fail this command and the shared repository quality gate. + +`vp exec tsc --noEmit` is the independent TypeScript compiler check. The project retains strict checking and includes Vite client, Node and Vite+ test global types. React and React DOM declarations are explicit development dependencies. + +The type-checking change repairs incomplete test fixtures, unsupported component variants, concrete API filter shapes and mismatched announcement timestamps and mutation return types. Both checkers were verified with a temporary incompatible assignment, which failed with TS2322; the probe was removed afterwards. + +Enabling the type-aware lint engine also surfaces advisory warnings in existing frontend code. They remain visible and are not suppressed; frontend architecture and advisory lint cleanup are deferred from this Rust cleanup round. Type errors are blocking. + +## Minimum supported Rust version + +The workspace declares Rust 1.88. `just msrv` uses cargo-msrv to verify every workspace target against that declaration, including test targets, with the committed lockfile and default application features. Install cargo-msrv 0.18.4 with `cargo install cargo-msrv --version 0.18.4 --locked` when it is unavailable. + +To find the minimum again after dependency or language changes: + +```sh +cargo msrv find --manifest-path apps/control-api/Cargo.toml --min 1.85 --no-log -- cargo check --workspace --all-targets --locked +just msrv +``` + +The lower search bound corresponds to the workspace's Rust 2024 edition. Preserve the lockfile during the search. An unavailable toolchain or network failure is a validation failure, not evidence that an older compiler is incompatible. + +On macOS ARM64, cargo-msrv tested Rust 1.90.0 successfully, rejected 1.87.0 because locked SeaQuery/time dependencies require 1.88, and passed 1.88.0. CI verifies the declared version on Linux x86_64 and macOS ARM64. Docker already uses Rust 1.88. Updating the minimum requires keeping the workspace declaration, Docker builder and self-hosting documentation aligned. + +## Embedded Console assets + +`just build` builds development binaries; run the Console separately with `just run console`. `just release` builds the Console and then produces distributable binaries under `target/release`. Direct `cargo build --release` requires a nonempty `apps/console/dist/index.html`, produced by `just build console` first. + +The asset build script watches the complete dist directory and copies resources into its profile/target-specific `OUT_DIR`. It does not generate files in the source tree. Missing or empty release HTML fails the build. Debug builds always use the development-server placeholder. + +`just assets-check` exercises the actual asset crate in a disposable fixture workspace: changed HTML, added and removed resources, alternating debug/release profiles, missing/empty dist HTML and a missing dist directory. It is included in `just quality` and CI. Its dependency/build cache lives under the selected Cargo target directory. + +## PostgreSQL and Redis regressions + +`just test rust` runs the fast suite and reports environment-dependent tests as ignored. `just test-services` requires `GRASS_TEST_DATABASE_URL` and `GRASS_TEST_REDIS_URL` for disposable test services. It exits before running any command when either variable is absent. Do not point it at production databases or shared infrastructure: the PostgreSQL tests create and remove their own test schemas. + +The service suite applies current migrations using a temporary runtime configuration, runs every ignored Control API test except the Chromium screenshot case, and runs all ignored Redis cache tests. This currently covers 30 PostgreSQL-related cases, one standalone Redis session authorization case and three Redis cache cases. It includes authentication-version shape/revocation, region backfill, domain onboarding, lifecycle transactions, upgrade/rollback and native schema assertions. + +CI provides disposable PostgreSQL 17 and Redis 7 services in a dedicated job. The Node Docker smoke test and Chromium screenshot test retain their separate runtime requirements; Chromium is not counted as an executed database regression. + +## Publication gate and prereleases + +Pull requests run Quality: formatting, lint/type checks, tests, dependency audits, service regressions and both MSRV platforms. The Node delivery smoke builds a test image to exercise real container execution; PRs do not build the three distributable runtime images. Ordinary feature branch pushes do not trigger a second Quality run. + +Main, develop and version tag pushes enter Release, which calls the reusable Quality workflow from the same commit. Both image publication and binary release uploads depend on successful validation of all Quality jobs. Release then builds and loads the Debian, Slim and Alpine runtime variants and checks both packaged binaries in every variant. Only after all six binary checks pass does it publish the images, reusing the verified build cache with the same inputs and preserving BuildKit publication metadata. + +Release tags use `vMAJOR.MINOR.PATCH` or `vMAJOR.MINOR.PATCH-PRERELEASE`. A stable `v0.1.0` publishes image aliases `0.1.0`, `0.1` and `latest`. A prerelease such as `v0.1.0-rc.1` publishes only its exact version alias and is explicitly marked as a GitHub prerelease without becoming the latest release. All image variants retain their SHA tag; main/develop retain their branch alias. Slim and Alpine apply their suffix to every alias. Build metadata (`+...`) is rejected because it is not valid in Docker tags. + +`just release-check` verifies stable, prerelease, branch and rejected-ref policies without creating tags or publishing artifacts. It is included in `just quality` and CI. diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 5bb21ee..aa3e2ec 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -12,7 +12,7 @@ logs, review, activation, and public access. - Git 2.49+ plus OpenSSH client tools (`ssh` and `ssh-keyscan`) available to the Node process - [Just](https://just.systems/) 1.56+ for repository task commands -- Rust 1.85+ and [Vite+](https://viteplus.dev/) when building from source +- Rust 1.88+ and [Vite+](https://viteplus.dev/) when building from source PostgreSQL and Redis belong to the Control API. Nodes do not connect to either service directly; they use the authenticated Control API instead. @@ -21,7 +21,7 @@ either service directly; they use the authenticated Control API instead. ```sh just install console -just build # builds the Console, embeds it, and builds both binaries +just release # builds both release binaries; Control API embeds the Console ``` Or use the Docker image (both binaries are included): diff --git a/docs/todo.md b/docs/todo.md index 74970f8..7820bd0 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -9,6 +9,20 @@ - 未经用户批准，不得把较低优先级或 Future 项目提前并入当前功能。 - 当前版本从本文档头部读取，并用于 GitHub Milestone 与 Project 命名。 +# 发布质量与 Rust 整理 + +父事项：[#219](https://github.com/Grass-Development-Team/grass-worker/issues/219)。当前版本按已批准范围实施；A 部分的 Console 架构和错误反馈整理留待后续。 + +- Q01 补齐 Console 类型环境、修复诊断并启用统一类型门禁。[#220](https://github.com/Grass-Development-Team/grass-worker/issues/220) +- Q02 使用 cargo-msrv 实测并验证 workspace 最低 Rust 版本，统一声明、Docker、文档和 CI。[#221](https://github.com/Grass-Development-Team/grass-worker/issues/221) +- Q03 修复 Console 嵌入资源追踪、profile 隔离和 release 构建入口。[#222](https://github.com/Grass-Development-Team/grass-worker/issues/222) +- Q04 完整接入 PostgreSQL/Redis 回归，明确快速测试与集成测试边界。[#223](https://github.com/Grass-Development-Team/grass-worker/issues/223) +- Q05 发布依赖同一提交的质量验证，并明确正式版与预发布元数据。[#224](https://github.com/Grass-Development-Team/grass-worker/issues/224) +- A02-node 按职责拆分 Node serve，保持 Host、预览、Peer Hop、静态与 SSR 行为。[#225](https://github.com/Grass-Development-Team/grass-worker/issues/225) +- A02-storage 拆分存储配置、后端适配、写入协调和流校验。[#226](https://github.com/Grass-Development-Team/grass-worker/issues/226) +- A02-tests 整理迁移测试模块与共享 fixture，保持回归清单和 CI 选择正确。[#227](https://github.com/Grass-Development-Team/grass-worker/issues/227) +- A03 统一剩余 Rust 导入、声明和复杂 SQL/JSON/mock 排版。[#228](https://github.com/Grass-Development-Team/grass-worker/issues/228) + # 第二阶段 ## P2：平台扩展 diff --git a/scripts/check-embedded-assets.py b/scripts/check-embedded-assets.py new file mode 100644 index 0000000..03a25db --- /dev/null +++ b/scripts/check-embedded-assets.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Exercise the real asset crate across incremental Cargo builds and profiles.""" + +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + with tempfile.TemporaryDirectory(prefix="grass-assets-check-") as temporary: + fixture = Path(temporary) + assets = fixture / "crates/assets" + (assets / "src").mkdir(parents=True) + for name in ("Cargo.toml", "build.rs", "src/lib.rs"): + shutil.copy2(ROOT / "crates/assets" / name, assets / name) + manifest = re.sub( + r"members = \[.*?\n\]", + 'members = ["crates/assets", "probe"]', + (ROOT / "Cargo.toml").read_text(), + count=1, + flags=re.S, + ) + (fixture / "Cargo.toml").write_text(manifest) + shutil.copy2(ROOT / "Cargo.lock", fixture / "Cargo.lock") + probe = fixture / "probe" + (probe / "src").mkdir(parents=True) + (probe / "Cargo.toml").write_text( + '[package]\nname = "embedded-assets-probe"\nversion = "0.0.0"\n' + 'edition.workspace = true\n[dependencies]\ngrass-assets.workspace = true\n' + ) + (probe / "src/main.rs").write_text( + 'fn main() {\n' + ' let key = std::env::args().nth(1).unwrap();\n' + ' match grass_assets::get(&key) {\n' + ' Some(file) => print!("{}", std::str::from_utf8(&file.data).unwrap()),\n' + ' None => print!(""),\n' + ' }\n}\n' + ) + dist = fixture / "apps/console/dist" + dist.mkdir(parents=True) + (dist / "index.html").write_text("console-build-A") + (dist / "old.txt").write_text("old-asset") + environment = os.environ.copy() + target = Path(environment.get("CARGO_TARGET_DIR", ROOT / "target")) + environment["CARGO_TARGET_DIR"] = str(target.resolve() / "assets-regression") + initialized = False + + def run(release=True, key="index.html", failure=False): + nonlocal initialized + command = ["cargo", "run", "--quiet", "-p", "embedded-assets-probe"] + if initialized: + command.append("--locked") + if release: + command.append("--release") + result = subprocess.run( + [*command, "--", key], cwd=fixture, env=environment, + capture_output=True, text=True, + ) + if failure: + assert result.returncode != 0, "release accepted missing Console output" + assert "Console release assets are missing" in result.stderr, result.stderr + elif result.returncode != 0: + raise RuntimeError(result.stderr) + initialized = True + return result.stdout + + assert run() == "console-build-A" + (dist / "index.html").write_text("console-build-B") + (dist / "new.txt").write_text("new-asset") + (dist / "old.txt").unlink() + assert run() == "console-build-B", "incremental release retained stale HTML" + assert run(key="new.txt") == "new-asset", "new asset was not embedded" + assert run(key="old.txt") == "", "deleted asset remained embedded" + assert "Vite dev server" in run(release=False) + assert run() == "console-build-B", "debug output contaminated release output" + (dist / "index.html").unlink() + run(failure=True) + (dist / "index.html").write_text("") + run(failure=True) + dist.rename(dist.with_name("hidden-dist")) + run(failure=True) + assert "Vite dev server" in run(release=False) + assert not (assets / "assets").exists(), "build wrote generated files into crate sources" + print("Embedded assets: content/addition/removal/profile/missing-output checks passed") + + +if __name__ == "__main__": + main() diff --git a/scripts/release-metadata.py b/scripts/release-metadata.py new file mode 100644 index 0000000..a6f556b --- /dev/null +++ b/scripts/release-metadata.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Select explicit Docker aliases and GitHub prerelease status for a release ref.""" + +import os +from pathlib import Path +import re + +VERSION = re.compile( + r"v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)" + r"(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?" +) + + +def release_policy(ref): + if ref in ("refs/heads/main", "refs/heads/develop"): + return [ref.removeprefix("refs/heads/")], False + if not ref.startswith("refs/tags/"): + raise ValueError("release ref must be main, develop or a version tag") + tag = ref.removeprefix("refs/tags/") + match = VERSION.fullmatch(tag) + if not match: + raise ValueError("release tags must be vMAJOR.MINOR.PATCH[-PRERELEASE]") + major, minor, _, prerelease = match.groups() + if prerelease and any( + part.isdigit() and len(part) > 1 and part.startswith("0") + for part in prerelease.split(".") + ): + raise ValueError("numeric prerelease identifiers cannot have leading zeroes") + aliases = [tag[1:]] + if not prerelease: + aliases.extend([f"{major}.{minor}", "latest"]) + return aliases, bool(prerelease) + + +def main(): + aliases, prerelease = release_policy(os.environ["GITHUB_REF"]) + # Docker metadata-action adds the registry, variant suffixes, SHA and OCI labels. + # Its automatic latest alias is disabled; only this policy may select aliases. + docker_tags = "\n".join([*(f"type=raw,value={tag}" for tag in aliases), "type=sha"]) + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write(f"prerelease={str(prerelease).lower()}\n") + output.write(f"docker_tags<