diff --git a/.github/scripts/release-tag-guard.sh b/.github/scripts/release-tag-guard.sh new file mode 100755 index 0000000000..0e14bfdecc --- /dev/null +++ b/.github/scripts/release-tag-guard.sh @@ -0,0 +1,66 @@ +#!/bin/sh +set -eu + +: "${VERSION:?VERSION is required}" +: "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" + +if ! printf '%s\n' "$VERSION" | grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'; then + echo "release tag must match vMAJOR.MINOR.PATCH" >&2 + exit 1 +fi + +tag_ref="refs/tags/$VERSION" +if ! git show-ref --verify --quiet "$tag_ref"; then + echo "release tag does not exist: $VERSION" >&2 + exit 1 +fi +commit=$(git rev-parse --verify "$tag_ref^{commit}") +if ! printf '%s\n' "$commit" | grep -Eq '^[0-9a-f]{40}$'; then + echo "release tag did not peel to a commit" >&2 + exit 1 +fi + +case "${GITHUB_EVENT_NAME:-}" in + push) + if [ "${GITHUB_REF:-}" != "$tag_ref" ]; then + echo "push ref does not match the release tag" >&2 + exit 1 + fi + if ! printf '%s\n' "${GITHUB_SHA:-}" | grep -Eq '^[0-9a-f]{40}$'; then + echo "push commit is not a full SHA" >&2 + exit 1 + fi + event_commit=$(git rev-parse --verify "${GITHUB_SHA}^{commit}" 2>/dev/null) || { + echo "push commit did not peel to a commit" >&2 + exit 1 + } + if [ "$event_commit" != "$commit" ]; then + echo "push commit does not match the release tag" >&2 + exit 1 + fi + ;; + workflow_dispatch) + # A dispatch runs from the selected workflow branch, not from the tag. + ;; + *) + echo "unsupported release event: ${GITHUB_EVENT_NAME:-unset}" >&2 + exit 1 + ;; +esac +if ! git merge-base --is-ancestor "$commit" origin/main; then + echo "release tag commit is not on origin/main" >&2 + exit 1 +fi + +git checkout --detach "$commit" >/dev/null 2>&1 +head=$(git rev-parse --verify HEAD) +if [ "$head" != "$commit" ]; then + echo "checked-out commit does not match the verified release tag" >&2 + exit 1 +fi + +{ + echo "version=$VERSION" + echo "commit=$commit" +} >>"$GITHUB_OUTPUT" +printf '%s is verified at %s\n' "$VERSION" "$commit" diff --git a/.github/scripts/release-tag-guard_test.sh b/.github/scripts/release-tag-guard_test.sh new file mode 100755 index 0000000000..af73be63a4 --- /dev/null +++ b/.github/scripts/release-tag-guard_test.sh @@ -0,0 +1,83 @@ +#!/bin/sh +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +script="$script_dir/release-tag-guard.sh" +work=${TEST_TMPDIR:-.scratch}/release-tag-guard-$$ +mkdir -p "$work" +work=$(CDPATH= cd -- "$work" && pwd -P) + +repo="$work/repo" +mkdir "$repo" +repo=$(CDPATH= cd -- "$repo" && pwd -P) +git -C "$repo" init -q -b main +git -C "$repo" config user.name test +git -C "$repo" config user.email test@example.com +git -C "$repo" config commit.gpgSign false +git -C "$repo" config tag.gpgSign false +printf 'release\n' >"$repo/file" +git -C "$repo" add file +git -C "$repo" commit -qm release +release_commit=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" tag v1.2.3 +printf 'main\n' >>"$repo/file" +git -C "$repo" commit -qam main +main=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" remote add origin "$repo" +git -C "$repo" fetch -q origin main:refs/remotes/origin/main + +run_ok() { + output="$work/output" + : >"$output" + (cd "$repo" && VERSION="$1" GITHUB_EVENT_NAME="$2" GITHUB_REF="$3" GITHUB_SHA="$4" GITHUB_OUTPUT="$output" "$script") >/dev/null + grep -Fx "version=$1" "$output" >/dev/null + grep -Fx "commit=$5" "$output" >/dev/null +} +run_fail() { + output="$work/output" + : >"$output" + if (cd "$repo" && VERSION="$1" GITHUB_EVENT_NAME="$2" GITHUB_REF="$3" GITHUB_SHA="$4" GITHUB_OUTPUT="$output" "$script") >/dev/null 2>&1; then + echo "unexpectedly accepted: $1" >&2 + exit 1 + fi +} + +# Dispatch starts at main HEAD, but must publish the selected older tag commit. +run_ok v1.2.3 workflow_dispatch refs/heads/main "$main" "$release_commit" +run_fail main workflow_dispatch refs/heads/main "$main" +run_fail refs/heads/main workflow_dispatch refs/heads/main "$main" +marker="$work/pwned" +run_fail "v1.2.3; touch $marker" workflow_dispatch refs/heads/main "$main" +[ ! -e "$marker" ] +run_fail v01.2.3 workflow_dispatch refs/heads/main "$main" +run_fail v9.9.9 workflow_dispatch refs/heads/main "$main" +run_fail v1.2.3 push refs/heads/main "$release_commit" +run_fail v1.2.3 push refs/tags/v1.2.3 0000000000000000000000000000000000000000 +run_ok v1.2.3 push refs/tags/v1.2.3 "$release_commit" "$release_commit" + +# An annotated tag push reports its tag-object SHA; both sides must peel it. +git -C "$repo" tag -a v1.2.4 -m annotated "$release_commit" +annotated_tag=$(git -C "$repo" rev-parse v1.2.4) +run_ok v1.2.4 push refs/tags/v1.2.4 "$annotated_tag" "$release_commit" + +git -C "$repo" switch -q --detach "$release_commit" +printf 'branch\n' >>"$repo/file" +git -C "$repo" commit -qam branch +off_main=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" tag v2.0.0 +run_fail v2.0.0 workflow_dispatch refs/heads/main "$main" +run_fail v2.0.0 push refs/tags/v2.0.0 "$off_main" +run_fail v1.2.3 schedule refs/heads/main "$main" + +workflow="$script_dir/../workflows/release.yml" +# The privileged graph consumes only the commit emitted by a guard implementation +# checked out from protected main; event-selected refs remain untrusted data. +grep -F "if: github.event_name == 'push' || github.ref == 'refs/heads/main'" "$workflow" >/dev/null +grep -F 'ref: refs/heads/main' "$workflow" >/dev/null +if grep -F 'org.opencontainers.image.revision=${{ github.sha }}' "$workflow" >/dev/null; then + echo "release image revision still uses the event workflow SHA" >&2 + exit 1 +fi +[ "$(grep -Fc 'org.opencontainers.image.revision=${{ needs.guard.outputs.commit }}' "$workflow")" -eq 5 ] + +printf 'release tag guard tests passed\n' diff --git a/.github/workflows/e2e-live.yml b/.github/workflows/e2e-live.yml index 9d1a019bfb..ebcb67bd21 100644 --- a/.github/workflows/e2e-live.yml +++ b/.github/workflows/e2e-live.yml @@ -18,9 +18,9 @@ # even when labelled; see ci.yml's header for the pwn-request rationale. # - The job `if:` also requires github.repository == 'stacklok/mecatl', so a # fork's own scheduled/dispatched runs never reference the secret. -# - A guard step checks OPENROUTER_API_KEY availability and skips the live -# run cleanly (notice, green) when it is absent — the workflow never -# hard-fails just because the secret isn't configured. +# - Ordinary live jobs skip cleanly when OPENROUTER_API_KEY is absent. The +# manual-only native qualification instead fails closed on a missing key. +# It requires a successful same-run production qualification before staging. # # All third-party actions are SHA-pinned (with a # vX.Y.Z comment) so a # re-pointed tag from a compromised maintainer cannot change what runs here. @@ -31,6 +31,15 @@ on: # Nightly, off-peak UTC. Odd minute to avoid the top-of-hour thundering herd. - cron: '17 3 * * *' workflow_dispatch: + inputs: + native_execution: + description: Run only native execution production + OpenRouter qualification + type: boolean + default: false + expected_sha: + description: Optional exact reviewed commit SHA (native execution only) + type: string + default: '' pull_request: types: [opened, synchronize, reopened, labeled] @@ -58,6 +67,7 @@ jobs: # Gate 2: PRs run only when a maintainer opted in via the `e2e-live` label. if: >- github.repository == 'stacklok/mecatl' && + !(github.event_name == 'workflow_dispatch' && inputs.native_execution) && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'e2e-live')) # One live run at a time, globally: overlapping dispatches/labelled pushes @@ -177,6 +187,7 @@ jobs: # and PRs run only when a maintainer opted in via the `e2e-live` label. if: >- github.repository == 'stacklok/mecatl' && + !(github.event_name == 'workflow_dispatch' && inputs.native_execution) && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'e2e-live')) # Separate group from `e2e-live` so the two live jobs run in parallel; they @@ -258,3 +269,206 @@ jobs: path: k8s-e2e-live-output.log retention-days: 7 if-no-files-found: warn + + native-execution-live: + name: Native execution (production + OpenRouter) + if: >- + github.repository == 'stacklok/mecatl' && + github.event_name == 'workflow_dispatch' && inputs.native_execution + runs-on: ubuntu-24.04 + timeout-minutes: 100 + concurrency: + group: e2e-live-native-execution + cancel-in-progress: false + env: + NATIVE_CI_DIR: ${{ github.workspace }}/.scratch/native-live-${{ github.run_id }}-${{ github.run_attempt }} + steps: + # Step ceilings sum to 98m: setup 9, production 50, credential 1, + # live 25, cleanup 10, reporting 3. The job retains 2m overhead. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Verify reviewed commit and reserve private CI directory + timeout-minutes: 1 + id: prepare + env: + EXPECTED_SHA: ${{ inputs.expected_sha }} + EVENT_SHA: ${{ github.sha }} + run: | + actual=$(git rev-parse HEAD) + test "$actual" = "$EVENT_SHA" + if [ -n "$EXPECTED_SHA" ] && [ "$actual" != "$EXPECTED_SHA" ]; then + echo '::error::Reviewed SHA does not match dispatch SHA' + exit 1 + fi + printf '## Native execution qualification\n\nCommit: %s\n' "$actual" >> "$GITHUB_STEP_SUMMARY" + umask 077 + mkdir -p .scratch + mkdir -m 700 "$NATIVE_CI_DIR" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + timeout-minutes: 2 + with: + go-version-file: go.mod + cache: false + - uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 + timeout-minutes: 1 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + timeout-minutes: 1 + with: + version: v3.18.4 + - name: Install pinned Kind and ko + timeout-minutes: 3 + run: | + go install sigs.k8s.io/kind@v0.33.0 + go install github.com/google/ko@v0.18.1 + command -v kubectl jq docker + # No provider secret here. run.sh installs pinned Calico and retains the + # owned cluster; the success dependency, not ownership.profile, admits live. + - name: Qualify production execution + timeout-minutes: 50 + id: production + env: + CONTAINER_ENGINE: docker + MECATL_EXECUTION_QUAL_CI: "0" + run: MECATL_EXECUTION_QUAL_OUTPUT="$GITHUB_OUTPUT" timeout --kill-after=15s 49m task e2e:k8s:execution:production + - name: Stage private provider credential + timeout-minutes: 1 + id: credential + if: success() && steps.production.outcome == 'success' + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: | + set +x + if [ -z "$OPENROUTER_API_KEY" ]; then + echo '::error::OPENROUTER_API_KEY is required for native qualification' + exit 1 + fi + umask 077 + set -C + printf '%s' "$OPENROUTER_API_KEY" > "$NATIVE_CI_DIR/provider-key" + unset OPENROUTER_API_KEY + - name: Qualify native OpenRouter coding + timeout-minutes: 25 + id: live + if: success() && steps.production.outcome == 'success' && steps.credential.outcome == 'success' + env: + MECATL_EXECUTION_QUAL_STATE: ${{ steps.production.outputs.state }} + NATIVE_CLUSTER: ${{ steps.production.outputs.cluster }} + run: | + test -n "$NATIVE_CLUSTER" + grep -Fx -- "cluster=$NATIVE_CLUSTER" "$MECATL_EXECUTION_QUAL_STATE/ownership" >/dev/null + MECATL_EXECUTION_CREDENTIAL_FILE="$NATIVE_CI_DIR/provider-key" \ + timeout --kill-after=15s 24m ./deploy/mecatl-execution-kind/live.sh + # Credential removal is independent of cluster availability. Fail closed + # on ownership drift; runner disposal is a last resort, not cleanup proof. + - name: Clean up exact owned resources + timeout-minutes: 10 + id: cleanup + if: always() && steps.prepare.outcome == 'success' + env: + PRODUCTION_OUTCOME: ${{ steps.production.outcome }} + LIVE_OUTCOME: ${{ steps.live.outcome }} + MECATL_EXECUTION_QUAL_STATE: ${{ steps.production.outputs.state }} + NATIVE_CLUSTER: ${{ steps.production.outputs.cluster }} + run: | + key="$NATIVE_CI_DIR/provider-key" + if [ -e "$key" ] || [ -L "$key" ]; then + test -f "$key" && test ! -L "$key" + rm -- "$key" + fi + if [ "$PRODUCTION_OUTCOME" = skipped ]; then exit 0; fi + trap 'echo "::error::Owned cleanup incomplete; hosted-runner disposal is the fallback, not cleanup proof"' ERR + state="$MECATL_EXECUTION_QUAL_STATE" + test -n "$state" && test -n "$NATIVE_CLUSTER" + test ! -L "$state" + state=$(cd -- "$state" && pwd -P) + case "$state" in "$GITHUB_WORKSPACE/.scratch/k8s-execution/"*) ;; *) exit 1 ;; esac + ownership="$state/ownership" + test -f "$ownership" && test ! -L "$ownership" + owned_value() { + awk -F= -v key="$1" '$1 == key {value=substr($0,length(key)+2); count++} END {if(count != 1 || value == "") exit 1; print value}' "$ownership" + } + cluster=$(owned_value cluster) + test "$cluster" = "$NATIVE_CLUSTER" + case "$cluster" in mecatl-execution-qual-*) ;; *) exit 1 ;; esac + case "$cluster" in *[!a-z0-9-]*) exit 1 ;; esac + test "${#cluster}" -le 63 + test "$(owned_value owner)" = "$USER" + test "$(owned_value runtime)" = docker + test "$(owned_value profile)" = production + test "$(owned_value namespace)" = execution-qualification + context="kind-$cluster" + test "$(owned_value context)" = "$context" + kubeconfig="$state/kubeconfig" + test "$(owned_value kubeconfig)" = "$kubeconfig" + test -f "$kubeconfig" && test ! -L "$kubeconfig" + test "$(stat -c '%a' "$kubeconfig")" = 600 + test "$(kubectl --kubeconfig "$kubeconfig" --context "$context" config current-context)" = "$context" + test "$(docker inspect "${cluster}-control-plane" --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}')" = "$cluster" + if [ "$PRODUCTION_OUTCOME" != success ] || [ "${LIVE_OUTCOME:-}" = failure ] || [ "${LIVE_OUTCOME:-}" = cancelled ]; then + # Keep pre-restoration real-process evidence distinct from any fallback + # collected after Helm has restored the mock process. + diagnostic_status=missing + evidence="$state/live-diagnostics.jsonl" + if [ -f "$evidence" ] && [ ! -L "$evidence" ] && [ "$(stat -c '%s' "$evidence")" -le 1048576 ] && \ + cp -- "$evidence" "$NATIVE_CI_DIR/live-diagnostics.jsonl"; then + if [ -f "$state/live-diagnostics.status" ] && [ ! -L "$state/live-diagnostics.status" ]; then + IFS= read -r diagnostic_status < "$state/live-diagnostics.status" || diagnostic_status=incomplete + fi + fi + case "$diagnostic_status" in complete|incomplete|missing) ;; *) diagnostic_status=incomplete ;; esac + printf 'pre_restore=%s\n' "$diagnostic_status" > "$NATIVE_CI_DIR/diagnostics-status.txt" || echo '::warning::Diagnostic status unavailable' + if [ "$diagnostic_status" != complete ]; then + if timeout --kill-after=5s 45s sh ./deploy/mecatl-execution-kind/collect-failure.sh \ + "$kubeconfig" "$context" "$NATIVE_CI_DIR/production-diagnostics.jsonl"; then + printf 'fallback=complete\n' >> "$NATIVE_CI_DIR/diagnostics-status.txt" || echo '::warning::Diagnostic status unavailable' + else + printf 'fallback=incomplete\n' >> "$NATIVE_CI_DIR/diagnostics-status.txt" || echo '::warning::Diagnostic status unavailable' + echo '::warning::Bounded production diagnostics incomplete' + fi + fi + fi + kind delete cluster --name "$cluster" + clusters=$(kind get clusters) + if printf '%s\n' "$clusters" | grep -Fx -- "$cluster"; then exit 1; fi + # This file is emitted by the typed live test, not a transcript. Bound + # its upload independently and never collect the surrounding state. + summary="$state/live-summary.json" + if [ -e "$summary" ]; then + test -f "$summary" && test ! -L "$summary" + test "$(stat -c '%s' "$summary")" -le 16384 + cp -- "$summary" "$NATIVE_CI_DIR/live-summary.json" + fi + - name: Record sanitized qualification status + timeout-minutes: 1 + if: always() && steps.prepare.outcome == 'success' + env: + PRODUCTION_OUTCOME: ${{ steps.production.outcome }} + CREDENTIAL_OUTCOME: ${{ steps.credential.outcome }} + LIVE_OUTCOME: ${{ steps.live.outcome }} + CLEANUP_OUTCOME: ${{ steps.cleanup.outcome }} + run: | + printf 'production=%s\ncredential=%s\nlive=%s\ncleanup=%s\n' \ + "$PRODUCTION_OUTCOME" "$CREDENTIAL_OUTCOME" "$LIVE_OUTCOME" "$CLEANUP_OUTCOME" \ + > "$NATIVE_CI_DIR/qualification-status.txt" + cat "$NATIVE_CI_DIR/qualification-status.txt" >> "$GITHUB_STEP_SUMMARY" + - name: Upload only bounded native qualification evidence + timeout-minutes: 2 + if: always() && steps.prepare.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-execution-live-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ env.NATIVE_CI_DIR }}/live-summary.json + ${{ env.NATIVE_CI_DIR }}/qualification-status.txt + ${{ env.NATIVE_CI_DIR }}/production-diagnostics.jsonl + ${{ env.NATIVE_CI_DIR }}/live-diagnostics.jsonl + ${{ env.NATIVE_CI_DIR }}/diagnostics-status.txt + include-hidden-files: true + retention-days: 7 + if-no-files-found: error diff --git a/.github/workflows/k8s-e2e.yml b/.github/workflows/k8s-e2e.yml index c610233e43..9134f1c4f8 100644 --- a/.github/workflows/k8s-e2e.yml +++ b/.github/workflows/k8s-e2e.yml @@ -37,6 +37,17 @@ on: - internal/adapter/server/grpc.go - internal/adapter/server/http.go - deploy/helm/mecak8s/** + - deploy/helm/mecatl-execution/** + - cmd/mecatl-execution-provider/** + - cmd/mecatl-executor/** + - internal/adapter/executioncontroller/** + - internal/adapter/executionclient/** + - internal/executionenv/** + - contracts/proto/mecatl/execution/** + - e2e/k8s_execution/** + - deploy/mecatl-execution-kind/** + - build/execution-provider/Dockerfile + - build/execution-workload/Dockerfile - e2e/k8s/** - go.mod - go.sum @@ -54,6 +65,17 @@ on: - internal/adapter/server/grpc.go - internal/adapter/server/http.go - deploy/helm/mecak8s/** + - deploy/helm/mecatl-execution/** + - cmd/mecatl-execution-provider/** + - cmd/mecatl-executor/** + - internal/adapter/executioncontroller/** + - internal/adapter/executionclient/** + - internal/executionenv/** + - contracts/proto/mecatl/execution/** + - e2e/k8s_execution/** + - deploy/mecatl-execution-kind/** + - build/execution-provider/Dockerfile + - build/execution-workload/Dockerfile - e2e/k8s/** - go.mod - go.sum @@ -123,9 +145,9 @@ jobs: # kind: pinned version, installed via `go install` (no third-party action — # the house idiom for go-installable tools, matching actionlint/govulncheck - # in ci.yml). v0.32.0 is the current release. + # in ci.yml). v0.33.0 is the current release. - name: Install kind - run: go install sigs.k8s.io/kind@v0.32.0 + run: go install sigs.k8s.io/kind@v0.33.0 # ko: pinned version, same `go install` idiom. v0.18.1 builds the mecak8s # image from .ko.yaml onto the distroless base. KO_DOCKER_REPO=ko.local below @@ -166,3 +188,44 @@ jobs: path: k8s-e2e-output.log retention-days: 7 if-no-files-found: warn + + execution-production-e2e: + name: execution provider production profile (kind) + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: | + go.sum + engine/go.sum + - uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v3.18.4 + - name: Install pinned Kind and ko + run: | + go install sigs.k8s.io/kind@v0.33.0 + go install github.com/google/ko@v0.18.1 + - name: Run production execution qualification + shell: bash + env: + CONTAINER_ENGINE: docker + MECATL_EXECUTION_QUAL_CI: "1" + run: task e2e:k8s:execution:production + - name: Upload bounded failure artifact + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: execution-e2e-artifacts-${{ github.run_id }} + path: .scratch/k8s-execution/*/production-failure-artifact.txt + retention-days: 7 + if-no-files-found: warn diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c606297e2a..036daddefd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,7 +48,8 @@ jobs: # KEEPS its own inline copy as well (like the pin gate it repeats): it is the one job that # writes outside this repository, so it re-verifies rather than inheriting. guard: - name: Ensure the release tag is on main + if: github.event_name == 'push' || github.ref == 'refs/heads/main' + name: Verify immutable release tag runs-on: ubuntu-24.04 # 10, not 5: the v0.0.35 release was LOST because an unfiltered depth-0 clone # took 5m18s and tripped a 5-minute cap, skipping all six publish jobs. The @@ -57,6 +58,9 @@ jobs: timeout-minutes: 10 permissions: contents: read + outputs: + version: ${{ steps.verify.outputs.version }} + commit: ${{ steps.verify.outputs.commit }} env: # On workflow_dispatch github.ref_name is the default branch, so prefer the # explicit input; on a tag push github.ref_name is the tag. @@ -65,26 +69,15 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - # 0 so origin/main is present to compare against. NOT `git fetch origin main`: - # persist-credentials is false, so a fetch has no credentials on this repo. fetch-depth: 0 - # This job reads NO file contents — only `git merge-base` and `git rev-parse` - # against the commit graph. An unfiltered depth-0 clone here cost ~5 minutes - # (306 MB, 184 branches, 123 tags) in front of every publish job, since all - # six `needs: guard`. blob:none fetches the graph without the blobs. filter: blob:none - ref: ${{ env.VERSION }} + # The guard implementation is trusted only from protected main. The tag + # and dispatch input remain data verified by that implementation. + ref: refs/heads/main - # HEAD, not $GITHUB_SHA: on workflow_dispatch $GITHUB_SHA is the branch head, while - # HEAD is the tag commit actually checked out above. - - name: Ensure the release tag is on main - run: | - set -euo pipefail - if ! git merge-base --is-ancestor HEAD origin/main; then - echo "::error::release tags must be cut from a commit on main; $(git rev-parse HEAD) is not on main" - exit 1 - fi - echo "$(git rev-parse HEAD) is on main" + - name: Verify and check out the immutable release tag + id: verify + run: .github/scripts/release-tag-guard.sh publish: needs: guard @@ -102,14 +95,14 @@ jobs: KO_DOCKER_REPO: ghcr.io/${{ github.repository }} # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # ko reads the base image digest from .ko.yaml; no deep history needed. fetch-depth: 1 - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: @@ -178,7 +171,7 @@ jobs: --image-refs=image-refs.txt \ --image-label=org.opencontainers.image.source=https://github.com/${{ github.repository }} \ --image-label=org.opencontainers.image.title=mecated \ - --image-label=org.opencontainers.image.revision=${{ github.sha }} \ + --image-label=org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} \ --image-label=org.opencontainers.image.vendor=Stacklok \ ./cmd/mecated # The digest ref is the same image regardless of tag; take the first. @@ -262,14 +255,14 @@ jobs: KO_DOCKER_REPO: ghcr.io/${{ github.repository }}/mecatui # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # ko reads the base image digest from .ko.yaml; no deep history needed. fetch-depth: 1 - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: @@ -340,7 +333,7 @@ jobs: --image-label=org.stacklok.broodbox.agent=/var/run/ko/agent.yaml \ --image-label=org.opencontainers.image.source=https://github.com/${{ github.repository }} \ --image-label=org.opencontainers.image.title=mecatui \ - --image-label=org.opencontainers.image.revision=${{ github.sha }} \ + --image-label=org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} \ --image-label=org.opencontainers.image.vendor=Stacklok \ ./cmd/mecatui # The digest ref is the same image regardless of tag; take the first. @@ -422,14 +415,14 @@ jobs: KO_DOCKER_REPO: ghcr.io/${{ github.repository }}/mecak8s # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # ko reads the base image digest from .ko.yaml; no deep history needed. fetch-depth: 1 - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: @@ -497,7 +490,7 @@ jobs: --image-refs=image-refs.txt \ --image-label=org.opencontainers.image.source=https://github.com/${{ github.repository }} \ --image-label=org.opencontainers.image.title=mecak8s \ - --image-label=org.opencontainers.image.revision=${{ github.sha }} \ + --image-label=org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} \ --image-label=org.opencontainers.image.vendor=Stacklok \ ./cmd/mecak8s # The digest ref is the same image regardless of tag; take the first. @@ -562,6 +555,128 @@ jobs: # silently failing at it. create-storage-record: false + publish-execution-images: + needs: guard + name: Build, publish, sign, attest execution ${{ matrix.name }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + packages: write + id-token: write + attestations: write + strategy: + fail-fast: false + matrix: + include: + - {name: provider, dockerfile: build/execution-provider/Dockerfile, image: mecatl-execution-provider} + - {name: workload, dockerfile: build/execution-workload/Dockerfile, image: mecatl-execution-workload} + env: + VERSION: ${{ needs.guard.outputs.version }} + IMAGE: ghcr.io/${{ github.repository }}/${{ matrix.image }} + GO_IMAGE: ${{ vars.EXECUTION_GO_IMAGE }} + PROVIDER_RUNTIME_IMAGE: ${{ vars.EXECUTION_PROVIDER_RUNTIME_IMAGE }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: {persist-credentials: false, fetch-depth: 1, ref: "${{ needs.guard.outputs.commit }}"} + - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + with: + version: v0.30.1 + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v4.0.0 + with: {registry: ghcr.io, username: "${{ github.actor }}", password: "${{ secrets.GITHUB_TOKEN }}"} + - name: Validate immutable base inputs + run: | + case "$GO_IMAGE" in *@sha256:*) ;; *) echo "EXECUTION_GO_IMAGE must be digest-pinned" >&2; exit 1;; esac + if [ "${{ matrix.name }}" = provider ]; then + case "$PROVIDER_RUNTIME_IMAGE" in *@sha256:*) ;; *) echo "EXECUTION_PROVIDER_RUNTIME_IMAGE must be digest-pinned" >&2; exit 1;; esac + fi + - name: Build and push image + id: build + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64,linux/arm64 + push: true + provenance: false + sbom: false + build-args: | + GO_IMAGE=${{ env.GO_IMAGE }} + RUNTIME_IMAGE=${{ matrix.name == 'provider' && env.PROVIDER_RUNTIME_IMAGE || env.GO_IMAGE }} + tags: | + ${{ env.IMAGE }}:${{ env.VERSION }} + ${{ env.IMAGE }}:latest + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} + - name: Sign image + run: cosign sign --yes "${IMAGE}@${{ steps.build.outputs.digest }}" + - name: Generate SBOM + uses: anchore/sbom-action@3ad7283483fc7af8ff2b4ea19663c2d5ca935e26 # v0.24.2 + with: {image: "${{ env.IMAGE }}@${{ steps.build.outputs.digest }}", format: spdx-json, output-file: sbom.spdx.json, upload-artifact: false} + - name: Attest SBOM + run: cosign attest --yes --predicate sbom.spdx.json --type spdxjson "${IMAGE}@${{ steps.build.outputs.digest }}" + - name: Attest build provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ env.IMAGE }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + create-storage-record: false + + publish-execution-chart: + needs: [guard, publish-execution-images] + name: Package, publish, sign, attest execution Helm chart + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + packages: write + id-token: write + attestations: write + env: + VERSION: ${{ needs.guard.outputs.version }} + OCI_REGISTRY: ghcr.io/${{ github.repository }}/charts + CHART: mecatl-execution + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: {persist-credentials: false, fetch-depth: 1, ref: "${{ needs.guard.outputs.commit }}"} + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: {version: v3.18.4} + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v4.0.0 + with: {registry: ghcr.io, username: "${{ github.actor }}", password: "${{ secrets.GITHUB_TOKEN }}"} + - name: Package chart + id: package + run: | + set -euo pipefail + chart_version="${VERSION#v}" + mkdir -p dist + helm package deploy/helm/mecatl-execution --version "$chart_version" --app-version "$VERSION" --destination dist + echo "chart_version=$chart_version" >> "$GITHUB_OUTPUT" + - name: Push chart + id: push + run: | + set -euo pipefail + helm push "dist/${CHART}-${{ steps.package.outputs.chart_version }}.tgz" "oci://${OCI_REGISTRY}" > push-output.txt 2>&1 + cat push-output.txt + digest="$(awk '/^Digest:/ {print $2}' push-output.txt)" + case "$digest" in sha256:*) ;; *) echo "could not parse chart digest" >&2; exit 1;; esac + echo "digest=$digest" >> "$GITHUB_OUTPUT" + - name: Sign chart + run: cosign sign --yes "${OCI_REGISTRY}/${CHART}@${{ steps.push.outputs.digest }}" + # The chart has no chart dependencies or bundled binaries. Its packaged + # templates and metadata are the artifact, so image SBOMs cover executable + # contents while this immutable chart digest receives provenance directly. + - name: Attest chart provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ env.OCI_REGISTRY }}/${{ env.CHART }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true + create-storage-record: false + # Not a Go binary, so this doesn't go through ko like the jobs above — a # plain docker/build-push-action build of the existing dev/Compose # Dockerfile. The bot's TypeScript source still lives under @@ -582,13 +697,13 @@ jobs: IMAGE: ghcr.io/${{ github.repository }}/slack-bot # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 1 - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} - uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0 - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 @@ -617,7 +732,7 @@ jobs: labels: | org.opencontainers.image.source=https://github.com/${{ github.repository }} org.opencontainers.image.title=slack-bot - org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} org.opencontainers.image.vendor=Stacklok # Sign/SBOM/attest the pushed manifest list BY DIGEST, matching the @@ -676,13 +791,13 @@ jobs: CHART_PATH: deploy/helm/mecak8s # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 1 - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 with: @@ -823,7 +938,7 @@ jobs: env: # On workflow_dispatch github.ref_name is the default branch, so prefer the # explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -844,7 +959,7 @@ jobs: # credentials for a lazy blob fetch, so do not add a step here that reads a # HISTORICAL file's contents. filter: blob:none - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} # Eight cross-compiles (2 binaries x 4 platforms) of this dependency graph # exhaust the runner's disk: the first v0.0.30 attempt died 18 minutes in diff --git a/Taskfile.yml b/Taskfile.yml index 0d298bfaea..e8c6c10493 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -72,6 +72,9 @@ tasks: # mecak8s: the storage-free, k8s-native agent binary (ADR 0048; a thin peer # of mecated composing app.Build with Redis store + k8s lease + drain gate). - go build -ldflags "{{.BUILD_LDFLAGS}}" -o bin/mecak8s ./cmd/mecak8s + # Optional native Kubernetes execution provider and its fixed workload helper. + - go build -ldflags "{{.BUILD_LDFLAGS}}" -o bin/mecatl-execution-provider ./cmd/mecatl-execution-provider + - go build -ldflags "{{.BUILD_LDFLAGS}}" -o bin/mecatl-executor ./cmd/mecatl-executor - go build -ldflags "{{.BUILD_LDFLAGS}}" -o bin/mecatui ./cmd/mecatui # engine/ is its own Go module (consumed via the committed go.work). Building # it here keeps the importable-core module compiling alongside the binaries. @@ -503,6 +506,50 @@ tasks: esac GOWORK=off go test -tags kind_e2e -count=1 -timeout 30m ./e2e/k8s/... + e2e:k8s:execution: + desc: | + Run the focused mock-only Kind qualification for the optional Kubernetes + execution provider. It uses a unique cluster and explicit kubeconfig under + .scratch, builds/loads immutable local images, and never reads a provider + credential. Docker is the default; set CONTAINER_ENGINE=podman for rootless + Podman. The owned cluster is intentionally retained; the command prints the + exact human-confirmed cleanup command. NOT part of task test. + preconditions: + - sh: command -v kind >/dev/null 2>&1 + msg: "kind is required" + - sh: | + runtime="${CONTAINER_ENGINE:-docker}" + case "$runtime" in docker|podman) command -v "$runtime" >/dev/null 2>&1 ;; *) exit 1 ;; esac + msg: "CONTAINER_ENGINE must be docker or podman and available on PATH" + cmds: + - deploy/mecatl-execution-kind/run.sh + + e2e:k8s:execution:production: + desc: | + Run the deterministic production-network and lifecycle execution-provider + qualification. CI sets MECATL_EXECUTION_QUAL_CI=1 so its uniquely owned Kind + cluster is removed automatically; local runs retain it for inspection and + print the human-confirmed cleanup command. No model credential is used. + cmds: + - MECATL_EXECUTION_QUAL_PROFILE=production task e2e:k8s:execution + + e2e:k8s:execution:live: + desc: | + EXPLICIT OPT-IN: run mock qualification followed by one bounded real-provider + coding smoke on the approved retained Kind cluster. Requires absolute + MECATL_EXECUTION_CREDENTIAL_FILE and MECATL_EXECUTION_QUAL_STATE paths. A + trusted loader stages a run-scoped Secret without exposing its value, the + test independently verifies files and go test through typed gRPC, then the + mock profile is restored and only that run-scoped Secret is removed. NOT + part of task test. + preconditions: + - sh: test -n "${MECATL_EXECUTION_CREDENTIAL_FILE:-}" + msg: "MECATL_EXECUTION_CREDENTIAL_FILE is required" + - sh: test -n "${MECATL_EXECUTION_QUAL_STATE:-}" + msg: "MECATL_EXECUTION_QUAL_STATE is required" + cmds: + - deploy/mecatl-execution-kind/live.sh + e2e:k8s:podman: desc: Run e2e:k8s with rootless Podman (requires the k8s-file Podman log driver) cmds: @@ -878,6 +925,7 @@ tasks: - task: lint:action-templates - bash .github/scripts/ensure-perf-dashboard-noindex_test.sh - bash .github/scripts/deslop-workflow_test.sh + - sh .github/scripts/release-tag-guard_test.sh lint:action-templates: desc: | diff --git a/build/execution-provider/Dockerfile b/build/execution-provider/Dockerfile new file mode 100644 index 0000000000..7c3574b94d --- /dev/null +++ b/build/execution-provider/Dockerfile @@ -0,0 +1,16 @@ +# Both bases are mandatory digest references supplied by the release workflow. +ARG GO_IMAGE +ARG RUNTIME_IMAGE +FROM ${GO_IMAGE} AS build +ARG GO_IMAGE +ARG RUNTIME_IMAGE +WORKDIR /src +COPY . . +RUN case "${GO_IMAGE}" in *@sha256:*) ;; *) echo "GO_IMAGE must be digest-pinned" >&2; exit 1;; esac \ + && case "${RUNTIME_IMAGE}" in *@sha256:*) ;; *) echo "RUNTIME_IMAGE must be digest-pinned" >&2; exit 1;; esac \ + && CGO_ENABLED=0 go build -trimpath -buildvcs=false -o /out/mecatl-execution-provider ./cmd/mecatl-execution-provider + +FROM ${RUNTIME_IMAGE} +COPY --from=build --chown=65532:65532 /out/mecatl-execution-provider /mecatl-execution-provider +USER 65532:65532 +ENTRYPOINT ["/mecatl-execution-provider"] diff --git a/build/execution-workload/Dockerfile b/build/execution-workload/Dockerfile new file mode 100644 index 0000000000..1654cf2acb --- /dev/null +++ b/build/execution-workload/Dockerfile @@ -0,0 +1,17 @@ +# GO_IMAGE is deliberately mandatory and must include a digest, for example: +# docker build --build-arg GO_IMAGE=golang:1.27@sha256: ... +ARG GO_IMAGE +FROM ${GO_IMAGE} AS build +ARG GO_IMAGE +WORKDIR /src +COPY . . +RUN case "${GO_IMAGE}" in *@sha256:*) ;; *) echo "GO_IMAGE must be digest-pinned" >&2; exit 1;; esac \ + && go build -trimpath -buildvcs=false -o /out/mecatl-executor ./cmd/mecatl-executor + +FROM ${GO_IMAGE} +COPY --from=build --chown=65532:65532 /out/mecatl-executor /mecatl-executor +RUN mkdir -p /workspace /tmp && chown 65532:65532 /workspace /tmp +USER 65532:65532 +WORKDIR /workspace +ENV HOME=/workspace TMPDIR=/tmp GOTMPDIR=/tmp +ENTRYPOINT ["/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"] diff --git a/cmd/mecak8s/execution_startup_test.go b/cmd/mecak8s/execution_startup_test.go new file mode 100644 index 0000000000..06c2812538 --- /dev/null +++ b/cmd/mecak8s/execution_startup_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" +) + +func TestDisabledExecutionStartupDoesNotContactExecutionOrKubernetes(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "config")) + t.Setenv("XDG_STATE_HOME", filepath.Join(home, "state")) + var requests atomic.Int32 + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer endpoint.Close() + kubeconfig := filepath.Join(home, "kubeconfig") + contents := "apiVersion: v1\nkind: Config\nclusters:\n- name: poison\n cluster:\n server: " + endpoint.URL + "\ncontexts:\n- name: poison\n context:\n cluster: poison\n user: offline\ncurrent-context: poison\nusers:\n- name: offline\n user: {}\n" + if err := os.WriteFile(kubeconfig, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("KUBECONFIG", kubeconfig) + oldArgs, oldLogger := os.Args, slog.Default() + t.Cleanup(func() { os.Args = oldArgs; slog.SetDefault(oldLogger) }) + // Exercise run(), including actual app.Build, stopping at the first listener. + // Poison TLS paths would fail before serve if the disabled client were built. + os.Args = []string{"mecak8s", "--mock", "--posture=strict", "--no-soul", "--no-user-model", "--no-scheduler", "--session-lease-k8s-namespace=", "--grpc-addr=invalid-address", "--execution-enabled=false", "--execution-endpoint=" + strings.TrimPrefix(endpoint.URL, "http://"), "--execution-profile=unused", "--execution-tls-ca=" + filepath.Join(home, "absent-ca"), "--execution-tls-cert=" + filepath.Join(home, "absent-cert"), "--execution-tls-key=" + filepath.Join(home, "absent-key")} + err := run() + if err == nil || !strings.Contains(err.Error(), "invalid-address") { + t.Fatalf("did not reach listener after composition: %v", err) + } + if requests.Load() != 0 { + t.Fatalf("disabled startup made %d API requests", requests.Load()) + } +} + +func TestExecutionPreflightRejectsIncompleteAndIncompatibleCLI(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + required := []string{"--execution-endpoint=127.0.0.1:1", "--execution-profile=go", "--execution-tls-ca=unused", "--execution-tls-cert=unused", "--execution-tls-key=unused"} + base := []string{"--mock", "--execution-enabled", "--oidc-issuer=https://issuer.example", "--oidc-audience=mecatl"} + if _, err := parseFlags(append(append([]string{}, base...), required...)); err != nil { + t.Fatalf("complete configuration rejected: %v", err) + } + for missing := range required { + args := append([]string{}, base...) + for i, flag := range required { + if i != missing { + args = append(args, flag) + } + } + if _, err := parseFlags(args); err == nil || !strings.Contains(err.Error(), "--execution-enabled requires") { + t.Fatalf("missing %s: %v", required[missing], err) + } + } + for _, extra := range []string{"--workspace=unused", "--redis-filesystem", "--enable-parallel", "--enable-teams"} { + args := append(append(append([]string{}, base...), required...), extra) + if _, err := parseFlags(args); err == nil { + t.Fatalf("incompatible %s accepted", extra) + } + } +} diff --git a/cmd/mecak8s/flags.go b/cmd/mecak8s/flags.go index 3d86ca44b5..505bfbbb78 100644 --- a/cmd/mecak8s/flags.go +++ b/cmd/mecak8s/flags.go @@ -120,6 +120,12 @@ type config struct { httpShutdownTimeout time.Duration closeTimeout time.Duration workspace string + executionEnabled bool + executionEndpoint string + executionProfile string + executionTLSCA string + executionTLSCert string + executionTLSKey string model string defaultProvider string defaultModel string @@ -352,6 +358,12 @@ func parseFlags(argv []string) (config, error) { positiveDurationFlag(fs, &cfg.httpShutdownTimeout, "http-shutdown-timeout", defaultHTTPShutdownTimeout, "Maximum time for graceful HTTP and metrics shutdown") positiveDurationFlag(fs, &cfg.closeTimeout, "close-timeout", defaultCloseTimeout, "Maximum time for final application cleanup") fs.StringVar(&cfg.workspace, "workspace", "", "Shared agent workspace root, such as a mounted PVC. Empty gives every session a shell-less, file-less workspace; clients cannot choose another root") + fs.BoolVar(&cfg.executionEnabled, "execution-enabled", false, "Use an independently deployed Kubernetes execution provider for default sessions") + fs.StringVar(&cfg.executionEndpoint, "execution-endpoint", "", "Host:port endpoint of the mTLS gRPC execution provider. Requires --execution-enabled") + fs.StringVar(&cfg.executionProfile, "execution-profile", "", "Operator-configured execution provider profile") + fs.StringVar(&cfg.executionTLSCA, "execution-tls-ca", "", "Mounted CA bundle used only by the execution client") + fs.StringVar(&cfg.executionTLSCert, "execution-tls-cert", "", "Mounted execution-provider mTLS client certificate") + fs.StringVar(&cfg.executionTLSKey, "execution-tls-key", "", "Mounted execution-provider mTLS client private key") fs.StringVar(&cfg.model, "model", "", "Model identifier sent to the provider. Empty uses the provider default") fs.StringVar(&cfg.defaultProvider, "default-provider", "", "Deployment-wide default provider ID, such as openai, openrouter, or anthropic. Invalid values prevent startup") fs.StringVar(&cfg.defaultModel, "default-model", "", "Deployment-wide default model ID for the default provider. Invalid values prevent startup") @@ -612,6 +624,21 @@ func parseFlags(argv []string) (config, error) { if cfg.workspace != "" && (!filepath.IsAbs(cfg.workspace) || filepath.Clean(cfg.workspace) != cfg.workspace) { return config{}, fmt.Errorf("--workspace %q must be a clean absolute path (a mounted filesystem root); leave it empty for a file-less deployment", cfg.workspace) } + if cfg.executionEnabled { + if cfg.executionEndpoint == "" || cfg.executionProfile == "" || cfg.executionTLSCA == "" || cfg.executionTLSCert == "" || cfg.executionTLSKey == "" { + return config{}, errors.New("--execution-enabled requires --execution-endpoint, --execution-profile, --execution-tls-ca, --execution-tls-cert, and --execution-tls-key") + } + if !cfg.oidc.Enabled() { + return config{}, errors.New("--execution-enabled requires OIDC caller ownership enforcement") + } + if cfg.workspace != "" || cfg.redisFilesystem { + return config{}, errors.New("--execution-enabled conflicts with --workspace and --redis-filesystem") + } + if cfg.enableParallel || cfg.enableTeams { + return config{}, errors.New("remote execution does not support --enable-parallel or --enable-teams") + } + cfg.noScheduler = true + } if cfg.redisFilesystem && cfg.workspace != "" { return config{}, errors.New("--redis-filesystem and --workspace are mutually exclusive") } @@ -659,6 +686,7 @@ func appConfig(cfg config, diag port.Diagnostics, obs observability) app.Config mcpAuthorityDefault = mcpauthority.Global } out := app.Config{ + RemoteExecution: cfg.executionEnabled, Workspace: cfg.workspace, Model: cfg.model, DefaultProvider: cfg.defaultProvider, diff --git a/cmd/mecak8s/main.go b/cmd/mecak8s/main.go index d1bcc0c430..aa994c75a1 100644 --- a/cmd/mecak8s/main.go +++ b/cmd/mecak8s/main.go @@ -10,6 +10,8 @@ import ( "os" "time" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/internal/adapter/executionclient" "github.com/stacklok/mecatl/internal/adapter/mockscript" "github.com/stacklok/mecatl/internal/adapter/slogdiag" "github.com/stacklok/mecatl/internal/app" @@ -67,6 +69,11 @@ func run() error { // port.Diagnostics (ban-guarded). Mirrors cmd/mecated. slog.SetDefault(logger) diag := slogdiag.NewFromLogger(logger) + // Native debug qualification consumes structured diagnostics only. Keep the + // ordinary daemon/third-party logger unchanged; no new operator flag is needed. + if cfg.executionEnabled && cfg.logLevel == slog.LevelDebug { + diag = slogdiag.New(os.Stderr, true, port.LevelDebug) + } cfg.diagnostics = diag ctx, stop := signalCtx() @@ -82,6 +89,26 @@ func run() error { } composition := appConfig(cfg, diag, obs) + if cfg.executionEnabled { + tlsConfig, tlsErr := executionclient.LoadTLSConfig(executionclient.TLSFiles{CA: cfg.executionTLSCA, Cert: cfg.executionTLSCert, Key: cfg.executionTLSKey}) + if tlsErr != nil { + flushTelemetry(os.Stderr, obs, cfg.otlpShutdownTimeout) + return tlsErr + } + client, clientErr := executionclient.New(cfg.executionEndpoint, tlsConfig) + if clientErr != nil { + flushTelemetry(os.Stderr, obs, cfg.otlpShutdownTimeout) + return clientErr + } + defer client.Close() + placement, placementErr := executionclient.NewProvider(client, cfg.executionProfile) + if placementErr != nil { + flushTelemetry(os.Stderr, obs, cfg.otlpShutdownTimeout) + return placementErr + } + composition.PlacementProvider = placement + composition.PlacementScope = "remote-execution" + } built, err := app.Build(ctx, composition) if err != nil { flushTelemetry(os.Stderr, obs, cfg.otlpShutdownTimeout) diff --git a/cmd/mecatl-execution-provider/main.go b/cmd/mecatl-execution-provider/main.go new file mode 100644 index 0000000000..87a772fa61 --- /dev/null +++ b/cmd/mecatl-execution-provider/main.go @@ -0,0 +1,161 @@ +// Command mecatl-execution-provider runs the authenticated Kubernetes execution provider. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func main() { + if err := run(); err != nil { + slog.Error("execution provider stopped", "error", err) + os.Exit(1) + } +} +func run() error { //nolint:gocyclo // Startup validation and owned-resource shutdown stay in one composition root. + var addr, healthAddr, namespace, profilesPath, manifestPath, keyDirectory, authorityConfigMap string + var reloadInterval time.Duration + var maxConcurrentStreams, maxConcurrentRPCs, maxConcurrentRPCsPerClient int + flag.StringVar(&addr, "listen", ":8443", "gRPC listen address") + flag.StringVar(&namespace, "namespace", "", "managed Kubernetes namespace") + flag.StringVar(&profilesPath, "profiles", "/etc/mecatl-execution/profiles.yaml", "strict operator profile file") + flag.StringVar(&healthAddr, "health-listen", ":8081", "operational HTTP health listen address; empty disables") + flag.StringVar(&manifestPath, "grant-keyring-manifest", "/etc/mecatl-execution/security/manifest.json", "versioned security manifest") + flag.StringVar(&keyDirectory, "grant-key-directory", "/etc/mecatl-execution/security", "projected security key and TLS directory") + flag.StringVar(&authorityConfigMap, "security-authority-configmap", "", "provider-owned security generation high-water ConfigMap") + flag.DurationVar(&reloadInterval, "security-reload-interval", 2*time.Second, "security material reload interval") + flag.IntVar(&maxConcurrentStreams, "max-concurrent-streams", 64, "maximum concurrent HTTP/2 streams per connection") + flag.IntVar(&maxConcurrentRPCs, "max-concurrent-rpcs", 128, "maximum active provider RPCs") + flag.IntVar(&maxConcurrentRPCsPerClient, "max-concurrent-rpcs-per-client", 32, "maximum active provider RPCs per authorized client") + flag.Parse() + if namespace == "" || authorityConfigMap == "" || reloadInterval <= 0 || reloadInterval > time.Minute || maxConcurrentStreams < 1 || maxConcurrentStreams > 1024 || maxConcurrentRPCs < 1 || maxConcurrentRPCs > 4096 || maxConcurrentRPCsPerClient < 1 || maxConcurrentRPCsPerClient > maxConcurrentRPCs { + return errors.New("required identity, security reload interval, or RPC concurrency bounds are invalid") + } + profiles, err := executioncontroller.LoadProfiles(profilesPath) + if err != nil { + return err + } + cfg, err := rest.InClusterConfig() + if err != nil { + return fmt.Errorf("build in-cluster Kubernetes config: %w", err) + } + kube, err := kubernetes.NewForConfig(cfg) + if err != nil { + return err + } + dyn, err := dynamic.NewForConfig(cfg) + if err != nil { + return err + } + security := executioncontroller.NewSecurityManager(manifestPath, keyDirectory, namespace, authorityConfigMap, kube) + if err := security.Reload(context.Background()); err != nil { + return fmt.Errorf("load security material: %w", err) + } + podexec := executioncontroller.NewPodExecutor(cfg, kube, namespace) + store := executioncontroller.NewStore(dyn, namespace, profiles, podexec).WithKubeClient(kube) + reconciler := executioncontroller.NewReconciler(dyn, kube, namespace, profiles) + handler := executioncontroller.NewHandler(executioncontroller.HandlerConfig{Security: security, Ready: func() bool { return reconciler.Ready() && security.Ready() }}, store) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + go security.Run(ctx, reloadInterval) + if err := reconciler.Initialize(ctx); err != nil { + return fmt.Errorf("initialize controller: %w", err) + } + controllerErr := make(chan error, 1) + go func() { controllerErr <- reconciler.Run(ctx) }() + ln, err := net.Listen("tcp", addr) + if err != nil { + return err + } + limiter, err := executioncontroller.NewRPCLimiter(security, maxConcurrentRPCs, maxConcurrentRPCsPerClient) + if err != nil { + return err + } + server := grpc.NewServer( + grpc.Creds(credentials.NewTLS(security.TLSConfig())), + grpc.ChainUnaryInterceptor(limiter.UnaryInterceptor), + grpc.MaxRecvMsgSize(executionenv.MaxMessageBytes), + grpc.MaxSendMsgSize(executionenv.MaxMessageBytes), + grpc.MaxConcurrentStreams(uint32(maxConcurrentStreams)), //nolint:gosec // validated to 1..1024 above. + grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: 30 * time.Minute, MaxConnectionAgeGrace: 2 * time.Minute, Time: 2 * time.Minute, Timeout: 20 * time.Second}), + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{MinTime: 30 * time.Second, PermitWithoutStream: false}), + ) + executionv1.RegisterExecutionProviderServiceServer(server, handler) + var healthServer *http.Server + if healthAddr != "" { + mux := http.NewServeMux() + mux.HandleFunc("GET /live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + mux.HandleFunc("GET /ready", func(w http.ResponseWriter, r *http.Request) { + if !reconciler.Ready() { + http.Error(w, "not ready: "+reconciler.ReadinessReason(), http.StatusServiceUnavailable) + return + } + readyCtx, stop := context.WithTimeout(r.Context(), 2*time.Second) + defer stop() + if !security.CheckReady(readyCtx) { + http.Error(w, "not ready: security-authority-or-expiry", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) + }) + healthServer = &http.Server{Addr: healthAddr, Handler: mux, ReadHeaderTimeout: 2 * time.Second, IdleTimeout: 30 * time.Second} + go func() { + if err := healthServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + cancel() + } + }() + } + defer func() { + if healthServer != nil { + shutdown, stop := context.WithTimeout(context.Background(), 3*time.Second) + defer stop() + _ = healthServer.Shutdown(shutdown) + } + }() + serveErr := make(chan error, 1) + go func() { serveErr <- server.Serve(ln) }() + select { + case err := <-controllerErr: + cancel() + server.GracefulStop() + if err != nil { + return fmt.Errorf("controller: %w", err) + } + return nil + case err := <-serveErr: + cancel() + if err != nil { + return err + } + return nil + case <-ctx.Done(): + done := make(chan struct{}) + go func() { server.GracefulStop(); close(done) }() + select { + case <-done: + case <-time.After(10 * time.Second): + server.Stop() + } + return nil + } +} diff --git a/cmd/mecatl-executor/main.go b/cmd/mecatl-executor/main.go new file mode 100644 index 0000000000..34c594783e --- /dev/null +++ b/cmd/mecatl-executor/main.go @@ -0,0 +1,81 @@ +// Command mecatl-executor runs the fixed credential-free workload helper. +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/signal" + "runtime" + "syscall" + "time" + + "github.com/stacklok/mecatl/internal/executionenv" + "github.com/stacklok/mecatl/internal/executionexecutor" +) + +const workspaceRoot = "/workspace" + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + if err := run(ctx, os.Stdin, os.Stdout); err != nil { + _, _ = fmt.Fprintln(os.Stderr, "mecatl-executor: protocol output failed") + os.Exit(1) + } +} + +func run(parent context.Context, in io.Reader, out io.Writer) error { + var resp executionenv.ExecutorResponse + var opErr error + if runtime.GOOS != "linux" { + opErr = errors.New("mecatl-executor requires Linux at runtime") + } else if err := executionexecutor.EnableCommandExecution(); err != nil { + opErr = err + } else { + reader := bufio.NewReader(&io.LimitedReader{R: in, N: executionenv.MaxJSONBody + 1}) + body, err := reader.ReadBytes('\n') + if err != nil && !errors.Is(err, io.EOF) { + opErr = err + } else if len(body) > executionenv.MaxJSONBody { + opErr = &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "executor request exceeds limit"} + } else { + var q executionenv.ExecutorRequest + if err := executionenv.DecodeStrict(body, &q); err != nil { + opErr = err + } else { + x, err := executionexecutor.New(workspaceRoot, executionexecutor.Limits{CommandTimeout: 30 * time.Minute}) + if err != nil { + opErr = err + } else { + defer func() { _ = x.Close() }() + opCtx, cancel := context.WithCancel(parent) + if q.Operation == executionenv.OpCommandStart { + go func() { + var one [1]byte + _, _ = reader.Read(one[:]) + cancel() + }() + } + resp, opErr = x.Execute(opCtx, q) + cancel() + } + } + } + } + envelope := executionenv.ExecutorEnvelope{} + if opErr != nil { + var pe *executionenv.Error + if !errors.As(opErr, &pe) { + pe = &executionenv.Error{Code: executionenv.CodeInternal, Message: "executor operation failed"} + } + envelope.Error = pe + } else { + envelope.Response = &resp + } + return json.NewEncoder(out).Encode(envelope) +} diff --git a/contracts/gen/go/mecatl/execution/v1/execution.pb.go b/contracts/gen/go/mecatl/execution/v1/execution.pb.go new file mode 100644 index 0000000000..83f59d3ff6 --- /dev/null +++ b/contracts/gen/go/mecatl/execution/v1/execution.pb.go @@ -0,0 +1,3684 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: LicenseRef-Stacklok-Proprietary + +// Private, mutually authenticated execution-provider protocol. This is not part +// of the public Harness API. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.4 +// protoc (unknown) +// source: mecatl/execution/v1/execution.proto + +package executionv1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type FileOperation int32 + +const ( + FileOperation_FILE_OPERATION_UNSPECIFIED FileOperation = 0 + FileOperation_FILE_OPERATION_READ FileOperation = 1 + FileOperation_FILE_OPERATION_RESOLVE_AUTHORITY FileOperation = 2 + FileOperation_FILE_OPERATION_STAT FileOperation = 3 + FileOperation_FILE_OPERATION_CREATE FileOperation = 4 + FileOperation_FILE_OPERATION_REPLACE FileOperation = 5 + FileOperation_FILE_OPERATION_LIST FileOperation = 6 + FileOperation_FILE_OPERATION_REMOVE FileOperation = 7 + FileOperation_FILE_OPERATION_RENAME FileOperation = 8 + FileOperation_FILE_OPERATION_COPY FileOperation = 9 + FileOperation_FILE_OPERATION_GLOB FileOperation = 10 + FileOperation_FILE_OPERATION_GREP FileOperation = 11 +) + +// Enum value maps for FileOperation. +var ( + FileOperation_name = map[int32]string{ + 0: "FILE_OPERATION_UNSPECIFIED", + 1: "FILE_OPERATION_READ", + 2: "FILE_OPERATION_RESOLVE_AUTHORITY", + 3: "FILE_OPERATION_STAT", + 4: "FILE_OPERATION_CREATE", + 5: "FILE_OPERATION_REPLACE", + 6: "FILE_OPERATION_LIST", + 7: "FILE_OPERATION_REMOVE", + 8: "FILE_OPERATION_RENAME", + 9: "FILE_OPERATION_COPY", + 10: "FILE_OPERATION_GLOB", + 11: "FILE_OPERATION_GREP", + } + FileOperation_value = map[string]int32{ + "FILE_OPERATION_UNSPECIFIED": 0, + "FILE_OPERATION_READ": 1, + "FILE_OPERATION_RESOLVE_AUTHORITY": 2, + "FILE_OPERATION_STAT": 3, + "FILE_OPERATION_CREATE": 4, + "FILE_OPERATION_REPLACE": 5, + "FILE_OPERATION_LIST": 6, + "FILE_OPERATION_REMOVE": 7, + "FILE_OPERATION_RENAME": 8, + "FILE_OPERATION_COPY": 9, + "FILE_OPERATION_GLOB": 10, + "FILE_OPERATION_GREP": 11, + } +) + +func (x FileOperation) Enum() *FileOperation { + p := new(FileOperation) + *p = x + return p +} + +func (x FileOperation) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FileOperation) Descriptor() protoreflect.EnumDescriptor { + return file_mecatl_execution_v1_execution_proto_enumTypes[0].Descriptor() +} + +func (FileOperation) Type() protoreflect.EnumType { + return &file_mecatl_execution_v1_execution_proto_enumTypes[0] +} + +func (x FileOperation) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use FileOperation.Descriptor instead. +func (FileOperation) EnumDescriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{0} +} + +type CommandState int32 + +const ( + CommandState_COMMAND_STATE_UNSPECIFIED CommandState = 0 + CommandState_COMMAND_STATE_RUNNING CommandState = 1 + CommandState_COMMAND_STATE_SUCCEEDED CommandState = 2 + CommandState_COMMAND_STATE_FAILED CommandState = 3 + CommandState_COMMAND_STATE_CANCELLED CommandState = 4 + CommandState_COMMAND_STATE_FENCE_UNKNOWN CommandState = 5 +) + +// Enum value maps for CommandState. +var ( + CommandState_name = map[int32]string{ + 0: "COMMAND_STATE_UNSPECIFIED", + 1: "COMMAND_STATE_RUNNING", + 2: "COMMAND_STATE_SUCCEEDED", + 3: "COMMAND_STATE_FAILED", + 4: "COMMAND_STATE_CANCELLED", + 5: "COMMAND_STATE_FENCE_UNKNOWN", + } + CommandState_value = map[string]int32{ + "COMMAND_STATE_UNSPECIFIED": 0, + "COMMAND_STATE_RUNNING": 1, + "COMMAND_STATE_SUCCEEDED": 2, + "COMMAND_STATE_FAILED": 3, + "COMMAND_STATE_CANCELLED": 4, + "COMMAND_STATE_FENCE_UNKNOWN": 5, + } +) + +func (x CommandState) Enum() *CommandState { + p := new(CommandState) + *p = x + return p +} + +func (x CommandState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CommandState) Descriptor() protoreflect.EnumDescriptor { + return file_mecatl_execution_v1_execution_proto_enumTypes[1].Descriptor() +} + +func (CommandState) Type() protoreflect.EnumType { + return &file_mecatl_execution_v1_execution_proto_enumTypes[1] +} + +func (x CommandState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CommandState.Descriptor instead. +func (CommandState) EnumDescriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{1} +} + +type EnvironmentRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Revision string `protobuf:"bytes,2,opt,name=revision,proto3" json:"revision,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnvironmentRef) Reset() { + *x = EnvironmentRef{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnvironmentRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnvironmentRef) ProtoMessage() {} + +func (x *EnvironmentRef) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnvironmentRef.ProtoReflect.Descriptor instead. +func (*EnvironmentRef) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{0} +} + +func (x *EnvironmentRef) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *EnvironmentRef) GetRevision() string { + if x != nil { + return x.Revision + } + return "" +} + +type Owner struct { + state protoimpl.MessageState `protogen:"open.v1"` + Issuer string `protobuf:"bytes,1,opt,name=issuer,proto3" json:"issuer,omitempty"` + Subject string `protobuf:"bytes,2,opt,name=subject,proto3" json:"subject,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Owner) Reset() { + *x = Owner{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Owner) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Owner) ProtoMessage() {} + +func (x *Owner) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Owner.ProtoReflect.Descriptor instead. +func (*Owner) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{1} +} + +func (x *Owner) GetIssuer() string { + if x != nil { + return x.Issuer + } + return "" +} + +func (x *Owner) GetSubject() string { + if x != nil { + return x.Subject + } + return "" +} + +type RequestContext struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + Epoch uint64 `protobuf:"varint,4,opt,name=epoch,proto3" json:"epoch,omitempty"` + Grant string `protobuf:"bytes,5,opt,name=grant,proto3" json:"grant,omitempty"` + RunId string `protobuf:"bytes,6,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ClaimId string `protobuf:"bytes,7,opt,name=claim_id,json=claimId,proto3" json:"claim_id,omitempty"` + GrantGeneration uint64 `protobuf:"varint,8,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestContext) Reset() { + *x = RequestContext{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestContext) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestContext) ProtoMessage() {} + +func (x *RequestContext) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestContext.ProtoReflect.Descriptor instead. +func (*RequestContext) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{2} +} + +func (x *RequestContext) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RequestContext) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RequestContext) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *RequestContext) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *RequestContext) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *RequestContext) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *RequestContext) GetClaimId() string { + if x != nil { + return x.ClaimId + } + return "" +} + +func (x *RequestContext) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type ValidateProfileRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Profile string `protobuf:"bytes,1,opt,name=profile,proto3" json:"profile,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ValidateProfileRequest) Reset() { + *x = ValidateProfileRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ValidateProfileRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ValidateProfileRequest) ProtoMessage() {} + +func (x *ValidateProfileRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ValidateProfileRequest.ProtoReflect.Descriptor instead. +func (*ValidateProfileRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{3} +} + +func (x *ValidateProfileRequest) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +type ValidateProfileResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Profile string `protobuf:"bytes,1,opt,name=profile,proto3" json:"profile,omitempty"` + Digest string `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"` + Capabilities []string `protobuf:"bytes,3,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + MaxFileBytes int64 `protobuf:"varint,4,opt,name=max_file_bytes,json=maxFileBytes,proto3" json:"max_file_bytes,omitempty"` + MaxCommandBytes int64 `protobuf:"varint,5,opt,name=max_command_bytes,json=maxCommandBytes,proto3" json:"max_command_bytes,omitempty"` + MaxCommandDurationMillis int64 `protobuf:"varint,6,opt,name=max_command_duration_millis,json=maxCommandDurationMillis,proto3" json:"max_command_duration_millis,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ValidateProfileResponse) Reset() { + *x = ValidateProfileResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ValidateProfileResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ValidateProfileResponse) ProtoMessage() {} + +func (x *ValidateProfileResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ValidateProfileResponse.ProtoReflect.Descriptor instead. +func (*ValidateProfileResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{4} +} + +func (x *ValidateProfileResponse) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +func (x *ValidateProfileResponse) GetDigest() string { + if x != nil { + return x.Digest + } + return "" +} + +func (x *ValidateProfileResponse) GetCapabilities() []string { + if x != nil { + return x.Capabilities + } + return nil +} + +func (x *ValidateProfileResponse) GetMaxFileBytes() int64 { + if x != nil { + return x.MaxFileBytes + } + return 0 +} + +func (x *ValidateProfileResponse) GetMaxCommandBytes() int64 { + if x != nil { + return x.MaxCommandBytes + } + return 0 +} + +func (x *ValidateProfileResponse) GetMaxCommandDurationMillis() int64 { + if x != nil { + return x.MaxCommandDurationMillis + } + return 0 +} + +type EnsureEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + BindingId string `protobuf:"bytes,1,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + Profile string `protobuf:"bytes,2,opt,name=profile,proto3" json:"profile,omitempty"` + Owner *Owner `protobuf:"bytes,3,opt,name=owner,proto3" json:"owner,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnsureEnvironmentRequest) Reset() { + *x = EnsureEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnsureEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnsureEnvironmentRequest) ProtoMessage() {} + +func (x *EnsureEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnsureEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*EnsureEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{5} +} + +func (x *EnsureEnvironmentRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *EnsureEnvironmentRequest) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +func (x *EnsureEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *EnsureEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type EnsureEnvironmentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` + Ready bool `protobuf:"varint,3,opt,name=ready,proto3" json:"ready,omitempty"` + Grant string `protobuf:"bytes,4,opt,name=grant,proto3" json:"grant,omitempty"` + GrantExpiresAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=grant_expires_at,json=grantExpiresAt,proto3" json:"grant_expires_at,omitempty"` + GrantGeneration uint64 `protobuf:"varint,6,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnsureEnvironmentResponse) Reset() { + *x = EnsureEnvironmentResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnsureEnvironmentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnsureEnvironmentResponse) ProtoMessage() {} + +func (x *EnsureEnvironmentResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnsureEnvironmentResponse.ProtoReflect.Descriptor instead. +func (*EnsureEnvironmentResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{6} +} + +func (x *EnsureEnvironmentResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *EnsureEnvironmentResponse) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *EnsureEnvironmentResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *EnsureEnvironmentResponse) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *EnsureEnvironmentResponse) GetGrantExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.GrantExpiresAt + } + return nil +} + +func (x *EnsureEnvironmentResponse) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type AttachEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + Purpose string `protobuf:"bytes,2,opt,name=purpose,proto3" json:"purpose,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttachEnvironmentRequest) Reset() { + *x = AttachEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttachEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttachEnvironmentRequest) ProtoMessage() {} + +func (x *AttachEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttachEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*AttachEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{7} +} + +func (x *AttachEnvironmentRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *AttachEnvironmentRequest) GetPurpose() string { + if x != nil { + return x.Purpose + } + return "" +} + +type AttachEnvironmentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` + Ready bool `protobuf:"varint,3,opt,name=ready,proto3" json:"ready,omitempty"` + Grant string `protobuf:"bytes,4,opt,name=grant,proto3" json:"grant,omitempty"` + GrantExpiresAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=grant_expires_at,json=grantExpiresAt,proto3" json:"grant_expires_at,omitempty"` + GrantGeneration uint64 `protobuf:"varint,6,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttachEnvironmentResponse) Reset() { + *x = AttachEnvironmentResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttachEnvironmentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttachEnvironmentResponse) ProtoMessage() {} + +func (x *AttachEnvironmentResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttachEnvironmentResponse.ProtoReflect.Descriptor instead. +func (*AttachEnvironmentResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{8} +} + +func (x *AttachEnvironmentResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *AttachEnvironmentResponse) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *AttachEnvironmentResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *AttachEnvironmentResponse) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *AttachEnvironmentResponse) GetGrantExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.GrantExpiresAt + } + return nil +} + +func (x *AttachEnvironmentResponse) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type AcquireRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + RunId string `protobuf:"bytes,4,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + OperationId string `protobuf:"bytes,5,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + TtlMillis int64 `protobuf:"varint,6,opt,name=ttl_millis,json=ttlMillis,proto3" json:"ttl_millis,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AcquireRunRequest) Reset() { + *x = AcquireRunRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AcquireRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AcquireRunRequest) ProtoMessage() {} + +func (x *AcquireRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AcquireRunRequest.ProtoReflect.Descriptor instead. +func (*AcquireRunRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{9} +} + +func (x *AcquireRunRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *AcquireRunRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *AcquireRunRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *AcquireRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *AcquireRunRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +func (x *AcquireRunRequest) GetTtlMillis() int64 { + if x != nil { + return x.TtlMillis + } + return 0 +} + +type RenewRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + RunId string `protobuf:"bytes,4,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ClaimId string `protobuf:"bytes,5,opt,name=claim_id,json=claimId,proto3" json:"claim_id,omitempty"` + Epoch uint64 `protobuf:"varint,6,opt,name=epoch,proto3" json:"epoch,omitempty"` + OperationId string `protobuf:"bytes,7,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + TtlMillis int64 `protobuf:"varint,8,opt,name=ttl_millis,json=ttlMillis,proto3" json:"ttl_millis,omitempty"` + GrantGeneration uint64 `protobuf:"varint,9,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RenewRunRequest) Reset() { + *x = RenewRunRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RenewRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RenewRunRequest) ProtoMessage() {} + +func (x *RenewRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RenewRunRequest.ProtoReflect.Descriptor instead. +func (*RenewRunRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{10} +} + +func (x *RenewRunRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RenewRunRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RenewRunRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *RenewRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *RenewRunRequest) GetClaimId() string { + if x != nil { + return x.ClaimId + } + return "" +} + +func (x *RenewRunRequest) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *RenewRunRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +func (x *RenewRunRequest) GetTtlMillis() int64 { + if x != nil { + return x.TtlMillis + } + return 0 +} + +func (x *RenewRunRequest) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type ReleaseRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + RunId string `protobuf:"bytes,4,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ClaimId string `protobuf:"bytes,5,opt,name=claim_id,json=claimId,proto3" json:"claim_id,omitempty"` + Epoch uint64 `protobuf:"varint,6,opt,name=epoch,proto3" json:"epoch,omitempty"` + OperationId string `protobuf:"bytes,7,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + GrantGeneration uint64 `protobuf:"varint,8,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReleaseRunRequest) Reset() { + *x = ReleaseRunRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReleaseRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReleaseRunRequest) ProtoMessage() {} + +func (x *ReleaseRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReleaseRunRequest.ProtoReflect.Descriptor instead. +func (*ReleaseRunRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{11} +} + +func (x *ReleaseRunRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReleaseRunRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ReleaseRunRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *ReleaseRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *ReleaseRunRequest) GetClaimId() string { + if x != nil { + return x.ClaimId + } + return "" +} + +func (x *ReleaseRunRequest) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *ReleaseRunRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +func (x *ReleaseRunRequest) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type RunClaimResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + BindingId string `protobuf:"bytes,2,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + RunId string `protobuf:"bytes,3,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ClaimId string `protobuf:"bytes,4,opt,name=claim_id,json=claimId,proto3" json:"claim_id,omitempty"` + Epoch uint64 `protobuf:"varint,5,opt,name=epoch,proto3" json:"epoch,omitempty"` + GrantGeneration uint64 `protobuf:"varint,6,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + Grant string `protobuf:"bytes,7,opt,name=grant,proto3" json:"grant,omitempty"` + ExpiresAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RunClaimResponse) Reset() { + *x = RunClaimResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RunClaimResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunClaimResponse) ProtoMessage() {} + +func (x *RunClaimResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunClaimResponse.ProtoReflect.Descriptor instead. +func (*RunClaimResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{12} +} + +func (x *RunClaimResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RunClaimResponse) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *RunClaimResponse) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *RunClaimResponse) GetClaimId() string { + if x != nil { + return x.ClaimId + } + return "" +} + +func (x *RunClaimResponse) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *RunClaimResponse) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +func (x *RunClaimResponse) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *RunClaimResponse) GetExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.ExpiresAt + } + return nil +} + +type ReferenceMutationRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReferenceMutationRequest) Reset() { + *x = ReferenceMutationRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReferenceMutationRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReferenceMutationRequest) ProtoMessage() {} + +func (x *ReferenceMutationRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReferenceMutationRequest.ProtoReflect.Descriptor instead. +func (*ReferenceMutationRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{13} +} + +func (x *ReferenceMutationRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReferenceMutationRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ReferenceMutationRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *ReferenceMutationRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type ReserveSuccessorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + SourceBindingId string `protobuf:"bytes,3,opt,name=source_binding_id,json=sourceBindingId,proto3" json:"source_binding_id,omitempty"` + DestinationBindingId string `protobuf:"bytes,4,opt,name=destination_binding_id,json=destinationBindingId,proto3" json:"destination_binding_id,omitempty"` + OperationId string `protobuf:"bytes,5,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReserveSuccessorRequest) Reset() { + *x = ReserveSuccessorRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReserveSuccessorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReserveSuccessorRequest) ProtoMessage() {} + +func (x *ReserveSuccessorRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReserveSuccessorRequest.ProtoReflect.Descriptor instead. +func (*ReserveSuccessorRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{14} +} + +func (x *ReserveSuccessorRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReserveSuccessorRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ReserveSuccessorRequest) GetSourceBindingId() string { + if x != nil { + return x.SourceBindingId + } + return "" +} + +func (x *ReserveSuccessorRequest) GetDestinationBindingId() string { + if x != nil { + return x.DestinationBindingId + } + return "" +} + +func (x *ReserveSuccessorRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type ReferenceReservationResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReferenceReservationResponse) Reset() { + *x = ReferenceReservationResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReferenceReservationResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReferenceReservationResponse) ProtoMessage() {} + +func (x *ReferenceReservationResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReferenceReservationResponse.ProtoReflect.Descriptor instead. +func (*ReferenceReservationResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{15} +} + +func (x *ReferenceReservationResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +type ListReferenceIntentsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Owner *Owner `protobuf:"bytes,1,opt,name=owner,proto3" json:"owner,omitempty"` + Limit int32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + Environment *EnvironmentRef `protobuf:"bytes,3,opt,name=environment,proto3" json:"environment,omitempty"` + BindingId string `protobuf:"bytes,4,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListReferenceIntentsRequest) Reset() { + *x = ListReferenceIntentsRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListReferenceIntentsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListReferenceIntentsRequest) ProtoMessage() {} + +func (x *ListReferenceIntentsRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListReferenceIntentsRequest.ProtoReflect.Descriptor instead. +func (*ListReferenceIntentsRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{16} +} + +func (x *ListReferenceIntentsRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ListReferenceIntentsRequest) GetLimit() int32 { + if x != nil { + return x.Limit + } + return 0 +} + +func (x *ListReferenceIntentsRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ListReferenceIntentsRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +type ReferenceIntent struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + BindingId string `protobuf:"bytes,2,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + State string `protobuf:"bytes,3,opt,name=state,proto3" json:"state,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + SourceBindingId string `protobuf:"bytes,5,opt,name=source_binding_id,json=sourceBindingId,proto3" json:"source_binding_id,omitempty"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + Owner *Owner `protobuf:"bytes,7,opt,name=owner,proto3" json:"owner,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReferenceIntent) Reset() { + *x = ReferenceIntent{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReferenceIntent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReferenceIntent) ProtoMessage() {} + +func (x *ReferenceIntent) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReferenceIntent.ProtoReflect.Descriptor instead. +func (*ReferenceIntent) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{17} +} + +func (x *ReferenceIntent) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReferenceIntent) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *ReferenceIntent) GetState() string { + if x != nil { + return x.State + } + return "" +} + +func (x *ReferenceIntent) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +func (x *ReferenceIntent) GetSourceBindingId() string { + if x != nil { + return x.SourceBindingId + } + return "" +} + +func (x *ReferenceIntent) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +func (x *ReferenceIntent) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +type ListReferenceIntentsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Intents []*ReferenceIntent `protobuf:"bytes,1,rep,name=intents,proto3" json:"intents,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListReferenceIntentsResponse) Reset() { + *x = ListReferenceIntentsResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListReferenceIntentsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListReferenceIntentsResponse) ProtoMessage() {} + +func (x *ListReferenceIntentsResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListReferenceIntentsResponse.ProtoReflect.Descriptor instead. +func (*ListReferenceIntentsResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{18} +} + +func (x *ListReferenceIntentsResponse) GetIntents() []*ReferenceIntent { + if x != nil { + return x.Intents + } + return nil +} + +type ReleaseReferenceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReleaseReferenceRequest) Reset() { + *x = ReleaseReferenceRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReleaseReferenceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReleaseReferenceRequest) ProtoMessage() {} + +func (x *ReleaseReferenceRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReleaseReferenceRequest.ProtoReflect.Descriptor instead. +func (*ReleaseReferenceRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{19} +} + +func (x *ReleaseReferenceRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +type RetireEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedExecutionEpoch uint64 `protobuf:"varint,3,opt,name=expected_execution_epoch,json=expectedExecutionEpoch,proto3" json:"expected_execution_epoch,omitempty"` + ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RetireEnvironmentRequest) Reset() { + *x = RetireEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RetireEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RetireEnvironmentRequest) ProtoMessage() {} + +func (x *RetireEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RetireEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*RetireEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{20} +} + +func (x *RetireEnvironmentRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RetireEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RetireEnvironmentRequest) GetExpectedExecutionEpoch() uint64 { + if x != nil { + return x.ExpectedExecutionEpoch + } + return 0 +} + +func (x *RetireEnvironmentRequest) GetExpectedPodUid() string { + if x != nil { + return x.ExpectedPodUid + } + return "" +} + +func (x *RetireEnvironmentRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid + } + return "" +} + +func (x *RetireEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type ReplaceExecutorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedExecutionEpoch uint64 `protobuf:"varint,3,opt,name=expected_execution_epoch,json=expectedExecutionEpoch,proto3" json:"expected_execution_epoch,omitempty"` + ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReplaceExecutorRequest) Reset() { + *x = ReplaceExecutorRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReplaceExecutorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReplaceExecutorRequest) ProtoMessage() {} + +func (x *ReplaceExecutorRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReplaceExecutorRequest.ProtoReflect.Descriptor instead. +func (*ReplaceExecutorRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{21} +} + +func (x *ReplaceExecutorRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReplaceExecutorRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ReplaceExecutorRequest) GetExpectedExecutionEpoch() uint64 { + if x != nil { + return x.ExpectedExecutionEpoch + } + return 0 +} + +func (x *ReplaceExecutorRequest) GetExpectedPodUid() string { + if x != nil { + return x.ExpectedPodUid + } + return "" +} + +func (x *ReplaceExecutorRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid + } + return "" +} + +func (x *ReplaceExecutorRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type RecoverEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedExecutionEpoch uint64 `protobuf:"varint,3,opt,name=expected_execution_epoch,json=expectedExecutionEpoch,proto3" json:"expected_execution_epoch,omitempty"` + ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RecoverEnvironmentRequest) Reset() { + *x = RecoverEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RecoverEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RecoverEnvironmentRequest) ProtoMessage() {} + +func (x *RecoverEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RecoverEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*RecoverEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{22} +} + +func (x *RecoverEnvironmentRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RecoverEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RecoverEnvironmentRequest) GetExpectedExecutionEpoch() uint64 { + if x != nil { + return x.ExpectedExecutionEpoch + } + return 0 +} + +func (x *RecoverEnvironmentRequest) GetExpectedPodUid() string { + if x != nil { + return x.ExpectedPodUid + } + return "" +} + +func (x *RecoverEnvironmentRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid + } + return "" +} + +func (x *RecoverEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type DeleteRetiredEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,3,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteRetiredEnvironmentRequest) Reset() { + *x = DeleteRetiredEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteRetiredEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteRetiredEnvironmentRequest) ProtoMessage() {} + +func (x *DeleteRetiredEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteRetiredEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*DeleteRetiredEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{23} +} + +func (x *DeleteRetiredEnvironmentRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *DeleteRetiredEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *DeleteRetiredEnvironmentRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid + } + return "" +} + +func (x *DeleteRetiredEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type RevokeEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedGrantGeneration uint64 `protobuf:"varint,3,opt,name=expected_grant_generation,json=expectedGrantGeneration,proto3" json:"expected_grant_generation,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RevokeEnvironmentRequest) Reset() { + *x = RevokeEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RevokeEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RevokeEnvironmentRequest) ProtoMessage() {} + +func (x *RevokeEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RevokeEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*RevokeEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{24} +} + +func (x *RevokeEnvironmentRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RevokeEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RevokeEnvironmentRequest) GetExpectedGrantGeneration() uint64 { + if x != nil { + return x.ExpectedGrantGeneration + } + return 0 +} + +func (x *RevokeEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type RevokeEnvironmentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + GrantGeneration uint64 `protobuf:"varint,1,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RevokeEnvironmentResponse) Reset() { + *x = RevokeEnvironmentResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RevokeEnvironmentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RevokeEnvironmentResponse) ProtoMessage() {} + +func (x *RevokeEnvironmentResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RevokeEnvironmentResponse.ProtoReflect.Descriptor instead. +func (*RevokeEnvironmentResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{25} +} + +func (x *RevokeEnvironmentResponse) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type MigrateEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedSchemaVersion *uint32 `protobuf:"varint,3,opt,name=expected_schema_version,json=expectedSchemaVersion,proto3,oneof" json:"expected_schema_version,omitempty"` + ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MigrateEnvironmentRequest) Reset() { + *x = MigrateEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MigrateEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MigrateEnvironmentRequest) ProtoMessage() {} + +func (x *MigrateEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MigrateEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*MigrateEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{26} +} + +func (x *MigrateEnvironmentRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *MigrateEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *MigrateEnvironmentRequest) GetExpectedSchemaVersion() uint32 { + if x != nil && x.ExpectedSchemaVersion != nil { + return *x.ExpectedSchemaVersion + } + return 0 +} + +func (x *MigrateEnvironmentRequest) GetExpectedPodUid() string { + if x != nil { + return x.ExpectedPodUid + } + return "" +} + +func (x *MigrateEnvironmentRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid + } + return "" +} + +func (x *MigrateEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type FileRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + Operation FileOperation `protobuf:"varint,2,opt,name=operation,proto3,enum=mecatl.execution.v1.FileOperation" json:"operation,omitempty"` + Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` + Destination string `protobuf:"bytes,4,opt,name=destination,proto3" json:"destination,omitempty"` + Pattern string `protobuf:"bytes,5,opt,name=pattern,proto3" json:"pattern,omitempty"` + Data []byte `protobuf:"bytes,6,opt,name=data,proto3" json:"data,omitempty"` + Version []byte `protobuf:"bytes,7,opt,name=version,proto3" json:"version,omitempty"` + Limit int32 `protobuf:"varint,8,opt,name=limit,proto3" json:"limit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileRequest) Reset() { + *x = FileRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileRequest) ProtoMessage() {} + +func (x *FileRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileRequest.ProtoReflect.Descriptor instead. +func (*FileRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{27} +} + +func (x *FileRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *FileRequest) GetOperation() FileOperation { + if x != nil { + return x.Operation + } + return FileOperation_FILE_OPERATION_UNSPECIFIED +} + +func (x *FileRequest) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *FileRequest) GetDestination() string { + if x != nil { + return x.Destination + } + return "" +} + +func (x *FileRequest) GetPattern() string { + if x != nil { + return x.Pattern + } + return "" +} + +func (x *FileRequest) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *FileRequest) GetVersion() []byte { + if x != nil { + return x.Version + } + return nil +} + +func (x *FileRequest) GetLimit() int32 { + if x != nil { + return x.Limit + } + return 0 +} + +type FileInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + Mode uint32 `protobuf:"varint,3,opt,name=mode,proto3" json:"mode,omitempty"` + ModTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=mod_time,json=modTime,proto3" json:"mod_time,omitempty"` + IsDir bool `protobuf:"varint,5,opt,name=is_dir,json=isDir,proto3" json:"is_dir,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileInfo) Reset() { + *x = FileInfo{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileInfo) ProtoMessage() {} + +func (x *FileInfo) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[28] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileInfo.ProtoReflect.Descriptor instead. +func (*FileInfo) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{28} +} + +func (x *FileInfo) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *FileInfo) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *FileInfo) GetMode() uint32 { + if x != nil { + return x.Mode + } + return 0 +} + +func (x *FileInfo) GetModTime() *timestamppb.Timestamp { + if x != nil { + return x.ModTime + } + return nil +} + +func (x *FileInfo) GetIsDir() bool { + if x != nil { + return x.IsDir + } + return false +} + +type GrepMatch struct { + state protoimpl.MessageState `protogen:"open.v1"` + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + Line int32 `protobuf:"varint,2,opt,name=line,proto3" json:"line,omitempty"` + Text string `protobuf:"bytes,3,opt,name=text,proto3" json:"text,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GrepMatch) Reset() { + *x = GrepMatch{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GrepMatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GrepMatch) ProtoMessage() {} + +func (x *GrepMatch) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[29] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GrepMatch.ProtoReflect.Descriptor instead. +func (*GrepMatch) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{29} +} + +func (x *GrepMatch) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *GrepMatch) GetLine() int32 { + if x != nil { + return x.Line + } + return 0 +} + +func (x *GrepMatch) GetText() string { + if x != nil { + return x.Text + } + return "" +} + +type FileResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + Version []byte `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Info *FileInfo `protobuf:"bytes,3,opt,name=info,proto3" json:"info,omitempty"` + Entries []*FileInfo `protobuf:"bytes,4,rep,name=entries,proto3" json:"entries,omitempty"` + Paths []string `protobuf:"bytes,5,rep,name=paths,proto3" json:"paths,omitempty"` + Matches []*GrepMatch `protobuf:"bytes,6,rep,name=matches,proto3" json:"matches,omitempty"` + AuthorityTarget string `protobuf:"bytes,7,opt,name=authority_target,json=authorityTarget,proto3" json:"authority_target,omitempty"` + AuthorityWorkspace string `protobuf:"bytes,8,opt,name=authority_workspace,json=authorityWorkspace,proto3" json:"authority_workspace,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileResponse) Reset() { + *x = FileResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileResponse) ProtoMessage() {} + +func (x *FileResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[30] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileResponse.ProtoReflect.Descriptor instead. +func (*FileResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{30} +} + +func (x *FileResponse) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *FileResponse) GetVersion() []byte { + if x != nil { + return x.Version + } + return nil +} + +func (x *FileResponse) GetInfo() *FileInfo { + if x != nil { + return x.Info + } + return nil +} + +func (x *FileResponse) GetEntries() []*FileInfo { + if x != nil { + return x.Entries + } + return nil +} + +func (x *FileResponse) GetPaths() []string { + if x != nil { + return x.Paths + } + return nil +} + +func (x *FileResponse) GetMatches() []*GrepMatch { + if x != nil { + return x.Matches + } + return nil +} + +func (x *FileResponse) GetAuthorityTarget() string { + if x != nil { + return x.AuthorityTarget + } + return "" +} + +func (x *FileResponse) GetAuthorityWorkspace() string { + if x != nil { + return x.AuthorityWorkspace + } + return "" +} + +type CommandStartRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + Command string `protobuf:"bytes,2,opt,name=command,proto3" json:"command,omitempty"` + TimeoutMillis int64 `protobuf:"varint,3,opt,name=timeout_millis,json=timeoutMillis,proto3" json:"timeout_millis,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommandStartRequest) Reset() { + *x = CommandStartRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommandStartRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommandStartRequest) ProtoMessage() {} + +func (x *CommandStartRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[31] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommandStartRequest.ProtoReflect.Descriptor instead. +func (*CommandStartRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{31} +} + +func (x *CommandStartRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *CommandStartRequest) GetCommand() string { + if x != nil { + return x.Command + } + return "" +} + +func (x *CommandStartRequest) GetTimeoutMillis() int64 { + if x != nil { + return x.TimeoutMillis + } + return 0 +} + +type CommandStartResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"` + State CommandState `protobuf:"varint,2,opt,name=state,proto3,enum=mecatl.execution.v1.CommandState" json:"state,omitempty"` + Result *CommandStatusResponse `protobuf:"bytes,3,opt,name=result,proto3" json:"result,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommandStartResponse) Reset() { + *x = CommandStartResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommandStartResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommandStartResponse) ProtoMessage() {} + +func (x *CommandStartResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[32] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommandStartResponse.ProtoReflect.Descriptor instead. +func (*CommandStartResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{32} +} + +func (x *CommandStartResponse) GetCommandId() string { + if x != nil { + return x.CommandId + } + return "" +} + +func (x *CommandStartResponse) GetState() CommandState { + if x != nil { + return x.State + } + return CommandState_COMMAND_STATE_UNSPECIFIED +} + +func (x *CommandStartResponse) GetResult() *CommandStatusResponse { + if x != nil { + return x.Result + } + return nil +} + +type CommandQueryRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + CommandId string `protobuf:"bytes,2,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"` + Offset int64 `protobuf:"varint,3,opt,name=offset,proto3" json:"offset,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommandQueryRequest) Reset() { + *x = CommandQueryRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommandQueryRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommandQueryRequest) ProtoMessage() {} + +func (x *CommandQueryRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[33] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommandQueryRequest.ProtoReflect.Descriptor instead. +func (*CommandQueryRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{33} +} + +func (x *CommandQueryRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *CommandQueryRequest) GetCommandId() string { + if x != nil { + return x.CommandId + } + return "" +} + +func (x *CommandQueryRequest) GetOffset() int64 { + if x != nil { + return x.Offset + } + return 0 +} + +type CommandStatusResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"` + State CommandState `protobuf:"varint,2,opt,name=state,proto3,enum=mecatl.execution.v1.CommandState" json:"state,omitempty"` + ExitCode int32 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` + Stdout []byte `protobuf:"bytes,4,opt,name=stdout,proto3" json:"stdout,omitempty"` + Stderr []byte `protobuf:"bytes,5,opt,name=stderr,proto3" json:"stderr,omitempty"` + NextOffset int64 `protobuf:"varint,6,opt,name=next_offset,json=nextOffset,proto3" json:"next_offset,omitempty"` + Truncated bool `protobuf:"varint,7,opt,name=truncated,proto3" json:"truncated,omitempty"` + TerminalReceipt string `protobuf:"bytes,8,opt,name=terminal_receipt,json=terminalReceipt,proto3" json:"terminal_receipt,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommandStatusResponse) Reset() { + *x = CommandStatusResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommandStatusResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommandStatusResponse) ProtoMessage() {} + +func (x *CommandStatusResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[34] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommandStatusResponse.ProtoReflect.Descriptor instead. +func (*CommandStatusResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{34} +} + +func (x *CommandStatusResponse) GetCommandId() string { + if x != nil { + return x.CommandId + } + return "" +} + +func (x *CommandStatusResponse) GetState() CommandState { + if x != nil { + return x.State + } + return CommandState_COMMAND_STATE_UNSPECIFIED +} + +func (x *CommandStatusResponse) GetExitCode() int32 { + if x != nil { + return x.ExitCode + } + return 0 +} + +func (x *CommandStatusResponse) GetStdout() []byte { + if x != nil { + return x.Stdout + } + return nil +} + +func (x *CommandStatusResponse) GetStderr() []byte { + if x != nil { + return x.Stderr + } + return nil +} + +func (x *CommandStatusResponse) GetNextOffset() int64 { + if x != nil { + return x.NextOffset + } + return 0 +} + +func (x *CommandStatusResponse) GetTruncated() bool { + if x != nil { + return x.Truncated + } + return false +} + +func (x *CommandStatusResponse) GetTerminalReceipt() string { + if x != nil { + return x.TerminalReceipt + } + return "" +} + +// ErrorDetail is a stable machine-readable classification. Status messages are +// intentionally generic so backend/Kubernetes details never cross the boundary. +type ErrorDetail struct { + state protoimpl.MessageState `protogen:"open.v1"` + Code string `protobuf:"bytes,1,opt,name=code,proto3" json:"code,omitempty"` + Retryable bool `protobuf:"varint,2,opt,name=retryable,proto3" json:"retryable,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ErrorDetail) Reset() { + *x = ErrorDetail{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ErrorDetail) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ErrorDetail) ProtoMessage() {} + +func (x *ErrorDetail) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[35] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ErrorDetail.ProtoReflect.Descriptor instead. +func (*ErrorDetail) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{35} +} + +func (x *ErrorDetail) GetCode() string { + if x != nil { + return x.Code + } + return "" +} + +func (x *ErrorDetail) GetRetryable() bool { + if x != nil { + return x.Retryable + } + return false +} + +var File_mecatl_execution_v1_execution_proto protoreflect.FileDescriptor + +var file_mecatl_execution_v1_execution_proto_rawDesc = string([]byte{ + 0x0a, 0x23, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2f, 0x76, 0x31, 0x2f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x13, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, + 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3c, 0x0a, 0x0e, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, + 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, + 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x39, 0x0a, 0x05, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x12, + 0x16, 0x0a, 0x06, 0x69, 0x73, 0x73, 0x75, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x69, 0x73, 0x73, 0x75, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x22, 0xb1, 0x02, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, + 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, + 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, + 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, + 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x12, + 0x19, 0x0a, 0x08, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x49, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, + 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x32, 0x0a, 0x16, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x80, 0x02, 0x0a, 0x17, 0x56, 0x61, + 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x12, + 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, + 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, + 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x6d, + 0x61, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x42, 0x79, 0x74, 0x65, + 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x6d, 0x61, + 0x78, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x3d, 0x0a, + 0x1b, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x64, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x18, 0x6d, 0x61, 0x78, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x44, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, 0xa8, 0x01, 0x0a, + 0x18, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, + 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, + 0x6c, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x95, 0x02, 0x0a, 0x19, 0x45, 0x6e, 0x73, 0x75, + 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, + 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, + 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, + 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x44, + 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, + 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0e, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x45, 0x78, 0x70, 0x69, 0x72, + 0x65, 0x73, 0x41, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, + 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0x73, 0x0a, 0x18, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x75, + 0x72, 0x70, 0x6f, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x75, 0x72, + 0x70, 0x6f, 0x73, 0x65, 0x22, 0x95, 0x02, 0x0a, 0x19, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, + 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, + 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, + 0x72, 0x65, 0x61, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x44, 0x0a, 0x10, 0x67, + 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x61, 0x74, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x0e, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, + 0x74, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, + 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x84, 0x02, 0x0a, + 0x11, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, + 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, + 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, + 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x74, 0x6c, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, + 0x69, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x74, 0x6c, 0x4d, 0x69, 0x6c, + 0x6c, 0x69, 0x73, 0x22, 0xde, 0x02, 0x0a, 0x0f, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, + 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, + 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, + 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x5f, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x49, + 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x74, + 0x6c, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, + 0x74, 0x74, 0x6c, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, 0x61, + 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc1, 0x02, 0x0a, 0x11, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, + 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, + 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, + 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6c, 0x61, + 0x69, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, 0x61, + 0x69, 0x6d, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x29, 0x0a, + 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbc, 0x02, 0x0a, 0x10, 0x52, 0x75, 0x6e, + 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, + 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, + 0x67, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6c, + 0x61, 0x69, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, + 0x61, 0x69, 0x6d, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x29, 0x0a, 0x10, 0x67, + 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x39, 0x0a, 0x0a, + 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x78, + 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, 0x22, 0xd5, 0x01, 0x0a, 0x18, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, + 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, + 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, + 0x97, 0x02, 0x0a, 0x17, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, + 0x12, 0x34, 0x0a, 0x16, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x14, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x69, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x65, 0x0a, 0x1c, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x22, 0xcb, 0x01, 0x0a, 0x1b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, + 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, + 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, + 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x22, 0xc9, + 0x02, 0x0a, 0x0f, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, + 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x64, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x62, 0x69, 0x6e, 0x64, + 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x39, 0x0a, + 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, + 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, + 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x22, 0x5e, 0x0a, 0x1c, 0x4c, 0x69, + 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x07, 0x69, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x52, 0x07, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x58, 0x0a, 0x17, 0x52, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x22, 0xc4, 0x02, 0x0a, 0x18, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, + 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, + 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x78, + 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, + 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, + 0x70, 0x6f, 0x63, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, + 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xc2, 0x02, 0x0a, 0x16, + 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, + 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, + 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, + 0x38, 0x0a, 0x18, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x16, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, + 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, + 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, + 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, + 0x22, 0xc5, 0x02, 0x0a, 0x19, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, + 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, + 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x78, 0x70, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x70, + 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, 0x70, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x70, 0x6f, 0x63, + 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, + 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, + 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, + 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, + 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, + 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x49, 0x64, 0x22, 0xf2, 0x01, 0x0a, 0x18, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, + 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x3a, 0x0a, 0x19, 0x65, 0x78, 0x70, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x5f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x17, 0x65, 0x78, 0x70, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x47, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x46, 0x0a, 0x19, 0x52, 0x65, 0x76, 0x6f, 0x6b, + 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, + 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0xe4, 0x02, 0x0a, 0x19, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, + 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, + 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x3b, 0x0a, 0x17, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x00, 0x52, 0x15, 0x65, 0x78, 0x70, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x88, 0x01, 0x01, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, + 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, + 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, + 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xa2, 0x02, 0x0a, 0x0b, 0x46, 0x69, 0x6c, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x40, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, + 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x20, 0x0a, 0x0b, 0x64, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, + 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x18, 0x0a, 0x07, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x94, 0x01, 0x0a, 0x08, + 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, + 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x6d, 0x6f, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x69, + 0x73, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x69, 0x73, 0x44, + 0x69, 0x72, 0x22, 0x47, 0x0a, 0x09, 0x47, 0x72, 0x65, 0x70, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, + 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x78, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, 0x22, 0xd4, 0x02, 0x0a, 0x0c, + 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, + 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x04, 0x69, 0x6e, + 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, + 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, + 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x65, + 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, + 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x12, 0x38, 0x0a, 0x07, + 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, 0x65, 0x70, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x6d, + 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, + 0x69, 0x74, 0x79, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x12, 0x2f, 0x0a, 0x13, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x77, + 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, + 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x22, 0x95, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x6d, + 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x74, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, 0xb2, 0x01, 0x0a, 0x14, 0x43, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x49, 0x64, 0x12, 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x42, 0x0a, 0x06, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, + 0x8b, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, + 0x64, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0xa6, 0x02, + 0x0a, 0x15, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, + 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, + 0x1b, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, + 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x74, + 0x64, 0x6f, 0x75, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, 0x12, 0x1f, 0x0a, 0x0b, + 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0a, 0x6e, 0x65, 0x78, 0x74, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, + 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x74, + 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x52, + 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, 0x3f, 0x0a, 0x0b, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x44, + 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x74, + 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, + 0x74, 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x2a, 0xd8, 0x02, 0x0a, 0x0d, 0x46, 0x69, 0x6c, 0x65, + 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x49, 0x4c, + 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, + 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x41, 0x44, + 0x10, 0x01, 0x12, 0x24, 0x0a, 0x20, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, + 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x53, 0x4f, 0x4c, 0x56, 0x45, 0x5f, 0x41, 0x55, 0x54, + 0x48, 0x4f, 0x52, 0x49, 0x54, 0x59, 0x10, 0x02, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, + 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x10, + 0x03, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, 0x04, 0x12, 0x1a, 0x0a, 0x16, + 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, + 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x10, 0x05, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, + 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, + 0x06, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x10, 0x07, 0x12, 0x19, 0x0a, 0x15, + 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, + 0x45, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x08, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, + 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x43, 0x4f, 0x50, 0x59, 0x10, 0x09, + 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, + 0x4f, 0x4e, 0x5f, 0x47, 0x4c, 0x4f, 0x42, 0x10, 0x0a, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, + 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x47, 0x52, 0x45, 0x50, + 0x10, 0x0b, 0x2a, 0xbd, 0x01, 0x0a, 0x0c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, + 0x41, 0x54, 0x45, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x1b, 0x0a, + 0x17, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, + 0x55, 0x43, 0x43, 0x45, 0x45, 0x44, 0x45, 0x44, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, + 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, + 0x45, 0x44, 0x10, 0x03, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, + 0x04, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x45, 0x5f, 0x46, 0x45, 0x4e, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, + 0x10, 0x05, 0x32, 0xe4, 0x12, 0x0a, 0x18, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, + 0x6c, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, + 0x6c, 0x65, 0x12, 0x2b, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, + 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, + 0x11, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x72, 0x0a, 0x11, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x74, 0x74, + 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x74, 0x74, 0x61, + 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5b, 0x0a, 0x0a, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, + 0x52, 0x75, 0x6e, 0x12, 0x26, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, + 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x57, 0x0a, 0x08, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, 0x12, 0x24, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6c, + 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x0a, 0x52, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x75, 0x6e, 0x12, 0x26, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x58, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, + 0x6d, 0x69, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2d, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x73, 0x0a, 0x10, + 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, + 0x12, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, 0x75, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, + 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x5f, 0x0a, 0x16, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x12, 0x5f, 0x0a, 0x16, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x12, 0x5e, 0x0a, 0x15, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x12, 0x7b, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x30, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x58, 0x0a, 0x10, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, + 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5a, 0x0a, 0x11, 0x52, 0x65, + 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, + 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x56, 0x0a, 0x0f, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x12, 0x2b, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5c, + 0x0a, 0x12, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x76, + 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x68, 0x0a, 0x18, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x34, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5c, 0x0a, 0x12, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, + 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x12, 0x72, 0x0a, 0x11, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x05, 0x46, 0x69, 0x6c, 0x65, + 0x73, 0x12, 0x20, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x0c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x29, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x43, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, + 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0xe4, 0x01, 0x0a, 0x17, 0x63, 0x6f, + 0x6d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x4b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x6c, 0x6f, 0x6b, 0x2f, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x73, 0x2f, 0x67, 0x65, + 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x31, 0x3b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4d, 0x45, 0x58, 0xaa, 0x02, 0x13, 0x4d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x56, 0x31, + 0xca, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x1f, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x5c, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x15, 0x4d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x3a, 0x3a, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x3a, 0x56, 0x31, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +}) + +var ( + file_mecatl_execution_v1_execution_proto_rawDescOnce sync.Once + file_mecatl_execution_v1_execution_proto_rawDescData []byte +) + +func file_mecatl_execution_v1_execution_proto_rawDescGZIP() []byte { + file_mecatl_execution_v1_execution_proto_rawDescOnce.Do(func() { + file_mecatl_execution_v1_execution_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mecatl_execution_v1_execution_proto_rawDesc), len(file_mecatl_execution_v1_execution_proto_rawDesc))) + }) + return file_mecatl_execution_v1_execution_proto_rawDescData +} + +var file_mecatl_execution_v1_execution_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_mecatl_execution_v1_execution_proto_msgTypes = make([]protoimpl.MessageInfo, 36) +var file_mecatl_execution_v1_execution_proto_goTypes = []any{ + (FileOperation)(0), // 0: mecatl.execution.v1.FileOperation + (CommandState)(0), // 1: mecatl.execution.v1.CommandState + (*EnvironmentRef)(nil), // 2: mecatl.execution.v1.EnvironmentRef + (*Owner)(nil), // 3: mecatl.execution.v1.Owner + (*RequestContext)(nil), // 4: mecatl.execution.v1.RequestContext + (*ValidateProfileRequest)(nil), // 5: mecatl.execution.v1.ValidateProfileRequest + (*ValidateProfileResponse)(nil), // 6: mecatl.execution.v1.ValidateProfileResponse + (*EnsureEnvironmentRequest)(nil), // 7: mecatl.execution.v1.EnsureEnvironmentRequest + (*EnsureEnvironmentResponse)(nil), // 8: mecatl.execution.v1.EnsureEnvironmentResponse + (*AttachEnvironmentRequest)(nil), // 9: mecatl.execution.v1.AttachEnvironmentRequest + (*AttachEnvironmentResponse)(nil), // 10: mecatl.execution.v1.AttachEnvironmentResponse + (*AcquireRunRequest)(nil), // 11: mecatl.execution.v1.AcquireRunRequest + (*RenewRunRequest)(nil), // 12: mecatl.execution.v1.RenewRunRequest + (*ReleaseRunRequest)(nil), // 13: mecatl.execution.v1.ReleaseRunRequest + (*RunClaimResponse)(nil), // 14: mecatl.execution.v1.RunClaimResponse + (*ReferenceMutationRequest)(nil), // 15: mecatl.execution.v1.ReferenceMutationRequest + (*ReserveSuccessorRequest)(nil), // 16: mecatl.execution.v1.ReserveSuccessorRequest + (*ReferenceReservationResponse)(nil), // 17: mecatl.execution.v1.ReferenceReservationResponse + (*ListReferenceIntentsRequest)(nil), // 18: mecatl.execution.v1.ListReferenceIntentsRequest + (*ReferenceIntent)(nil), // 19: mecatl.execution.v1.ReferenceIntent + (*ListReferenceIntentsResponse)(nil), // 20: mecatl.execution.v1.ListReferenceIntentsResponse + (*ReleaseReferenceRequest)(nil), // 21: mecatl.execution.v1.ReleaseReferenceRequest + (*RetireEnvironmentRequest)(nil), // 22: mecatl.execution.v1.RetireEnvironmentRequest + (*ReplaceExecutorRequest)(nil), // 23: mecatl.execution.v1.ReplaceExecutorRequest + (*RecoverEnvironmentRequest)(nil), // 24: mecatl.execution.v1.RecoverEnvironmentRequest + (*DeleteRetiredEnvironmentRequest)(nil), // 25: mecatl.execution.v1.DeleteRetiredEnvironmentRequest + (*RevokeEnvironmentRequest)(nil), // 26: mecatl.execution.v1.RevokeEnvironmentRequest + (*RevokeEnvironmentResponse)(nil), // 27: mecatl.execution.v1.RevokeEnvironmentResponse + (*MigrateEnvironmentRequest)(nil), // 28: mecatl.execution.v1.MigrateEnvironmentRequest + (*FileRequest)(nil), // 29: mecatl.execution.v1.FileRequest + (*FileInfo)(nil), // 30: mecatl.execution.v1.FileInfo + (*GrepMatch)(nil), // 31: mecatl.execution.v1.GrepMatch + (*FileResponse)(nil), // 32: mecatl.execution.v1.FileResponse + (*CommandStartRequest)(nil), // 33: mecatl.execution.v1.CommandStartRequest + (*CommandStartResponse)(nil), // 34: mecatl.execution.v1.CommandStartResponse + (*CommandQueryRequest)(nil), // 35: mecatl.execution.v1.CommandQueryRequest + (*CommandStatusResponse)(nil), // 36: mecatl.execution.v1.CommandStatusResponse + (*ErrorDetail)(nil), // 37: mecatl.execution.v1.ErrorDetail + (*timestamppb.Timestamp)(nil), // 38: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 39: google.protobuf.Empty +} +var file_mecatl_execution_v1_execution_proto_depIdxs = []int32{ + 2, // 0: mecatl.execution.v1.RequestContext.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 1: mecatl.execution.v1.RequestContext.owner:type_name -> mecatl.execution.v1.Owner + 3, // 2: mecatl.execution.v1.EnsureEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 3: mecatl.execution.v1.EnsureEnvironmentResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 38, // 4: mecatl.execution.v1.EnsureEnvironmentResponse.grant_expires_at:type_name -> google.protobuf.Timestamp + 4, // 5: mecatl.execution.v1.AttachEnvironmentRequest.context:type_name -> mecatl.execution.v1.RequestContext + 2, // 6: mecatl.execution.v1.AttachEnvironmentResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 38, // 7: mecatl.execution.v1.AttachEnvironmentResponse.grant_expires_at:type_name -> google.protobuf.Timestamp + 2, // 8: mecatl.execution.v1.AcquireRunRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 9: mecatl.execution.v1.AcquireRunRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 10: mecatl.execution.v1.RenewRunRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 11: mecatl.execution.v1.RenewRunRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 12: mecatl.execution.v1.ReleaseRunRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 13: mecatl.execution.v1.ReleaseRunRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 14: mecatl.execution.v1.RunClaimResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 38, // 15: mecatl.execution.v1.RunClaimResponse.expires_at:type_name -> google.protobuf.Timestamp + 2, // 16: mecatl.execution.v1.ReferenceMutationRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 17: mecatl.execution.v1.ReferenceMutationRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 18: mecatl.execution.v1.ReserveSuccessorRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 19: mecatl.execution.v1.ReserveSuccessorRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 20: mecatl.execution.v1.ReferenceReservationResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 21: mecatl.execution.v1.ListReferenceIntentsRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 22: mecatl.execution.v1.ListReferenceIntentsRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 2, // 23: mecatl.execution.v1.ReferenceIntent.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 38, // 24: mecatl.execution.v1.ReferenceIntent.created_at:type_name -> google.protobuf.Timestamp + 3, // 25: mecatl.execution.v1.ReferenceIntent.owner:type_name -> mecatl.execution.v1.Owner + 19, // 26: mecatl.execution.v1.ListReferenceIntentsResponse.intents:type_name -> mecatl.execution.v1.ReferenceIntent + 4, // 27: mecatl.execution.v1.ReleaseReferenceRequest.context:type_name -> mecatl.execution.v1.RequestContext + 2, // 28: mecatl.execution.v1.RetireEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 29: mecatl.execution.v1.RetireEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 30: mecatl.execution.v1.ReplaceExecutorRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 31: mecatl.execution.v1.ReplaceExecutorRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 32: mecatl.execution.v1.RecoverEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 33: mecatl.execution.v1.RecoverEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 34: mecatl.execution.v1.DeleteRetiredEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 35: mecatl.execution.v1.DeleteRetiredEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 36: mecatl.execution.v1.RevokeEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 37: mecatl.execution.v1.RevokeEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 38: mecatl.execution.v1.MigrateEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 39: mecatl.execution.v1.MigrateEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 4, // 40: mecatl.execution.v1.FileRequest.context:type_name -> mecatl.execution.v1.RequestContext + 0, // 41: mecatl.execution.v1.FileRequest.operation:type_name -> mecatl.execution.v1.FileOperation + 38, // 42: mecatl.execution.v1.FileInfo.mod_time:type_name -> google.protobuf.Timestamp + 30, // 43: mecatl.execution.v1.FileResponse.info:type_name -> mecatl.execution.v1.FileInfo + 30, // 44: mecatl.execution.v1.FileResponse.entries:type_name -> mecatl.execution.v1.FileInfo + 31, // 45: mecatl.execution.v1.FileResponse.matches:type_name -> mecatl.execution.v1.GrepMatch + 4, // 46: mecatl.execution.v1.CommandStartRequest.context:type_name -> mecatl.execution.v1.RequestContext + 1, // 47: mecatl.execution.v1.CommandStartResponse.state:type_name -> mecatl.execution.v1.CommandState + 36, // 48: mecatl.execution.v1.CommandStartResponse.result:type_name -> mecatl.execution.v1.CommandStatusResponse + 4, // 49: mecatl.execution.v1.CommandQueryRequest.context:type_name -> mecatl.execution.v1.RequestContext + 1, // 50: mecatl.execution.v1.CommandStatusResponse.state:type_name -> mecatl.execution.v1.CommandState + 5, // 51: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:input_type -> mecatl.execution.v1.ValidateProfileRequest + 7, // 52: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:input_type -> mecatl.execution.v1.EnsureEnvironmentRequest + 9, // 53: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:input_type -> mecatl.execution.v1.AttachEnvironmentRequest + 11, // 54: mecatl.execution.v1.ExecutionProviderService.AcquireRun:input_type -> mecatl.execution.v1.AcquireRunRequest + 12, // 55: mecatl.execution.v1.ExecutionProviderService.RenewRun:input_type -> mecatl.execution.v1.RenewRunRequest + 13, // 56: mecatl.execution.v1.ExecutionProviderService.ReleaseRun:input_type -> mecatl.execution.v1.ReleaseRunRequest + 15, // 57: mecatl.execution.v1.ExecutionProviderService.CommitReference:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 15, // 58: mecatl.execution.v1.ExecutionProviderService.AbortReference:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 16, // 59: mecatl.execution.v1.ExecutionProviderService.ReserveSuccessor:input_type -> mecatl.execution.v1.ReserveSuccessorRequest + 15, // 60: mecatl.execution.v1.ExecutionProviderService.PrepareReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 15, // 61: mecatl.execution.v1.ExecutionProviderService.ConfirmReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 15, // 62: mecatl.execution.v1.ExecutionProviderService.CancelReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 18, // 63: mecatl.execution.v1.ExecutionProviderService.ListReferenceIntents:input_type -> mecatl.execution.v1.ListReferenceIntentsRequest + 21, // 64: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:input_type -> mecatl.execution.v1.ReleaseReferenceRequest + 22, // 65: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:input_type -> mecatl.execution.v1.RetireEnvironmentRequest + 23, // 66: mecatl.execution.v1.ExecutionProviderService.ReplaceExecutor:input_type -> mecatl.execution.v1.ReplaceExecutorRequest + 24, // 67: mecatl.execution.v1.ExecutionProviderService.RecoverEnvironment:input_type -> mecatl.execution.v1.RecoverEnvironmentRequest + 25, // 68: mecatl.execution.v1.ExecutionProviderService.DeleteRetiredEnvironment:input_type -> mecatl.execution.v1.DeleteRetiredEnvironmentRequest + 28, // 69: mecatl.execution.v1.ExecutionProviderService.MigrateEnvironment:input_type -> mecatl.execution.v1.MigrateEnvironmentRequest + 26, // 70: mecatl.execution.v1.ExecutionProviderService.RevokeEnvironment:input_type -> mecatl.execution.v1.RevokeEnvironmentRequest + 29, // 71: mecatl.execution.v1.ExecutionProviderService.Files:input_type -> mecatl.execution.v1.FileRequest + 33, // 72: mecatl.execution.v1.ExecutionProviderService.StartCommand:input_type -> mecatl.execution.v1.CommandStartRequest + 35, // 73: mecatl.execution.v1.ExecutionProviderService.CommandStatus:input_type -> mecatl.execution.v1.CommandQueryRequest + 35, // 74: mecatl.execution.v1.ExecutionProviderService.CancelCommand:input_type -> mecatl.execution.v1.CommandQueryRequest + 6, // 75: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:output_type -> mecatl.execution.v1.ValidateProfileResponse + 8, // 76: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:output_type -> mecatl.execution.v1.EnsureEnvironmentResponse + 10, // 77: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:output_type -> mecatl.execution.v1.AttachEnvironmentResponse + 14, // 78: mecatl.execution.v1.ExecutionProviderService.AcquireRun:output_type -> mecatl.execution.v1.RunClaimResponse + 14, // 79: mecatl.execution.v1.ExecutionProviderService.RenewRun:output_type -> mecatl.execution.v1.RunClaimResponse + 39, // 80: mecatl.execution.v1.ExecutionProviderService.ReleaseRun:output_type -> google.protobuf.Empty + 39, // 81: mecatl.execution.v1.ExecutionProviderService.CommitReference:output_type -> google.protobuf.Empty + 39, // 82: mecatl.execution.v1.ExecutionProviderService.AbortReference:output_type -> google.protobuf.Empty + 17, // 83: mecatl.execution.v1.ExecutionProviderService.ReserveSuccessor:output_type -> mecatl.execution.v1.ReferenceReservationResponse + 39, // 84: mecatl.execution.v1.ExecutionProviderService.PrepareReferenceDelete:output_type -> google.protobuf.Empty + 39, // 85: mecatl.execution.v1.ExecutionProviderService.ConfirmReferenceDelete:output_type -> google.protobuf.Empty + 39, // 86: mecatl.execution.v1.ExecutionProviderService.CancelReferenceDelete:output_type -> google.protobuf.Empty + 20, // 87: mecatl.execution.v1.ExecutionProviderService.ListReferenceIntents:output_type -> mecatl.execution.v1.ListReferenceIntentsResponse + 39, // 88: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:output_type -> google.protobuf.Empty + 39, // 89: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:output_type -> google.protobuf.Empty + 39, // 90: mecatl.execution.v1.ExecutionProviderService.ReplaceExecutor:output_type -> google.protobuf.Empty + 39, // 91: mecatl.execution.v1.ExecutionProviderService.RecoverEnvironment:output_type -> google.protobuf.Empty + 39, // 92: mecatl.execution.v1.ExecutionProviderService.DeleteRetiredEnvironment:output_type -> google.protobuf.Empty + 39, // 93: mecatl.execution.v1.ExecutionProviderService.MigrateEnvironment:output_type -> google.protobuf.Empty + 27, // 94: mecatl.execution.v1.ExecutionProviderService.RevokeEnvironment:output_type -> mecatl.execution.v1.RevokeEnvironmentResponse + 32, // 95: mecatl.execution.v1.ExecutionProviderService.Files:output_type -> mecatl.execution.v1.FileResponse + 34, // 96: mecatl.execution.v1.ExecutionProviderService.StartCommand:output_type -> mecatl.execution.v1.CommandStartResponse + 36, // 97: mecatl.execution.v1.ExecutionProviderService.CommandStatus:output_type -> mecatl.execution.v1.CommandStatusResponse + 36, // 98: mecatl.execution.v1.ExecutionProviderService.CancelCommand:output_type -> mecatl.execution.v1.CommandStatusResponse + 75, // [75:99] is the sub-list for method output_type + 51, // [51:75] is the sub-list for method input_type + 51, // [51:51] is the sub-list for extension type_name + 51, // [51:51] is the sub-list for extension extendee + 0, // [0:51] is the sub-list for field type_name +} + +func init() { file_mecatl_execution_v1_execution_proto_init() } +func file_mecatl_execution_v1_execution_proto_init() { + if File_mecatl_execution_v1_execution_proto != nil { + return + } + file_mecatl_execution_v1_execution_proto_msgTypes[26].OneofWrappers = []any{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_mecatl_execution_v1_execution_proto_rawDesc), len(file_mecatl_execution_v1_execution_proto_rawDesc)), + NumEnums: 2, + NumMessages: 36, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_mecatl_execution_v1_execution_proto_goTypes, + DependencyIndexes: file_mecatl_execution_v1_execution_proto_depIdxs, + EnumInfos: file_mecatl_execution_v1_execution_proto_enumTypes, + MessageInfos: file_mecatl_execution_v1_execution_proto_msgTypes, + }.Build() + File_mecatl_execution_v1_execution_proto = out.File + file_mecatl_execution_v1_execution_proto_goTypes = nil + file_mecatl_execution_v1_execution_proto_depIdxs = nil +} diff --git a/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go b/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go new file mode 100644 index 0000000000..abfd835935 --- /dev/null +++ b/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go @@ -0,0 +1,1003 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: LicenseRef-Stacklok-Proprietary + +// Private, mutually authenticated execution-provider protocol. This is not part +// of the public Harness API. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc (unknown) +// source: mecatl/execution/v1/execution.proto + +package executionv1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + emptypb "google.golang.org/protobuf/types/known/emptypb" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + ExecutionProviderService_ValidateProfile_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ValidateProfile" + ExecutionProviderService_EnsureEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/EnsureEnvironment" + ExecutionProviderService_AttachEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/AttachEnvironment" + ExecutionProviderService_AcquireRun_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/AcquireRun" + ExecutionProviderService_RenewRun_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RenewRun" + ExecutionProviderService_ReleaseRun_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReleaseRun" + ExecutionProviderService_CommitReference_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CommitReference" + ExecutionProviderService_AbortReference_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/AbortReference" + ExecutionProviderService_ReserveSuccessor_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReserveSuccessor" + ExecutionProviderService_PrepareReferenceDelete_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/PrepareReferenceDelete" + ExecutionProviderService_ConfirmReferenceDelete_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ConfirmReferenceDelete" + ExecutionProviderService_CancelReferenceDelete_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CancelReferenceDelete" + ExecutionProviderService_ListReferenceIntents_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ListReferenceIntents" + ExecutionProviderService_ReleaseReference_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReleaseReference" + ExecutionProviderService_RetireEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RetireEnvironment" + ExecutionProviderService_ReplaceExecutor_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReplaceExecutor" + ExecutionProviderService_RecoverEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RecoverEnvironment" + ExecutionProviderService_DeleteRetiredEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/DeleteRetiredEnvironment" + ExecutionProviderService_MigrateEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/MigrateEnvironment" + ExecutionProviderService_RevokeEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RevokeEnvironment" + ExecutionProviderService_Files_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/Files" + ExecutionProviderService_StartCommand_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/StartCommand" + ExecutionProviderService_CommandStatus_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CommandStatus" + ExecutionProviderService_CancelCommand_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CancelCommand" +) + +// ExecutionProviderServiceClient is the client API for ExecutionProviderService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ExecutionProviderServiceClient interface { + ValidateProfile(ctx context.Context, in *ValidateProfileRequest, opts ...grpc.CallOption) (*ValidateProfileResponse, error) + EnsureEnvironment(ctx context.Context, in *EnsureEnvironmentRequest, opts ...grpc.CallOption) (*EnsureEnvironmentResponse, error) + AttachEnvironment(ctx context.Context, in *AttachEnvironmentRequest, opts ...grpc.CallOption) (*AttachEnvironmentResponse, error) + AcquireRun(ctx context.Context, in *AcquireRunRequest, opts ...grpc.CallOption) (*RunClaimResponse, error) + RenewRun(ctx context.Context, in *RenewRunRequest, opts ...grpc.CallOption) (*RunClaimResponse, error) + ReleaseRun(ctx context.Context, in *ReleaseRunRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + CommitReference(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + AbortReference(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + ReserveSuccessor(ctx context.Context, in *ReserveSuccessorRequest, opts ...grpc.CallOption) (*ReferenceReservationResponse, error) + PrepareReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + ConfirmReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + CancelReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + ListReferenceIntents(ctx context.Context, in *ListReferenceIntentsRequest, opts ...grpc.CallOption) (*ListReferenceIntentsResponse, error) + ReleaseReference(ctx context.Context, in *ReleaseReferenceRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + RetireEnvironment(ctx context.Context, in *RetireEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + ReplaceExecutor(ctx context.Context, in *ReplaceExecutorRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + RecoverEnvironment(ctx context.Context, in *RecoverEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + DeleteRetiredEnvironment(ctx context.Context, in *DeleteRetiredEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + MigrateEnvironment(ctx context.Context, in *MigrateEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + RevokeEnvironment(ctx context.Context, in *RevokeEnvironmentRequest, opts ...grpc.CallOption) (*RevokeEnvironmentResponse, error) + Files(ctx context.Context, in *FileRequest, opts ...grpc.CallOption) (*FileResponse, error) + StartCommand(ctx context.Context, in *CommandStartRequest, opts ...grpc.CallOption) (*CommandStartResponse, error) + CommandStatus(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) + CancelCommand(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) +} + +type executionProviderServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewExecutionProviderServiceClient(cc grpc.ClientConnInterface) ExecutionProviderServiceClient { + return &executionProviderServiceClient{cc} +} + +func (c *executionProviderServiceClient) ValidateProfile(ctx context.Context, in *ValidateProfileRequest, opts ...grpc.CallOption) (*ValidateProfileResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ValidateProfileResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_ValidateProfile_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) EnsureEnvironment(ctx context.Context, in *EnsureEnvironmentRequest, opts ...grpc.CallOption) (*EnsureEnvironmentResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(EnsureEnvironmentResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_EnsureEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) AttachEnvironment(ctx context.Context, in *AttachEnvironmentRequest, opts ...grpc.CallOption) (*AttachEnvironmentResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(AttachEnvironmentResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_AttachEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) AcquireRun(ctx context.Context, in *AcquireRunRequest, opts ...grpc.CallOption) (*RunClaimResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RunClaimResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_AcquireRun_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) RenewRun(ctx context.Context, in *RenewRunRequest, opts ...grpc.CallOption) (*RunClaimResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RunClaimResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_RenewRun_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ReleaseRun(ctx context.Context, in *ReleaseRunRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_ReleaseRun_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) CommitReference(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_CommitReference_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) AbortReference(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_AbortReference_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ReserveSuccessor(ctx context.Context, in *ReserveSuccessorRequest, opts ...grpc.CallOption) (*ReferenceReservationResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReferenceReservationResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_ReserveSuccessor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) PrepareReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_PrepareReferenceDelete_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ConfirmReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_ConfirmReferenceDelete_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) CancelReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_CancelReferenceDelete_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ListReferenceIntents(ctx context.Context, in *ListReferenceIntentsRequest, opts ...grpc.CallOption) (*ListReferenceIntentsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListReferenceIntentsResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_ListReferenceIntents_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ReleaseReference(ctx context.Context, in *ReleaseReferenceRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_ReleaseReference_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) RetireEnvironment(ctx context.Context, in *RetireEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_RetireEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ReplaceExecutor(ctx context.Context, in *ReplaceExecutorRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_ReplaceExecutor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) RecoverEnvironment(ctx context.Context, in *RecoverEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_RecoverEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) DeleteRetiredEnvironment(ctx context.Context, in *DeleteRetiredEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_DeleteRetiredEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) MigrateEnvironment(ctx context.Context, in *MigrateEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_MigrateEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) RevokeEnvironment(ctx context.Context, in *RevokeEnvironmentRequest, opts ...grpc.CallOption) (*RevokeEnvironmentResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RevokeEnvironmentResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_RevokeEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) Files(ctx context.Context, in *FileRequest, opts ...grpc.CallOption) (*FileResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(FileResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_Files_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) StartCommand(ctx context.Context, in *CommandStartRequest, opts ...grpc.CallOption) (*CommandStartResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CommandStartResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_StartCommand_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) CommandStatus(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CommandStatusResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_CommandStatus_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) CancelCommand(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CommandStatusResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_CancelCommand_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ExecutionProviderServiceServer is the server API for ExecutionProviderService service. +// All implementations must embed UnimplementedExecutionProviderServiceServer +// for forward compatibility. +type ExecutionProviderServiceServer interface { + ValidateProfile(context.Context, *ValidateProfileRequest) (*ValidateProfileResponse, error) + EnsureEnvironment(context.Context, *EnsureEnvironmentRequest) (*EnsureEnvironmentResponse, error) + AttachEnvironment(context.Context, *AttachEnvironmentRequest) (*AttachEnvironmentResponse, error) + AcquireRun(context.Context, *AcquireRunRequest) (*RunClaimResponse, error) + RenewRun(context.Context, *RenewRunRequest) (*RunClaimResponse, error) + ReleaseRun(context.Context, *ReleaseRunRequest) (*emptypb.Empty, error) + CommitReference(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + AbortReference(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + ReserveSuccessor(context.Context, *ReserveSuccessorRequest) (*ReferenceReservationResponse, error) + PrepareReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + ConfirmReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + CancelReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + ListReferenceIntents(context.Context, *ListReferenceIntentsRequest) (*ListReferenceIntentsResponse, error) + ReleaseReference(context.Context, *ReleaseReferenceRequest) (*emptypb.Empty, error) + RetireEnvironment(context.Context, *RetireEnvironmentRequest) (*emptypb.Empty, error) + ReplaceExecutor(context.Context, *ReplaceExecutorRequest) (*emptypb.Empty, error) + RecoverEnvironment(context.Context, *RecoverEnvironmentRequest) (*emptypb.Empty, error) + DeleteRetiredEnvironment(context.Context, *DeleteRetiredEnvironmentRequest) (*emptypb.Empty, error) + MigrateEnvironment(context.Context, *MigrateEnvironmentRequest) (*emptypb.Empty, error) + RevokeEnvironment(context.Context, *RevokeEnvironmentRequest) (*RevokeEnvironmentResponse, error) + Files(context.Context, *FileRequest) (*FileResponse, error) + StartCommand(context.Context, *CommandStartRequest) (*CommandStartResponse, error) + CommandStatus(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) + CancelCommand(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) + mustEmbedUnimplementedExecutionProviderServiceServer() +} + +// UnimplementedExecutionProviderServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedExecutionProviderServiceServer struct{} + +func (UnimplementedExecutionProviderServiceServer) ValidateProfile(context.Context, *ValidateProfileRequest) (*ValidateProfileResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ValidateProfile not implemented") +} +func (UnimplementedExecutionProviderServiceServer) EnsureEnvironment(context.Context, *EnsureEnvironmentRequest) (*EnsureEnvironmentResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method EnsureEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) AttachEnvironment(context.Context, *AttachEnvironmentRequest) (*AttachEnvironmentResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AttachEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) AcquireRun(context.Context, *AcquireRunRequest) (*RunClaimResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AcquireRun not implemented") +} +func (UnimplementedExecutionProviderServiceServer) RenewRun(context.Context, *RenewRunRequest) (*RunClaimResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RenewRun not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ReleaseRun(context.Context, *ReleaseRunRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReleaseRun not implemented") +} +func (UnimplementedExecutionProviderServiceServer) CommitReference(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method CommitReference not implemented") +} +func (UnimplementedExecutionProviderServiceServer) AbortReference(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method AbortReference not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ReserveSuccessor(context.Context, *ReserveSuccessorRequest) (*ReferenceReservationResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReserveSuccessor not implemented") +} +func (UnimplementedExecutionProviderServiceServer) PrepareReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method PrepareReferenceDelete not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ConfirmReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ConfirmReferenceDelete not implemented") +} +func (UnimplementedExecutionProviderServiceServer) CancelReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method CancelReferenceDelete not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ListReferenceIntents(context.Context, *ListReferenceIntentsRequest) (*ListReferenceIntentsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListReferenceIntents not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ReleaseReference(context.Context, *ReleaseReferenceRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReleaseReference not implemented") +} +func (UnimplementedExecutionProviderServiceServer) RetireEnvironment(context.Context, *RetireEnvironmentRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method RetireEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ReplaceExecutor(context.Context, *ReplaceExecutorRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReplaceExecutor not implemented") +} +func (UnimplementedExecutionProviderServiceServer) RecoverEnvironment(context.Context, *RecoverEnvironmentRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method RecoverEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) DeleteRetiredEnvironment(context.Context, *DeleteRetiredEnvironmentRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteRetiredEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) MigrateEnvironment(context.Context, *MigrateEnvironmentRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method MigrateEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) RevokeEnvironment(context.Context, *RevokeEnvironmentRequest) (*RevokeEnvironmentResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RevokeEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) Files(context.Context, *FileRequest) (*FileResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Files not implemented") +} +func (UnimplementedExecutionProviderServiceServer) StartCommand(context.Context, *CommandStartRequest) (*CommandStartResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method StartCommand not implemented") +} +func (UnimplementedExecutionProviderServiceServer) CommandStatus(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CommandStatus not implemented") +} +func (UnimplementedExecutionProviderServiceServer) CancelCommand(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CancelCommand not implemented") +} +func (UnimplementedExecutionProviderServiceServer) mustEmbedUnimplementedExecutionProviderServiceServer() { +} +func (UnimplementedExecutionProviderServiceServer) testEmbeddedByValue() {} + +// UnsafeExecutionProviderServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ExecutionProviderServiceServer will +// result in compilation errors. +type UnsafeExecutionProviderServiceServer interface { + mustEmbedUnimplementedExecutionProviderServiceServer() +} + +func RegisterExecutionProviderServiceServer(s grpc.ServiceRegistrar, srv ExecutionProviderServiceServer) { + // If the following call pancis, it indicates UnimplementedExecutionProviderServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&ExecutionProviderService_ServiceDesc, srv) +} + +func _ExecutionProviderService_ValidateProfile_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ValidateProfileRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ValidateProfile(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ValidateProfile_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ValidateProfile(ctx, req.(*ValidateProfileRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_EnsureEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(EnsureEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).EnsureEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_EnsureEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).EnsureEnvironment(ctx, req.(*EnsureEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_AttachEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AttachEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).AttachEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_AttachEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).AttachEnvironment(ctx, req.(*AttachEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_AcquireRun_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AcquireRunRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).AcquireRun(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_AcquireRun_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).AcquireRun(ctx, req.(*AcquireRunRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_RenewRun_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RenewRunRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).RenewRun(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_RenewRun_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).RenewRun(ctx, req.(*RenewRunRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ReleaseRun_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReleaseRunRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ReleaseRun(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ReleaseRun_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ReleaseRun(ctx, req.(*ReleaseRunRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_CommitReference_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).CommitReference(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_CommitReference_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).CommitReference(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_AbortReference_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).AbortReference(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_AbortReference_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).AbortReference(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ReserveSuccessor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReserveSuccessorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ReserveSuccessor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ReserveSuccessor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ReserveSuccessor(ctx, req.(*ReserveSuccessorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_PrepareReferenceDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).PrepareReferenceDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_PrepareReferenceDelete_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).PrepareReferenceDelete(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ConfirmReferenceDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ConfirmReferenceDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ConfirmReferenceDelete_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ConfirmReferenceDelete(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_CancelReferenceDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).CancelReferenceDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_CancelReferenceDelete_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).CancelReferenceDelete(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ListReferenceIntents_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListReferenceIntentsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ListReferenceIntents(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ListReferenceIntents_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ListReferenceIntents(ctx, req.(*ListReferenceIntentsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ReleaseReference_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReleaseReferenceRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ReleaseReference(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ReleaseReference_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ReleaseReference(ctx, req.(*ReleaseReferenceRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_RetireEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RetireEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).RetireEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_RetireEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).RetireEnvironment(ctx, req.(*RetireEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ReplaceExecutor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReplaceExecutorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ReplaceExecutor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ReplaceExecutor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ReplaceExecutor(ctx, req.(*ReplaceExecutorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_RecoverEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RecoverEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).RecoverEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_RecoverEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).RecoverEnvironment(ctx, req.(*RecoverEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_DeleteRetiredEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteRetiredEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).DeleteRetiredEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_DeleteRetiredEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).DeleteRetiredEnvironment(ctx, req.(*DeleteRetiredEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_MigrateEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(MigrateEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).MigrateEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_MigrateEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).MigrateEnvironment(ctx, req.(*MigrateEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_RevokeEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RevokeEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).RevokeEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_RevokeEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).RevokeEnvironment(ctx, req.(*RevokeEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_Files_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FileRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).Files(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_Files_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).Files(ctx, req.(*FileRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_StartCommand_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CommandStartRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).StartCommand(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_StartCommand_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).StartCommand(ctx, req.(*CommandStartRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_CommandStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CommandQueryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).CommandStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_CommandStatus_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).CommandStatus(ctx, req.(*CommandQueryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_CancelCommand_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CommandQueryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).CancelCommand(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_CancelCommand_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).CancelCommand(ctx, req.(*CommandQueryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// ExecutionProviderService_ServiceDesc is the grpc.ServiceDesc for ExecutionProviderService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ExecutionProviderService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "mecatl.execution.v1.ExecutionProviderService", + HandlerType: (*ExecutionProviderServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ValidateProfile", + Handler: _ExecutionProviderService_ValidateProfile_Handler, + }, + { + MethodName: "EnsureEnvironment", + Handler: _ExecutionProviderService_EnsureEnvironment_Handler, + }, + { + MethodName: "AttachEnvironment", + Handler: _ExecutionProviderService_AttachEnvironment_Handler, + }, + { + MethodName: "AcquireRun", + Handler: _ExecutionProviderService_AcquireRun_Handler, + }, + { + MethodName: "RenewRun", + Handler: _ExecutionProviderService_RenewRun_Handler, + }, + { + MethodName: "ReleaseRun", + Handler: _ExecutionProviderService_ReleaseRun_Handler, + }, + { + MethodName: "CommitReference", + Handler: _ExecutionProviderService_CommitReference_Handler, + }, + { + MethodName: "AbortReference", + Handler: _ExecutionProviderService_AbortReference_Handler, + }, + { + MethodName: "ReserveSuccessor", + Handler: _ExecutionProviderService_ReserveSuccessor_Handler, + }, + { + MethodName: "PrepareReferenceDelete", + Handler: _ExecutionProviderService_PrepareReferenceDelete_Handler, + }, + { + MethodName: "ConfirmReferenceDelete", + Handler: _ExecutionProviderService_ConfirmReferenceDelete_Handler, + }, + { + MethodName: "CancelReferenceDelete", + Handler: _ExecutionProviderService_CancelReferenceDelete_Handler, + }, + { + MethodName: "ListReferenceIntents", + Handler: _ExecutionProviderService_ListReferenceIntents_Handler, + }, + { + MethodName: "ReleaseReference", + Handler: _ExecutionProviderService_ReleaseReference_Handler, + }, + { + MethodName: "RetireEnvironment", + Handler: _ExecutionProviderService_RetireEnvironment_Handler, + }, + { + MethodName: "ReplaceExecutor", + Handler: _ExecutionProviderService_ReplaceExecutor_Handler, + }, + { + MethodName: "RecoverEnvironment", + Handler: _ExecutionProviderService_RecoverEnvironment_Handler, + }, + { + MethodName: "DeleteRetiredEnvironment", + Handler: _ExecutionProviderService_DeleteRetiredEnvironment_Handler, + }, + { + MethodName: "MigrateEnvironment", + Handler: _ExecutionProviderService_MigrateEnvironment_Handler, + }, + { + MethodName: "RevokeEnvironment", + Handler: _ExecutionProviderService_RevokeEnvironment_Handler, + }, + { + MethodName: "Files", + Handler: _ExecutionProviderService_Files_Handler, + }, + { + MethodName: "StartCommand", + Handler: _ExecutionProviderService_StartCommand_Handler, + }, + { + MethodName: "CommandStatus", + Handler: _ExecutionProviderService_CommandStatus_Handler, + }, + { + MethodName: "CancelCommand", + Handler: _ExecutionProviderService_CancelCommand_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "mecatl/execution/v1/execution.proto", +} diff --git a/contracts/proto/mecatl/execution/v1/execution.proto b/contracts/proto/mecatl/execution/v1/execution.proto new file mode 100644 index 0000000000..9b3b369092 --- /dev/null +++ b/contracts/proto/mecatl/execution/v1/execution.proto @@ -0,0 +1,266 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: LicenseRef-Stacklok-Proprietary + +// Private, mutually authenticated execution-provider protocol. This is not part +// of the public Harness API. +syntax = "proto3"; + +package mecatl.execution.v1; + +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1;executionv1"; + +service ExecutionProviderService { + rpc ValidateProfile(ValidateProfileRequest) returns (ValidateProfileResponse); + rpc EnsureEnvironment(EnsureEnvironmentRequest) returns (EnsureEnvironmentResponse); + rpc AttachEnvironment(AttachEnvironmentRequest) returns (AttachEnvironmentResponse); + rpc AcquireRun(AcquireRunRequest) returns (RunClaimResponse); + rpc RenewRun(RenewRunRequest) returns (RunClaimResponse); + rpc ReleaseRun(ReleaseRunRequest) returns (google.protobuf.Empty); + rpc CommitReference(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc AbortReference(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc ReserveSuccessor(ReserveSuccessorRequest) returns (ReferenceReservationResponse); + rpc PrepareReferenceDelete(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc ConfirmReferenceDelete(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc CancelReferenceDelete(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc ListReferenceIntents(ListReferenceIntentsRequest) returns (ListReferenceIntentsResponse); + rpc ReleaseReference(ReleaseReferenceRequest) returns (google.protobuf.Empty); + rpc RetireEnvironment(RetireEnvironmentRequest) returns (google.protobuf.Empty); + rpc ReplaceExecutor(ReplaceExecutorRequest) returns (google.protobuf.Empty); + rpc RecoverEnvironment(RecoverEnvironmentRequest) returns (google.protobuf.Empty); + rpc DeleteRetiredEnvironment(DeleteRetiredEnvironmentRequest) returns (google.protobuf.Empty); + rpc MigrateEnvironment(MigrateEnvironmentRequest) returns (google.protobuf.Empty); + rpc RevokeEnvironment(RevokeEnvironmentRequest) returns (RevokeEnvironmentResponse); + rpc Files(FileRequest) returns (FileResponse); + rpc StartCommand(CommandStartRequest) returns (CommandStartResponse); + rpc CommandStatus(CommandQueryRequest) returns (CommandStatusResponse); + rpc CancelCommand(CommandQueryRequest) returns (CommandStatusResponse); +} + +message EnvironmentRef { string id = 1; string revision = 2; } +message Owner { string issuer = 1; string subject = 2; } +message RequestContext { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + uint64 epoch = 4; + string grant = 5; + string run_id = 6; + string claim_id = 7; + uint64 grant_generation = 8; +} + +message ValidateProfileRequest { string profile = 1; } +message ValidateProfileResponse { + string profile = 1; + string digest = 2; + repeated string capabilities = 3; + int64 max_file_bytes = 4; + int64 max_command_bytes = 5; + int64 max_command_duration_millis = 6; +} +message EnsureEnvironmentRequest { string binding_id = 1; string profile = 2; Owner owner = 3; string operation_id = 4; } +message EnsureEnvironmentResponse { + EnvironmentRef environment = 1; + uint64 epoch = 2; + bool ready = 3; + string grant = 4; + google.protobuf.Timestamp grant_expires_at = 5; + uint64 grant_generation = 6; +} +message AttachEnvironmentRequest { RequestContext context = 1; string purpose = 2; } +message AttachEnvironmentResponse { + EnvironmentRef environment = 1; + uint64 epoch = 2; + bool ready = 3; + string grant = 4; + google.protobuf.Timestamp grant_expires_at = 5; + uint64 grant_generation = 6; +} +message AcquireRunRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + string run_id = 4; + string operation_id = 5; + int64 ttl_millis = 6; +} +message RenewRunRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + string run_id = 4; + string claim_id = 5; + uint64 epoch = 6; + string operation_id = 7; + int64 ttl_millis = 8; + uint64 grant_generation = 9; +} +message ReleaseRunRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + string run_id = 4; + string claim_id = 5; + uint64 epoch = 6; + string operation_id = 7; + uint64 grant_generation = 8; +} +message RunClaimResponse { + EnvironmentRef environment = 1; + string binding_id = 2; + string run_id = 3; + string claim_id = 4; + uint64 epoch = 5; + uint64 grant_generation = 6; + string grant = 7; + google.protobuf.Timestamp expires_at = 8; +} +message ReferenceMutationRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + string operation_id = 4; +} +message ReserveSuccessorRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string source_binding_id = 3; + string destination_binding_id = 4; + string operation_id = 5; +} +message ReferenceReservationResponse { EnvironmentRef environment = 1; } +message ListReferenceIntentsRequest { + Owner owner = 1; + int32 limit = 2; + EnvironmentRef environment = 3; + string binding_id = 4; +} +message ReferenceIntent { + EnvironmentRef environment = 1; + string binding_id = 2; + string state = 3; + string operation_id = 4; + string source_binding_id = 5; + google.protobuf.Timestamp created_at = 6; + Owner owner = 7; +} +message ListReferenceIntentsResponse { repeated ReferenceIntent intents = 1; } +message ReleaseReferenceRequest { RequestContext context = 1; } +message RetireEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint64 expected_execution_epoch = 3; + string expected_pod_uid = 4; + string expected_pvc_uid = 5; + string operation_id = 6; +} +message ReplaceExecutorRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint64 expected_execution_epoch = 3; + string expected_pod_uid = 4; + string expected_pvc_uid = 5; + string operation_id = 6; +} +message RecoverEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint64 expected_execution_epoch = 3; + string expected_pod_uid = 4; + string expected_pvc_uid = 5; + string operation_id = 6; +} +message DeleteRetiredEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string expected_pvc_uid = 3; + string operation_id = 4; +} +message RevokeEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint64 expected_grant_generation = 3; + string operation_id = 4; +} +message RevokeEnvironmentResponse { uint64 grant_generation = 1; } + +message MigrateEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + optional uint32 expected_schema_version = 3; + string expected_pod_uid = 4; + string expected_pvc_uid = 5; + string operation_id = 6; +} + +enum FileOperation { + FILE_OPERATION_UNSPECIFIED = 0; + FILE_OPERATION_READ = 1; + FILE_OPERATION_RESOLVE_AUTHORITY = 2; + FILE_OPERATION_STAT = 3; + FILE_OPERATION_CREATE = 4; + FILE_OPERATION_REPLACE = 5; + FILE_OPERATION_LIST = 6; + FILE_OPERATION_REMOVE = 7; + FILE_OPERATION_RENAME = 8; + FILE_OPERATION_COPY = 9; + FILE_OPERATION_GLOB = 10; + FILE_OPERATION_GREP = 11; +} +message FileRequest { + RequestContext context = 1; + FileOperation operation = 2; + string path = 3; + string destination = 4; + string pattern = 5; + bytes data = 6; + bytes version = 7; + int32 limit = 8; +} +message FileInfo { + string name = 1; + int64 size = 2; + uint32 mode = 3; + google.protobuf.Timestamp mod_time = 4; + bool is_dir = 5; +} +message GrepMatch { string path = 1; int32 line = 2; string text = 3; } +message FileResponse { + bytes data = 1; + bytes version = 2; + FileInfo info = 3; + repeated FileInfo entries = 4; + repeated string paths = 5; + repeated GrepMatch matches = 6; + string authority_target = 7; + string authority_workspace = 8; +} + +enum CommandState { + COMMAND_STATE_UNSPECIFIED = 0; + COMMAND_STATE_RUNNING = 1; + COMMAND_STATE_SUCCEEDED = 2; + COMMAND_STATE_FAILED = 3; + COMMAND_STATE_CANCELLED = 4; + COMMAND_STATE_FENCE_UNKNOWN = 5; +} +message CommandStartRequest { RequestContext context = 1; string command = 2; int64 timeout_millis = 3; } +message CommandStartResponse { string command_id = 1; CommandState state = 2; CommandStatusResponse result = 3; } +message CommandQueryRequest { RequestContext context = 1; string command_id = 2; int64 offset = 3; } +message CommandStatusResponse { + string command_id = 1; + CommandState state = 2; + int32 exit_code = 3; + bytes stdout = 4; + bytes stderr = 5; + int64 next_offset = 6; + bool truncated = 7; + string terminal_receipt = 8; +} + +// ErrorDetail is a stable machine-readable classification. Status messages are +// intentionally generic so backend/Kubernetes details never cross the boundary. +message ErrorDetail { string code = 1; bool retryable = 2; } diff --git a/deploy/helm/mecak8s/templates/deployment.yaml b/deploy/helm/mecak8s/templates/deployment.yaml index 9fd32c5019..78a15d6edf 100644 --- a/deploy/helm/mecak8s/templates/deployment.yaml +++ b/deploy/helm/mecak8s/templates/deployment.yaml @@ -130,6 +130,14 @@ spec: {{- if .Values.workspace }} - {{ printf "--workspace=%s" .Values.workspace | quote }} {{- end }} + {{- if .Values.execution.enabled }} + - --execution-enabled + - {{ printf "--execution-endpoint=%s" .Values.execution.endpoint | quote }} + - {{ printf "--execution-profile=%s" .Values.execution.profile | quote }} + - {{ printf "--execution-tls-ca=/var/run/secrets/execution/%s" .Values.execution.caKey | quote }} + - {{ printf "--execution-tls-cert=/var/run/secrets/execution/%s" .Values.execution.certKey | quote }} + - {{ printf "--execution-tls-key=/var/run/secrets/execution/%s" .Values.execution.keyKey | quote }} + {{- end }} {{- if .Values.mockProvider }} - --mock {{- end }} @@ -294,6 +302,9 @@ spec: {{- if .Values.oidc.caSecret }} - {name: oidc-ca, mountPath: /var/run/secrets/oidc-ca, readOnly: true} {{- end }} + {{- if .Values.execution.enabled }} + - {name: execution-mtls, mountPath: /var/run/secrets/execution, readOnly: true} + {{- end }} {{- if .Values.learning.store.tls.caSecret }} - {name: learning-store-ca, mountPath: /var/run/secrets/learning-store-ca, readOnly: true} {{- end }} @@ -342,6 +353,16 @@ spec: items: - {key: {{ .Values.oidc.caKey | quote }}, path: {{ .Values.oidc.caKey | quote }}} {{- end }} + {{- if .Values.execution.enabled }} + - name: execution-mtls + secret: + secretName: {{ .Values.execution.tlsSecret }} + defaultMode: 0440 + items: + - {key: {{ .Values.execution.caKey | quote }}, path: {{ .Values.execution.caKey | quote }}} + - {key: {{ .Values.execution.certKey | quote }}, path: {{ .Values.execution.certKey | quote }}} + - {key: {{ .Values.execution.keyKey | quote }}, path: {{ .Values.execution.keyKey | quote }}} + {{- end }} {{- if .Values.learning.store.tls.caSecret }} - name: learning-store-ca secret: diff --git a/deploy/helm/mecak8s/values.schema.json b/deploy/helm/mecak8s/values.schema.json index b17f2cd3f0..ab7daa5a15 100644 --- a/deploy/helm/mecak8s/values.schema.json +++ b/deploy/helm/mecak8s/values.schema.json @@ -116,6 +116,20 @@ } }, "workspace": {"type": "string"}, + "execution": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "endpoint", "profile", "tlsSecret", "caKey", "certKey", "keyKey"], + "properties": { + "enabled": {"type": "boolean"}, + "endpoint": {"type": "string", "anyOf": [{"const": ""}, {"pattern": "^(?:[A-Za-z0-9.-]+|\\[[0-9A-Fa-f:]+\\]):[0-9]+$"}]}, + "profile": {"type": "string"}, + "tlsSecret": {"type": "string"}, + "caKey": {"type": "string", "minLength": 1}, + "certKey": {"type": "string", "minLength": 1}, + "keyKey": {"type": "string", "minLength": 1} + } + }, "imagePullSecrets": {"type": "array", "items": {"type": "object", "additionalProperties": false, "required": ["name"], "properties": {"name": {"type": "string", "minLength": 1}}}}, "mcp": { "type": "object", @@ -371,6 +385,17 @@ "podAnnotations": {"type": "object"} }, "allOf": [ + { + "if": {"properties": {"execution": {"properties": {"enabled": {"const": true}}}}}, + "then": { + "properties": { + "workspace": {"maxLength": 0}, + "execution": {"properties": {"endpoint": {"minLength": 1}, "profile": {"minLength": 1}, "tlsSecret": {"minLength": 1}}}, + "redis": {"properties": {"filesystem": {"properties": {"enabled": {"const": false}}}}}, + "oidc": {"properties": {"enabled": {"const": true}}} + } + } + }, { "if": {"properties": {"redis": {"properties": {"filesystem": {"properties": {"enabled": {"const": true}}}}}}}, "then": {"properties": {"workspace": {"maxLength": 0}}} diff --git a/deploy/helm/mecak8s/values.yaml b/deploy/helm/mecak8s/values.yaml index 9b370513a1..21d4564426 100644 --- a/deploy/helm/mecak8s/values.yaml +++ b/deploy/helm/mecak8s/values.yaml @@ -160,6 +160,17 @@ podDisruptionBudget: # the volume yourself; the chart only passes the flag. workspace: "" +# Optional client for the separately installed execution-provider service. The +# provider chart is not a dependency of this chart. +execution: + enabled: false + endpoint: "" + profile: "" + tlsSecret: "" + caKey: ca.crt + certKey: tls.crt + keyKey: tls.key + redis: # A production release never creates Redis. Supply the managed endpoint and, # when the service needs a private CA or ACL credentials, a Secret holding diff --git a/deploy/helm/mecatl-execution/Chart.yaml b/deploy/helm/mecatl-execution/Chart.yaml new file mode 100644 index 0000000000..7d35df551f --- /dev/null +++ b/deploy/helm/mecatl-execution/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: mecatl-execution +version: 0.1.0-draft +appVersion: dev +description: Optional native Kubernetes execution provider (draft) +type: application diff --git a/deploy/helm/mecatl-execution/chart_test.go b/deploy/helm/mecatl-execution/chart_test.go new file mode 100644 index 0000000000..a3da6bdbad --- /dev/null +++ b/deploy/helm/mecatl-execution/chart_test.go @@ -0,0 +1,78 @@ +package chart_test + +import ( + "os" + "strings" + "testing" +) + +func TestChartRetainsCRDAndDoesNotGrantSecretAPI(t *testing.T) { + crd, err := os.ReadFile("crds/executionenvironment.yaml") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(crd), "helm.sh/resource-policy: keep") { + t.Fatal("CRD is not retained on uninstall") + } + rbac, err := os.ReadFile("templates/rbac.yaml") + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(rbac), "secrets") { + t.Fatal("provider RBAC must have no Secret access") + } + for _, required := range []string{"kind: ClusterRole", `resources: ["runtimeclasses"]`, `resources: ["storageclasses"]`, "resourceNames:", `verbs: ["get"]`} { + if !strings.Contains(string(rbac), required) { + t.Fatalf("profile preflight RBAC missing %q", required) + } + } + if strings.Contains(string(rbac), `resources: ["*"]`) { + t.Fatal("cluster-scoped profile preflight RBAC must not grant wildcard resources") + } +} +func TestChartHasNoDeletionHook(t *testing.T) { + entries, err := os.ReadDir("templates") + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + b, err := os.ReadFile("templates/" + entry.Name()) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(b), "helm.sh/hook") || strings.Contains(string(b), "PersistentVolumeClaim\nmetadata") { + t.Fatalf("unsafe lifecycle resource in %s", entry.Name()) + } + } +} + +func TestProductionHardeningIsFailClosed(t *testing.T) { + provider, err := os.ReadFile("templates/provider.yaml") + if err != nil { + t.Fatal(err) + } + network, err := os.ReadFile("templates/network-policy.yaml") + if err != nil { + t.Fatal(err) + } + values, err := os.ReadFile("values.schema.json") + if err != nil { + t.Fatal(err) + } + for _, required := range []string{"runAsNonRoot: true", "seccompProfile", "capabilities: {drop: [\"ALL\"]}", "readOnlyRootFilesystem: true", "httpGet: {path: /ready", "security-authority-configmap", "projected:", "maxUnavailable: 0"} { + if !strings.Contains(string(provider), required) { + t.Fatalf("provider hardening missing %q", required) + } + } + for _, required := range []string{"workload-default-deny", "ingress: []", "egress: []", "ResourceQuota", "LimitRange", "count/executionenvironments.execution.mecatl.dev", "requests.ephemeral-storage", "limits.ephemeral-storage"} { + if !strings.Contains(string(network), required) { + t.Fatalf("network/resource hardening missing %q", required) + } + } + if strings.Contains(string(network), "0.0.0.0/0") || !strings.Contains(string(values), `"minimum":2`) { + t.Fatal("chart permits an unsafe replica or egress default") + } +} diff --git a/deploy/helm/mecatl-execution/crds/executionenvironment.yaml b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml new file mode 100644 index 0000000000..f31fc8340c --- /dev/null +++ b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml @@ -0,0 +1,213 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: executionenvironments.execution.mecatl.dev + annotations: + helm.sh/resource-policy: keep +spec: + group: execution.mecatl.dev + scope: Namespaced + names: + plural: executionenvironments + singular: executionenvironment + kind: ExecutionEnvironment + shortNames: [execenv] + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + spec: + type: object + x-kubernetes-validations: + - rule: "!has(oldSelf.allocationID) || self.allocationID == oldSelf.allocationID" + message: allocationID is immutable + - rule: "!has(oldSelf.revision) || self.revision == oldSelf.revision" + message: revision is immutable + - rule: "!has(oldSelf.ownerHash) || self.ownerHash == oldSelf.ownerHash" + message: ownerHash is immutable + - rule: "!has(oldSelf.ownerIssuer) || self.ownerIssuer == oldSelf.ownerIssuer" + message: ownerIssuer is immutable + - rule: "!has(oldSelf.ownerSubject) || self.ownerSubject == oldSelf.ownerSubject" + message: ownerSubject is immutable + - rule: "!has(oldSelf.clientHash) || self.clientHash == oldSelf.clientHash" + message: clientHash is immutable + - rule: "!has(oldSelf.bindingID) || self.bindingID == oldSelf.bindingID" + message: bindingID is immutable + - rule: "!has(oldSelf.requestFingerprint) || self.requestFingerprint == oldSelf.requestFingerprint" + message: requestFingerprint is immutable + - rule: "!has(oldSelf.profileDigest) || self.profileDigest == oldSelf.profileDigest" + message: profileDigest is immutable + - rule: "!has(oldSelf.profile) || self.profile == oldSelf.profile" + message: profile is immutable + - rule: "!has(oldSelf.image) || self.image == oldSelf.image" + message: image is immutable + - rule: "!has(oldSelf.storageClass) || self.storageClass == oldSelf.storageClass" + message: storageClass is immutable + - rule: "!has(oldSelf.storageSize) || self.storageSize == oldSelf.storageSize" + message: storageSize is immutable + - rule: "!has(oldSelf.resources) || self.resources == oldSelf.resources" + message: resources are immutable + required: [allocationID, revision, ownerHash, clientHash, bindingID, requestFingerprint, profile, profileDigest, image, storageClass, storageSize, resources, desired] + properties: + schemaVersion: {type: integer, format: int64, enum: [0, 1, 2]} + allocationID: {type: string, minLength: 1, maxLength: 63} + revision: {type: string, minLength: 1, maxLength: 64} + ownerHash: {type: string, pattern: "^[0-9a-f]{64}$"} + ownerIssuer: {type: string, minLength: 1, maxLength: 1024} + ownerSubject: {type: string, minLength: 1, maxLength: 1024} + clientHash: {type: string, pattern: "^[0-9a-f]{64}$"} + bindingID: {type: string, minLength: 1, maxLength: 253} + requestFingerprint: {type: string, pattern: "^[0-9a-f]{64}$"} + profile: {type: string, minLength: 1, maxLength: 63} + profileDigest: {type: string, pattern: "^sha256:[0-9a-f]{64}$"} + image: {type: string, pattern: "@sha256:[0-9a-fA-F]{64}$"} + storageClass: {type: string, minLength: 1} + storageSize: {type: string, minLength: 1} + resources: + type: object + required: [cpuRequest, memoryRequest, cpuLimit, memoryLimit] + properties: + cpuRequest: {type: string, minLength: 1} + memoryRequest: {type: string, minLength: 1} + cpuLimit: {type: string, minLength: 1} + memoryLimit: {type: string, minLength: 1} + desired: {type: string, enum: [Active, Retiring]} + status: + type: object + properties: + schemaVersion: {type: integer, format: int64, enum: [0, 1, 2]} + observedGeneration: {type: integer, format: int64} + epoch: {type: integer, format: int64, minimum: 1} + fenceState: {type: string, enum: [Healthy, FenceUnknown]} + grantGeneration: {type: integer, format: int64, minimum: 1} + revocationReceipts: + type: array + maxItems: 32 + items: + type: object + required: [operationID, fingerprint, expectedGrantGeneration, grantGeneration] + properties: + operationID: {type: string, minLength: 1, maxLength: 1024} + fingerprint: {type: string, pattern: "^[0-9a-f]{64}$"} + expectedGrantGeneration: {type: integer, format: int64, minimum: 1} + grantGeneration: {type: integer, format: int64, minimum: 1} + references: + type: array + maxItems: 64 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [bindingID] + items: + type: object + required: [bindingID, state, operationID, createdAt] + properties: + bindingID: {type: string, minLength: 1, maxLength: 253} + state: {type: string, enum: [PendingCreate, Published, PendingDelete]} + operationID: {type: string, minLength: 1, maxLength: 1024} + sourceBindingID: {type: string, maxLength: 253} + createdAt: {type: string, format: date-time} + pvc: + type: object + properties: + name: {type: string} + uid: {type: string} + pod: + type: object + properties: + name: {type: string} + uid: {type: string} + activeRun: + type: object + required: [bindingID, runID, claimID, operationID, ownerHash, clientHash, epoch, grantGeneration, expiresAt] + properties: + bindingID: {type: string, maxLength: 253} + runID: {type: string, maxLength: 1024} + claimID: {type: string, maxLength: 1024} + operationID: {type: string, maxLength: 1024} + ownerHash: {type: string, pattern: "^[0-9a-f]{64}$"} + clientHash: {type: string, pattern: "^[0-9a-f]{64}$"} + epoch: {type: integer, format: int64, minimum: 1} + grantGeneration: {type: integer, format: int64, minimum: 1} + expiresAt: {type: string, format: date-time} + renewReceipts: + type: array + maxItems: 32 + items: + type: object + required: [operationID, fingerprint, expiresAt] + properties: + operationID: {type: string, minLength: 1, maxLength: 1024} + fingerprint: {type: string, pattern: "^[0-9a-f]{64}$"} + expiresAt: {type: string, format: date-time} + activeOperation: + type: object + required: [id, operation, startedAt, claimID, runID, epoch, holderID, renewedAt, expiresAt] + properties: + id: {type: string, minLength: 1, maxLength: 1024} + operation: {type: string, minLength: 1, maxLength: 128} + startedAt: {type: string, format: date-time} + claimID: {type: string, maxLength: 1024} + runID: {type: string, maxLength: 1024} + epoch: {type: integer, format: int64, minimum: 1} + holderID: {type: string, minLength: 1, maxLength: 1024} + renewedAt: {type: string, format: date-time} + expiresAt: {type: string, format: date-time} + lifecycleOperation: + type: object + required: [id, type, phase, createdAt] + properties: + id: {type: string, minLength: 1, maxLength: 1024} + type: {type: string, enum: [ReplaceExecutor, RetireEnvironment, DeleteRetiredEnvironment]} + phase: {type: string, enum: [Quiescing, WaitingForTermination, RemovingPodFinalizer, WaitingForPodDeletion, CreatingReplacement, DeletingPVC, ReleasingSlot]} + expectedEpoch: {type: integer, format: int64, minimum: 1} + expectedPodUID: {type: string, maxLength: 128} + expectedPVCUID: {type: string, minLength: 1, maxLength: 128} + createdAt: {type: string, format: date-time} + terminationProof: + type: object + required: [operationID, podUID, pvcUID, epoch, podPhase, observedAt] + properties: + operationID: {type: string, minLength: 1, maxLength: 1024} + podUID: {type: string, minLength: 1, maxLength: 128} + pvcUID: {type: string, minLength: 1, maxLength: 128} + epoch: {type: integer, format: int64, minimum: 1} + podPhase: {type: string, enum: [Succeeded, Failed]} + observedAt: {type: string, format: date-time} + lastReplacement: + type: object + required: [operationID, previousPodUID, replacementPodUID, pvcUID, previousEpoch, replacementEpoch] + properties: + operationID: {type: string, minLength: 1, maxLength: 1024} + previousPodUID: {type: string, minLength: 1, maxLength: 128} + replacementPodUID: {type: string, minLength: 1, maxLength: 128} + pvcUID: {type: string, minLength: 1, maxLength: 128} + previousEpoch: {type: integer, format: int64, minimum: 1} + replacementEpoch: {type: integer, format: int64, minimum: 2} + migrationOperation: + type: object + required: [id, fromSchema, expectedPodUID, expectedPVCUID] + properties: + id: {type: string, minLength: 1, maxLength: 1024} + fromSchema: {type: integer, format: int64, enum: [0, 1]} + expectedPodUID: {type: string, minLength: 1, maxLength: 128} + expectedPVCUID: {type: string, minLength: 1, maxLength: 128} + lastMigrationOperationID: {type: string, maxLength: 1024} + lastMigrationFromSchema: {type: integer, format: int64, enum: [0, 1]} + conditions: + type: array + items: + type: object + required: [type, status, reason, message, observedGeneration, lastTransitionTime] + properties: + type: {type: string} + status: {type: string, enum: ["True", "False", Unknown]} + reason: {type: string} + message: {type: string, maxLength: 1024} + observedGeneration: {type: integer, format: int64} + lastTransitionTime: {type: string, format: date-time} diff --git a/deploy/helm/mecatl-execution/lifetime_test.go b/deploy/helm/mecatl-execution/lifetime_test.go new file mode 100644 index 0000000000..fa4d0b2c5f --- /dev/null +++ b/deploy/helm/mecatl-execution/lifetime_test.go @@ -0,0 +1,268 @@ +package chart_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/yaml" +) + +// Exercise Helm's actual lookup/render path, with a loopback API containing only +// synthetic nonsecret objects. No ambient kubeconfig or cluster is consulted. +func TestChartRetainedLifetime(t *testing.T) { + fresh, err := renderLifetime(t, nil) + if err != nil { + t.Fatal(err) + } + retained := map[string]map[string]any{} + for _, obj := range fresh { + u := &unstructured.Unstructured{Object: obj} + if u.GetAnnotations()["helm.sh/resource-policy"] == "keep" { + retained[u.GetKind()+"/"+u.GetName()] = obj + } + } + if len(retained) != 6 { + t.Fatalf("retained resources=%d, want four ConfigMaps and two workload policies", len(retained)) + } + authority := "ConfigMap/test-mecatl-execution-security-authority" + capacity := "ConfigMap/mecatl-execution-profile-allocations" + retained[authority]["data"] = map[string]any{"state.json": `{"generation":7,"digest":"synthetic","fingerprints":{"k1:1":"synthetic"}}`} + retained[capacity]["data"] = map[string]any{"profile-go": `["allocation-uid"]`} + retained["ExecutionEnvironment/exec-test"] = map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "exec-test", "namespace": "ns"}} + for _, upgrade := range []bool{false, true} { + t.Run(fmt.Sprintf("reuse-upgrade-%t", upgrade), func(t *testing.T) { + var args []string + if upgrade { + args = append(args, "--is-upgrade") + } + rendered, err := renderLifetime(t, retained, args...) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{authority, capacity} { + if !reflect.DeepEqual(rendered[key]["data"], retained[key]["data"]) { + t.Fatalf("%s data was reset", key) + } + } + for key, obj := range retained { + if strings.HasPrefix(key, "NetworkPolicy/") && !reflect.DeepEqual(rendered[key]["spec"], obj["spec"]) { + t.Fatalf("%s changed confinement", key) + } + } + }) + } + tests := []struct { + name string + mutate func(map[string]map[string]any) + args []string + want string + }{ + {name: "active provider deployment", mutate: func(m map[string]map[string]any) { + m["Deployment/test-mecatl-execution"] = map[string]any{"apiVersion": "apps/v1", "kind": "Deployment", "metadata": map[string]any{"name": "test-mecatl-execution", "namespace": "ns"}, "spec": map[string]any{"replicas": 2}} + }, want: "quiesce the execution provider"}, + {name: "provider pod still terminating", mutate: func(m map[string]map[string]any) { + m["Pod/provider"] = map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": map[string]any{"name": "provider", "namespace": "ns", "deletionTimestamp": "2026-09-21T00:00:00Z", "labels": map[string]any{"app.kubernetes.io/name": "test-mecatl-execution"}}} + }, want: "wait for all execution provider Pods"}, + {name: "reserved capacity without CR", mutate: func(m map[string]map[string]any) { + delete(m, "ExecutionEnvironment/exec-test") + delete(m, authority) + }, want: "bootstrap is forbidden"}, + {name: "surviving pod without CR", mutate: func(m map[string]map[string]any) { + delete(m, "ExecutionEnvironment/exec-test") + delete(m, authority) + delete(m, capacity) + m["Pod/executor"] = map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": map[string]any{"name": "executor", "namespace": "ns", "labels": map[string]any{"execution.mecatl.dev/environment": "exec-test"}}} + }, want: "bootstrap is forbidden"}, + {name: "surviving PVC without CR", mutate: func(m map[string]map[string]any) { + delete(m, "ExecutionEnvironment/exec-test") + delete(m, authority) + delete(m, capacity) + m["PersistentVolumeClaim/workspace"] = map[string]any{"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": map[string]any{"name": "workspace", "namespace": "ns", "labels": map[string]any{"execution.mecatl.dev/environment": "exec-test"}}} + }, want: "bootstrap is forbidden"}, + {name: "missing lifetime configuration", mutate: func(m map[string]map[string]any) { + delete(m["ConfigMap/test-mecatl-execution-profiles"]["data"].(map[string]any), "lifetime.json") + }, want: "missing lifetime.json history"}, + {name: "missing authority", mutate: func(m map[string]map[string]any) { delete(m, authority) }, want: "bootstrap is forbidden"}, + {name: "empty authority", mutate: func(m map[string]map[string]any) { m[authority]["data"] = map[string]any{} }, want: "bootstrap is forbidden"}, + {name: "missing state", mutate: func(m map[string]map[string]any) { m[authority]["data"] = map[string]any{"other": "value"} }, want: "state.json"}, + {name: "missing capacity", mutate: func(m map[string]map[string]any) { delete(m, capacity) }, want: "bootstrap is forbidden"}, + {name: "missing deny", mutate: func(m map[string]map[string]any) { + delete(m, "NetworkPolicy/test-mecatl-execution-workload-default-deny") + }, want: "existing workload NetworkPolicies"}, + {name: "foreign release", mutate: func(m map[string]map[string]any) { + u := &unstructured.Unstructured{Object: m[capacity]} + a := u.GetAnnotations() + a["meta.helm.sh/release-name"] = "foreign" + u.SetAnnotations(a) + }, want: "foreign or ambiguous"}, + {name: "foreign namespace", mutate: func(m map[string]map[string]any) { + u := &unstructured.Unstructured{Object: m[authority]} + a := u.GetAnnotations() + a["meta.helm.sh/release-namespace"] = "foreign" + u.SetAnnotations(a) + }, want: "foreign or ambiguous"}, + {name: "unmanaged", mutate: func(m map[string]map[string]any) { + u := &unstructured.Unstructured{Object: m[authority]} + u.SetLabels(nil) + }, want: "foreign or ambiguous"}, + {name: "removed profile policy", args: []string{"--set-json", "networkPolicy.workloadProfiles={}"}, want: "configuration is incompatible"}, + {name: "changed egress", args: []string{"--set", "networkPolicy.workloadProfiles.go.egress[0].cidr=10.3.0.0/16"}, want: "configuration is incompatible"}, + {name: "changed profile", args: []string{"--set", "profiles.go.maxEnvironments=30"}, want: "configuration is incompatible"}, + {name: "changed fullname", args: []string{"--set", "fullnameOverride=other"}, want: "bootstrap is forbidden"}, + {name: "missing profiles with orphan policies", mutate: func(m map[string]map[string]any) { + delete(m, "ExecutionEnvironment/exec-test") + delete(m, "ConfigMap/test-mecatl-execution-profiles") + delete(m, capacity) + }, want: "original release profiles"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objects := make(map[string]map[string]any, len(retained)) + for key, obj := range retained { + objects[key] = (&unstructured.Unstructured{Object: obj}).DeepCopy().Object + } + if tt.mutate != nil { + tt.mutate(objects) + } + _, err := renderLifetime(t, objects, tt.args...) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("render error=%v, want %q", err, tt.want) + } + }) + } +} + +func renderLifetime(t *testing.T, objects map[string]map[string]any, extra ...string) (map[string]map[string]any, error) { + t.Helper() + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is required for real chart rendering") + } + resources := map[string][]map[string]any{} + for gv, kinds := range map[string][]string{ + "v1": {"ConfigMap", "Pod", "PersistentVolumeClaim", "ResourceQuota", "LimitRange", "ServiceAccount", "Service"}, + "execution.mecatl.dev/v1alpha1": {"ExecutionEnvironment"}, + "networking.k8s.io/v1": {"NetworkPolicy"}, + "apps/v1": {"Deployment"}, + "policy/v1": {"PodDisruptionBudget"}, + "rbac.authorization.k8s.io/v1": {"Role", "RoleBinding", "ClusterRole", "ClusterRoleBinding"}, + } { + for _, kind := range kinds { + plural := strings.ToLower(kind) + "s" + if kind == "NetworkPolicy" { + plural = "networkpolicies" + } + resources[gv] = append(resources[gv], map[string]any{"name": plural, "kind": kind, "namespaced": !strings.HasPrefix(kind, "Cluster")}) + } + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var response any + switch r.URL.Path { + case "/version": + response = map[string]any{"major": "1", "minor": "35", "gitVersion": "v1.35.0"} + case "/api": + response = map[string]any{"kind": "APIVersions", "apiVersion": "v1", "versions": []string{"v1"}} + case "/apis": + groups := []any{} + for _, group := range []string{"execution.mecatl.dev", "networking.k8s.io", "apps", "policy", "rbac.authorization.k8s.io"} { + version := "v1" + if group == "execution.mecatl.dev" { + version = "v1alpha1" + } + gv := map[string]any{"groupVersion": group + "/" + version, "version": version} + groups = append(groups, map[string]any{"name": group, "versions": []any{gv}, "preferredVersion": gv}) + } + response = map[string]any{"kind": "APIGroupList", "apiVersion": "v1", "groups": groups} + default: + for gv, rs := range resources { + prefix := "/apis/" + gv + if gv == "v1" { + prefix = "/api/v1" + } + if r.URL.Path == prefix { + response = map[string]any{"kind": "APIResourceList", "apiVersion": "v1", "groupVersion": gv, "resources": rs} + break + } + for _, resource := range rs { + base := prefix + "/namespaces/ns/" + resource["name"].(string) + if r.URL.Path == base { + items := []any{} + for _, obj := range objects { + if obj["kind"] == resource["kind"] { + items = append(items, obj) + } + } + response = map[string]any{"kind": resource["kind"].(string) + "List", "apiVersion": gv, "items": items} + } else if strings.HasPrefix(r.URL.Path, base+"/") { + if obj := objects[resource["kind"].(string)+"/"+strings.TrimPrefix(r.URL.Path, base+"/")]; obj != nil { + response = obj + } + } + } + } + } + if response == nil { + w.WriteHeader(http.StatusNotFound) + response = map[string]any{"kind": "Status", "apiVersion": "v1", "status": "Failure", "reason": "NotFound", "code": 404} + } + if err := json.NewEncoder(w).Encode(response); err != nil { + t.Error(err) + } + })) + defer server.Close() + // Repo-local scratch: synthetic kubeconfig has no credentials. + if err := os.MkdirAll("../../../.scratch", 0o700); err != nil { + t.Fatal(err) + } + dir, err := os.MkdirTemp("../../../.scratch", "chart-lifetime-") + if err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, "kubeconfig") + body := fmt.Sprintf("apiVersion: v1\nkind: Config\nclusters:\n- name: offline\n cluster:\n server: %s\ncontexts:\n- name: offline\n context:\n cluster: offline\n namespace: ns\ncurrent-context: offline\n", server.URL) + if err := os.WriteFile(config, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + args := []string{"template", "test", ".", "--dry-run=server", "--disable-openapi-validation", "--kubeconfig", config, "--kube-context", "offline", "--namespace", "ns", "-f", "../../mecatl-execution-kind/execution-values.yaml", "--set-string", "provider.image=example.invalid/provider@sha256:" + strings.Repeat("a", 64), "--set", "provider.securitySecretName=synthetic", "--set-literal", "provider.securityManifest={}", "--set-json", `networkPolicy.workloadProfiles={"go":{"egress":[{"cidr":"10.2.0.0/16","ports":[{"port":8080}]}]}}`} + cmd := exec.CommandContext(ctx, "helm", append(args, extra...)...) + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + dir, "KUBECONFIG=" + config} + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("helm rendering: %w: %s", err, stderr.String()) + } + result := map[string]map[string]any{} + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(out), 4096) + for { + var obj map[string]any + err := decoder.Decode(&obj) + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if len(obj) == 0 { + continue + } + u := &unstructured.Unstructured{Object: obj} + result[u.GetKind()+"/"+u.GetName()] = obj + } + return result, nil +} diff --git a/deploy/helm/mecatl-execution/templates/_helpers.tpl b/deploy/helm/mecatl-execution/templates/_helpers.tpl new file mode 100644 index 0000000000..bee4eefeb5 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/_helpers.tpl @@ -0,0 +1,26 @@ +{{- define "mecatl-execution.fullname" -}} +{{- default (printf "%s-mecatl-execution" .Release.Name) .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Retained resources are adoptable only by their original Helm identity. */}} +{{- define "mecatl-execution.retainedMetadata" -}} +{{- $live := lookup .apiVersion .kind .root.Release.Namespace .name -}} +{{- if $live -}} +{{- if or $live.metadata.deletionTimestamp (ne (dig "app.kubernetes.io/managed-by" "" ($live.metadata.labels | default dict)) "Helm") (ne (dig "meta.helm.sh/release-name" "" ($live.metadata.annotations | default dict)) .root.Release.Name) (ne (dig "meta.helm.sh/release-namespace" "" ($live.metadata.annotations | default dict)) .root.Release.Namespace) -}} +{{- fail (printf "retained %s %s has foreign or ambiguous Helm ownership" .kind .name) -}} +{{- end -}} +{{- end -}} +name: {{ .name }} +namespace: {{ .root.Release.Namespace }} +labels: + app.kubernetes.io/managed-by: Helm +annotations: + helm.sh/resource-policy: keep + meta.helm.sh/release-name: {{ .root.Release.Name | quote }} + meta.helm.sh/release-namespace: {{ .root.Release.Namespace | quote }} +{{- end -}} + +{{/* Frozen workload identity prevents orphan allow policies from widening egress. */}} +{{- define "mecatl-execution.lifetimeConfig" -}} +{{- dict "profiles" .Values.profiles "networkPolicy" .Values.networkPolicy "securitySecretName" .Values.provider.securitySecretName | toJson -}} +{{- end -}} diff --git a/deploy/helm/mecatl-execution/templates/lifetime-check.yaml b/deploy/helm/mecatl-execution/templates/lifetime-check.yaml new file mode 100644 index 0000000000..592a7cdff3 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/lifetime-check.yaml @@ -0,0 +1,59 @@ +{{- $root := . -}} +{{- $name := include "mecatl-execution.fullname" . -}} +{{- $provider := lookup "apps/v1" "Deployment" .Release.Namespace $name -}} +{{- if gt (int (dig "spec" "replicas" 0 $provider)) 0 -}} +{{- fail "quiesce the execution provider (scale to zero and wait for its Pods to disappear) before upgrading" -}} +{{- end -}} +{{- $profiles := lookup "v1" "ConfigMap" .Release.Namespace (printf "%s-profiles" $name) -}} +{{- if $profiles -}} +{{- if not (dig "lifetime.json" "" ($profiles.data | default dict)) -}} +{{- fail "retained execution profiles are missing lifetime.json history; automatic adoption is unsupported, recover trusted retained history before upgrading" -}} +{{- end -}} +{{- if ne (dig "lifetime.json" "" ($profiles.data | default dict)) (include "mecatl-execution.lifetimeConfig" .) -}} +{{- fail "retained execution profiles/network configuration is incompatible; preserve the original lifetime configuration" -}} +{{- end -}} +{{- end -}} +{{/* A reserved slot can precede CR publication; no CR is not bootstrap proof. */}} +{{- $capacity := lookup "v1" "ConfigMap" .Release.Namespace "mecatl-execution-profile-allocations" -}} +{{- $retained := not (empty $capacity.data) -}} +{{- range (lookup "execution.mecatl.dev/v1alpha1" "ExecutionEnvironment" .Release.Namespace "").items -}} +{{- $retained = true -}} +{{- end -}} +{{- range $kind := list "Pod" "PersistentVolumeClaim" -}} +{{- range (lookup "v1" $kind $root.Release.Namespace "").items -}} +{{- if and (eq $kind "Pod") (eq (dig "app.kubernetes.io/name" "" (.metadata.labels | default dict)) $name) -}} +{{- fail "wait for all execution provider Pods to disappear before upgrading" -}} +{{- end -}} +{{- if hasKey (.metadata.labels | default dict) "execution.mecatl.dev/environment" -}} +{{- $retained = true -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- if $retained -}} +{{- range $cm := list (printf "%s-profiles" $name) (printf "%s-security-manifest" $name) (printf "%s-security-authority" $name) "mecatl-execution-profile-allocations" -}} +{{- $live := lookup "v1" "ConfigMap" $root.Release.Namespace $cm -}} +{{- if not ($live.data | default dict) -}} +{{- fail (printf "retained execution allocations require nonempty %s; authority/capacity bootstrap is forbidden" $cm) -}} +{{- end -}} +{{- end -}} +{{- $authority := lookup "v1" "ConfigMap" .Release.Namespace (printf "%s-security-authority" $name) -}} +{{- if not (dig "state.json" "" ($authority.data | default dict)) -}} +{{- fail "retained execution allocations require security authority state.json; bootstrap is forbidden" -}} +{{- end -}} +{{- $policies := list (printf "%s-workload-default-deny" $name) -}} +{{- range $profile, $_ := .Values.networkPolicy.workloadProfiles -}} +{{- $policies = append $policies (printf "%s-profile-%s" $name (sha256sum $profile | trunc 16)) -}} +{{- end -}} +{{- range $policy := $policies -}} +{{- if not (lookup "networking.k8s.io/v1" "NetworkPolicy" $root.Release.Namespace $policy) -}} +{{- fail "retained execution allocations require their existing workload NetworkPolicies" -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- range (lookup "networking.k8s.io/v1" "NetworkPolicy" .Release.Namespace "").items -}} +{{- if or (hasKey (.spec.podSelector.matchLabels | default dict) "execution.mecatl.dev/profile") (hasSuffix "-workload-default-deny" .metadata.name) -}} +{{- if not $profiles -}} +{{- fail "retained workload policies require the original release profiles ConfigMap" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/deploy/helm/mecatl-execution/templates/network-policy.yaml b/deploy/helm/mecatl-execution/templates/network-policy.yaml new file mode 100644 index 0000000000..43f77f268b --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/network-policy.yaml @@ -0,0 +1,101 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "mecatl-execution.fullname" . }}-provider +spec: + podSelector: + matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + policyTypes: [Ingress, Egress] + ingress: + - from: +{{- range .Values.provider.clientIngressSelectors }} + - namespaceSelector: + matchLabels: +{{ toYaml .namespaceLabels | indent 12 }} + podSelector: + matchLabels: +{{ toYaml .podLabels | indent 12 }} +{{- end }} + ports: + - {protocol: TCP, port: {{ .Values.service.port }}} + egress: +{{- range .Values.provider.apiServerCIDRs }} + - to: [{ipBlock: {cidr: {{ . | quote }}}}] + ports: +{{- range $.Values.networkPolicy.apiServerPorts }} + - {protocol: TCP, port: {{ . }}} +{{- end }} +{{- end }} +{{- range .Values.provider.dnsCIDRs }} + - to: [{ipBlock: {cidr: {{ . | quote }}}}] + ports: +{{- range $.Values.networkPolicy.dnsPorts }} + - {protocol: UDP, port: {{ . }}} + - {protocol: TCP, port: {{ . }}} +{{- end }} +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + {{- include "mecatl-execution.retainedMetadata" (dict "root" . "apiVersion" "networking.k8s.io/v1" "kind" "NetworkPolicy" "name" (printf "%s-workload-default-deny" (include "mecatl-execution.fullname" .))) | nindent 2 }} +spec: + podSelector: + matchExpressions: + - {key: execution.mecatl.dev/environment, operator: Exists} + policyTypes: [Ingress, Egress] + ingress: [] + egress: [] +{{- range $name, $profile := .Values.networkPolicy.workloadProfiles }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + {{- include "mecatl-execution.retainedMetadata" (dict "root" $ "apiVersion" "networking.k8s.io/v1" "kind" "NetworkPolicy" "name" (printf "%s-profile-%s" (include "mecatl-execution.fullname" $) (sha256sum $name | trunc 16))) | nindent 2 }} +spec: + podSelector: + matchLabels: + execution.mecatl.dev/profile: {{ sha256sum $name | trunc 16 }} + policyTypes: [Egress] + egress: +{{- range $profile.egress }} + - to: [{ipBlock: {cidr: {{ .cidr | quote }}}}] + ports: +{{- range .ports }} + - {protocol: {{ default "TCP" .protocol }}, port: {{ .port }}} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +--- +{{- if .Values.resourceGovernance.enabled }} +apiVersion: v1 +kind: ResourceQuota +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + hard: + pods: {{ .Values.resourceGovernance.pods | quote }} + persistentvolumeclaims: {{ .Values.resourceGovernance.persistentVolumeClaims | quote }} + count/executionenvironments.execution.mecatl.dev: {{ .Values.resourceGovernance.executionEnvironments | quote }} + requests.cpu: {{ .Values.resourceGovernance.requestsCPU | quote }} + requests.memory: {{ .Values.resourceGovernance.requestsMemory | quote }} + requests.storage: {{ .Values.resourceGovernance.requestsStorage | quote }} + requests.ephemeral-storage: {{ .Values.resourceGovernance.requestsEphemeralStorage | quote }} + limits.cpu: {{ .Values.resourceGovernance.limitsCPU | quote }} + limits.memory: {{ .Values.resourceGovernance.limitsMemory | quote }} + limits.ephemeral-storage: {{ .Values.resourceGovernance.limitsEphemeralStorage | quote }} +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + limits: + - type: Container + defaultRequest: {cpu: 50m, memory: 64Mi, ephemeral-storage: 32Mi} + default: {cpu: "1", memory: 1Gi, ephemeral-storage: 1Gi} + - type: PersistentVolumeClaim + max: {storage: {{ .Values.resourceGovernance.requestsStorage | quote }}} +{{- end }} diff --git a/deploy/helm/mecatl-execution/templates/pdb.yaml b/deploy/helm/mecatl-execution/templates/pdb.yaml new file mode 100644 index 0000000000..43ff755726 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/pdb.yaml @@ -0,0 +1,10 @@ +{{- if gt (int .Values.provider.replicas) 1 }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + minAvailable: 1 + selector: + matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} +{{- end }} diff --git a/deploy/helm/mecatl-execution/templates/provider.yaml b/deploy/helm/mecatl-execution/templates/provider.yaml new file mode 100644 index 0000000000..f17f92ee45 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/provider.yaml @@ -0,0 +1,113 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + {{- include "mecatl-execution.retainedMetadata" (dict "root" . "apiVersion" "v1" "kind" "ConfigMap" "name" (printf "%s-profiles" (include "mecatl-execution.fullname" .))) | nindent 2 }} +data: + lifetime.json: {{ include "mecatl-execution.lifetimeConfig" . | quote }} + profiles.yaml: | + profiles: +{{ toYaml .Values.profiles | indent 6 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + {{- include "mecatl-execution.retainedMetadata" (dict "root" . "apiVersion" "v1" "kind" "ConfigMap" "name" (printf "%s-security-manifest" (include "mecatl-execution.fullname" .))) | nindent 2 }} +data: + manifest.json: | +{{ required "provider.securityManifest is required" .Values.provider.securityManifest | indent 4 }} +--- +{{- range $name := list (printf "%s-security-authority" (include "mecatl-execution.fullname" .)) "mecatl-execution-profile-allocations" }} +apiVersion: v1 +kind: ConfigMap +metadata: + {{- include "mecatl-execution.retainedMetadata" (dict "root" $ "apiVersion" "v1" "kind" "ConfigMap" "name" $name) | nindent 2 }} +data: {{ (lookup "v1" "ConfigMap" $.Release.Namespace $name).data | default dict | toJson }} +--- +{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + replicas: {{ .Values.provider.replicas }} + strategy: + type: RollingUpdate + rollingUpdate: {maxUnavailable: 0, maxSurge: 1} + selector: + matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + template: + metadata: + labels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + spec: + serviceAccountName: {{ include "mecatl-execution.fullname" . }} + automountServiceAccountToken: true + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: {type: RuntimeDefault} + containers: + - name: provider + image: {{ required "provider.image must be digest-pinned" .Values.provider.image | quote }} + imagePullPolicy: {{ .Values.provider.imagePullPolicy }} + args: + - --namespace={{ .Release.Namespace }} + - --profiles=/etc/mecatl-execution/profiles.yaml + - --grant-keyring-manifest=/etc/mecatl-execution/security/manifest.json + - --grant-key-directory=/etc/mecatl-execution/security + - --security-authority-configmap={{ include "mecatl-execution.fullname" . }}-security-authority + - --security-reload-interval={{ .Values.provider.securityReloadInterval }} + - --max-concurrent-streams={{ .Values.provider.maxConcurrentStreams }} + - --max-concurrent-rpcs={{ .Values.provider.maxConcurrentRPCs }} + - --max-concurrent-rpcs-per-client={{ .Values.provider.maxConcurrentRPCsPerClient }} + - --health-listen=:{{ .Values.provider.healthPort }} + ports: + - {name: grpc, containerPort: 8443} + - {name: health, containerPort: {{ .Values.provider.healthPort }}} + startupProbe: + httpGet: {path: /ready, port: health} + failureThreshold: 30 + periodSeconds: 2 + readinessProbe: + httpGet: {path: /ready, port: health} + periodSeconds: 5 + livenessProbe: + httpGet: {path: /live, port: health} + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: ["ALL"]} + resources: +{{ toYaml .Values.provider.resources | indent 10 }} + volumeMounts: + - {name: profiles, mountPath: /etc/mecatl-execution/profiles.yaml, subPath: profiles.yaml, readOnly: true} + - {name: security, mountPath: /etc/mecatl-execution/security, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: profiles + configMap: {name: {{ include "mecatl-execution.fullname" . }}-profiles} + - name: security + projected: + sources: + - configMap: + name: {{ include "mecatl-execution.fullname" . }}-security-manifest + - secret: + name: {{ required "provider.securitySecretName is required" .Values.provider.securitySecretName }} + - name: tmp + emptyDir: {sizeLimit: 64Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + selector: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + ports: + - {name: grpc, port: {{ .Values.service.port }}, targetPort: grpc} diff --git a/deploy/helm/mecatl-execution/templates/rbac.yaml b/deploy/helm/mecatl-execution/templates/rbac.yaml new file mode 100644 index 0000000000..ef3dab3635 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/rbac.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "mecatl-execution.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "mecatl-execution.fullname" . }} +rules: +- apiGroups: ["execution.mecatl.dev"] + resources: ["executionenvironments"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["execution.mecatl.dev"] + resources: ["executionenvironments/status"] + verbs: ["get", "update", "patch"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: + - {{ printf "%s-security-authority" (include "mecatl-execution.fullname" .) | quote }} + - "mecatl-execution-profile-allocations" + verbs: ["get", "update"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "mecatl-execution.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "mecatl-execution.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "mecatl-execution.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ printf "%s-%s-profile-preflight" .Release.Namespace (include "mecatl-execution.fullname" .) }} +rules: +- apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + resourceNames: +{{- range $name, $profile := .Values.profiles }} + - {{ $profile.runtimeClassName | quote }} +{{- end }} + verbs: ["get"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + resourceNames: +{{- range $name, $profile := .Values.profiles }} + - {{ $profile.storageClass | quote }} +{{- end }} + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ printf "%s-%s-profile-preflight" .Release.Namespace (include "mecatl-execution.fullname" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "mecatl-execution.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ printf "%s-%s-profile-preflight" .Release.Namespace (include "mecatl-execution.fullname" .) }} diff --git a/deploy/helm/mecatl-execution/values.schema.json b/deploy/helm/mecatl-execution/values.schema.json new file mode 100644 index 0000000000..fc37d2ea8a --- /dev/null +++ b/deploy/helm/mecatl-execution/values.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": ["provider", "service", "profiles", "networkPolicy", "resourceGovernance"], + "properties": { + "fullnameOverride": {"type":"string"}, + "provider": { + "type":"object", "additionalProperties":false, + "required":["image","replicas","securitySecretName","securityManifest","maxConcurrentStreams","maxConcurrentRPCs","maxConcurrentRPCsPerClient","apiServerCIDRs","dnsCIDRs","resources"], + "properties": { + "image":{"type":"string","pattern":"@sha256:[0-9a-fA-F]{64}$"}, + "imagePullPolicy":{"enum":["Always","IfNotPresent","Never"]}, + "replicas":{"type":"integer","minimum":2,"maximum":32}, + "securitySecretName":{"type":"string","minLength":1}, + "securityManifest":{"type":"string","minLength":2}, + "securityReloadInterval":{"type":"string","pattern":"^[1-9][0-9]*(ms|s|m)$"}, + "maxConcurrentStreams":{"type":"integer","minimum":1,"maximum":1024}, + "maxConcurrentRPCs":{"type":"integer","minimum":1,"maximum":4096}, + "maxConcurrentRPCsPerClient":{"type":"integer","minimum":1,"maximum":4096}, + "healthPort":{"type":"integer","minimum":1,"maximum":65535}, + "clientIngressSelectors":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["namespaceLabels","podLabels"],"properties":{"namespaceLabels":{"type":"object","minProperties":1,"additionalProperties":{"type":"string"}},"podLabels":{"type":"object","minProperties":1,"additionalProperties":{"type":"string"}}}}}, + "apiServerCIDRs":{"type":"array","minItems":1,"uniqueItems":true,"items":{"type":"string","pattern":"^[0-9a-fA-F:.]+/[0-9]+$"}}, + "dnsCIDRs":{"type":"array","minItems":1,"uniqueItems":true,"items":{"type":"string","pattern":"^[0-9a-fA-F:.]+/[0-9]+$"}}, + "resources":{"type":"object","required":["requests","limits"],"properties":{"requests":{"type":"object","required":["cpu","memory","ephemeral-storage"]},"limits":{"type":"object","required":["cpu","memory","ephemeral-storage"]}}} + } + }, + "service":{"type":"object","additionalProperties":false,"required":["port"],"properties":{"port":{"type":"integer","minimum":1,"maximum":65535}}}, + "networkPolicy":{"type":"object","additionalProperties":false,"required":["enabled","dnsPorts","apiServerPorts","workloadProfiles"],"properties":{"enabled":{"const":true},"dnsPorts":{"type":"array","minItems":1,"items":{"type":"integer","minimum":1,"maximum":65535}},"apiServerPorts":{"type":"array","minItems":1,"items":{"type":"integer","minimum":1,"maximum":65535}},"workloadProfiles":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"required":["egress"],"properties":{"egress":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["cidr","ports"],"properties":{"cidr":{"type":"string","pattern":"^[0-9a-fA-F:.]+/[0-9]+$"},"ports":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["port"],"properties":{"port":{"type":"integer","minimum":1,"maximum":65535},"protocol":{"enum":["TCP","UDP"]}}}}}}}}}}}}, + "resourceGovernance":{"type":"object","additionalProperties":false,"required":["enabled","pods","persistentVolumeClaims","executionEnvironments","requestsCPU","requestsMemory","requestsStorage","requestsEphemeralStorage","limitsCPU","limitsMemory","limitsEphemeralStorage"],"properties":{"enabled":{"const":true},"pods":{"type":"string","minLength":1},"persistentVolumeClaims":{"type":"string","minLength":1},"executionEnvironments":{"type":"string","minLength":1},"requestsCPU":{"type":"string","minLength":1},"requestsMemory":{"type":"string","minLength":1},"requestsStorage":{"type":"string","minLength":1},"requestsEphemeralStorage":{"type":"string","minLength":1},"limitsCPU":{"type":"string","minLength":1},"limitsMemory":{"type":"string","minLength":1},"limitsEphemeralStorage":{"type":"string","minLength":1}}}, + "profiles":{"type":"object","minProperties":1,"additionalProperties":{"type":"object","additionalProperties":false,"required":["image","storageClass","storageSize","cpuRequest","memoryRequest","cpuLimit","memoryLimit","ephemeralStorageRequest","ephemeralStorageLimit","tmpSizeLimit","runtimeClassName","maxFileBytes","maxCommandBytes","maxCommandDuration","maxEnvironments"],"properties":{"image":{"type":"string","pattern":"@sha256:[0-9a-fA-F]{64}$"},"storageClass":{"type":"string","minLength":1},"storageSize":{"type":"string","minLength":1},"cpuRequest":{"type":"string","minLength":1},"memoryRequest":{"type":"string","minLength":1},"cpuLimit":{"type":"string","minLength":1},"memoryLimit":{"type":"string","minLength":1},"ephemeralStorageRequest":{"type":"string","minLength":1},"ephemeralStorageLimit":{"type":"string","minLength":1},"tmpSizeLimit":{"type":"string","minLength":1},"runtimeClassName":{"type":"string","minLength":1},"maxFileBytes":{"type":"integer","minimum":1,"maximum":5242880},"maxCommandBytes":{"type":"integer","minimum":1,"maximum":1048576},"maxCommandDuration":{"type":"string","pattern":"^[1-9][0-9]*(ms|s|m)$"},"maxEnvironments":{"type":"integer","minimum":1,"maximum":10000}}}} + } +} diff --git a/deploy/helm/mecatl-execution/values.yaml b/deploy/helm/mecatl-execution/values.yaml new file mode 100644 index 0000000000..d97077f19f --- /dev/null +++ b/deploy/helm/mecatl-execution/values.yaml @@ -0,0 +1,58 @@ +fullnameOverride: "" +provider: + image: "" + imagePullPolicy: IfNotPresent + replicas: 2 + securitySecretName: "" + securityManifest: "" + securityReloadInterval: 2s + maxConcurrentStreams: 64 + maxConcurrentRPCs: 128 + maxConcurrentRPCsPerClient: 32 + healthPort: 8081 + clientIngressSelectors: [] + apiServerCIDRs: [] + dnsCIDRs: [] + resources: + requests: {cpu: 100m, memory: 128Mi, ephemeral-storage: 64Mi} + limits: {cpu: "1", memory: 512Mi, ephemeral-storage: 256Mi} + # Deprecated single-key/TLS settings are intentionally unsupported. +service: + port: 8443 +networkPolicy: + enabled: true + dnsPorts: [53] + apiServerPorts: [443] + workloadProfiles: {} +resourceGovernance: + enabled: true + pods: "200" + persistentVolumeClaims: "100" + executionEnvironments: "100" + requestsCPU: "20" + requestsMemory: 40Gi + requestsStorage: 500Gi + requestsEphemeralStorage: 40Gi + limitsCPU: "40" + limitsMemory: 80Gi + limitsEphemeralStorage: 80Gi +profiles: {} + +# Example profile (all fields are required): +# profiles: +# go: +# image: registry.example/workload@sha256: +# storageClass: standard +# storageSize: 2Gi +# cpuRequest: 100m +# memoryRequest: 128Mi +# cpuLimit: "1" +# memoryLimit: 1Gi +# ephemeralStorageRequest: 64Mi +# ephemeralStorageLimit: 1Gi +# tmpSizeLimit: 256Mi +# runtimeClassName: gvisor +# maxFileBytes: 5242880 +# maxCommandBytes: 1048576 +# maxCommandDuration: 5m +# maxEnvironments: 100 diff --git a/deploy/mecatl-execution-kind/ci_test.go b/deploy/mecatl-execution-kind/ci_test.go new file mode 100644 index 0000000000..c2509f3a69 --- /dev/null +++ b/deploy/mecatl-execution-kind/ci_test.go @@ -0,0 +1,284 @@ +package executionkind_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/goccy/go-yaml" +) + +type workflowStep struct { + ID string `yaml:"id"` + Timeout int `yaml:"timeout-minutes"` + Run string `yaml:"run"` + If string `yaml:"if"` + Env map[string]string `yaml:"env"` + With map[string]any `yaml:"with"` +} + +func nativeSteps(t *testing.T) map[string]workflowStep { + t.Helper() + data, err := os.ReadFile("../../.github/workflows/e2e-live.yml") + if err != nil { + t.Fatal(err) + } + var workflow struct { + Jobs map[string]struct { + If string `yaml:"if"` + Steps []workflowStep `yaml:"steps"` + Timeout int `yaml:"timeout-minutes"` + } `yaml:"jobs"` + } + if err := yaml.Unmarshal(data, &workflow); err != nil { + t.Fatal(err) + } + job := workflow.Jobs["native-execution-live"] + if strings.Join(strings.Fields(job.If), " ") != "github.repository == 'stacklok/mecatl' && github.event_name == 'workflow_dispatch' && inputs.native_execution" || job.Timeout != 100 { + t.Fatal("native job must remain explicitly dispatched, repo-gated, and bounded") + } + steps := make(map[string]workflowStep) + total := 0 + for _, step := range job.Steps { + if step.Timeout <= 0 { + t.Fatal("every native step needs an explicit ceiling") + } + total += step.Timeout + if step.ID != "" { + steps[step.ID] = step + } + if strings.Contains(step.Run, "${{") { + t.Fatal("workflow expressions must cross into shell through env") + } + if step.With["cache"] == true { + t.Fatal("native credential job must not save a cache") + } + } + if total > job.Timeout-2 || steps["cleanup"].Timeout < 10 { + t.Fatal("stage ceilings must reserve cleanup and job overhead") + } + for _, id := range []string{"live", "cleanup"} { + step := steps[id] + if step.Env["MECATL_EXECUTION_QUAL_STATE"] != "${{ steps.production.outputs.state }}" || step.Env["NATIVE_CLUSTER"] != "${{ steps.production.outputs.cluster }}" || strings.Contains(step.Run, "/current") { + t.Fatal("downstream ownership must use production outputs, never current") + } + } + if steps["production"].Env["MECATL_EXECUTION_QUAL_CI"] != "0" || len(steps["credential"].Env) != 1 || steps["credential"].Env["OPENROUTER_API_KEY"] != "${{ secrets.OPENROUTER_API_KEY }}" { + t.Fatal("production retention or step-scoped credential wiring drifted") + } + if steps["credential"].If != "success() && steps.production.outcome == 'success'" || steps["live"].If != "success() && steps.production.outcome == 'success' && steps.credential.outcome == 'success'" || steps["cleanup"].If != "always() && steps.prepare.outcome == 'success'" { + t.Fatal("qualification must depend on actual production success and always clean up") + } + return steps +} + +func writeFixture(t *testing.T, path, body string, mode os.FileMode) { + t.Helper() + if err := os.WriteFile(path, []byte(body), mode); err != nil { + t.Fatal(err) + } +} + +func runStep(t *testing.T, root, script string, env ...string) ([]byte, error) { + t.Helper() + cmd := exec.CommandContext(t.Context(), "bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", script) + cmd.Dir = root + cmd.Env = append([]string{"PATH=" + os.Getenv("PATH"), "HOME=" + root}, env...) + return cmd.CombinedOutput() +} + +func TestNativeWorkflowCommitAndCredentialBoundary(t *testing.T) { + steps := nativeSteps(t) + for _, expected := range []string{"", "reviewed", "wrong", "$(touch injected)"} { + t.Run("sha="+expected, func(t *testing.T) { + root := t.TempDir() + bin := filepath.Join(root, "bin") + if err := os.Mkdir(bin, 0o700); err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(bin, "git"), "#!/bin/sh\nprintf 'reviewed\\n'\n", 0o700) + dir := filepath.Join(root, ".scratch", "ci") + out, err := runStep(t, root, steps["prepare"].Run, "PATH="+bin+":"+os.Getenv("PATH"), "NATIVE_CI_DIR="+dir, "EVENT_SHA=reviewed", "EXPECTED_SHA="+expected, "GITHUB_STEP_SUMMARY="+filepath.Join(root, "summary")) + wantOK := expected == "" || expected == "reviewed" + if (err == nil) != wantOK { + t.Fatalf("SHA gate: %v: %s", err, out) + } + if _, err := os.Stat(filepath.Join(root, "injected")); !os.IsNotExist(err) { + t.Fatal("expected SHA was executed") + } + if !wantOK { + return + } + info, err := os.Stat(dir) + if err != nil || info.Mode().Perm() != 0o700 { + t.Fatal("CI directory is not private") + } + if _, err := runStep(t, root, steps["credential"].Run, "NATIVE_CI_DIR="+dir, "OPENROUTER_API_KEY="); err == nil { + t.Fatal("missing credential skipped successfully") + } + const fake = "synthetic-offline-key" + out, err = runStep(t, root, steps["credential"].Run+"\ntest -z \"${OPENROUTER_API_KEY+x}\"", "NATIVE_CI_DIR="+dir, "OPENROUTER_API_KEY="+fake) + if err != nil || strings.Contains(string(out), fake) { + t.Fatal("credential staging failed or disclosed synthetic credential") + } + info, err = os.Stat(filepath.Join(dir, "provider-key")) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatal("credential file is not private") + } + if _, err := runStep(t, root, steps["credential"].Run, "NATIVE_CI_DIR="+dir, "OPENROUTER_API_KEY=replacement"); err == nil { + t.Fatal("credential staging overwrote existing file") + } + data, err := os.ReadFile(filepath.Join(dir, "provider-key")) + if err != nil || string(data) != fake { + t.Fatal("existing synthetic credential changed") + } + }) + } +} + +func TestNativeWorkflowCleanupOwnership(t *testing.T) { + steps := nativeSteps(t) + for _, fault := range []string{"", "owner", "context", "label", "duplicate", "symlink", "delete", "list", "switched-pointer", "captured-cluster", "missing-kubeconfig", "missing-ownership", "missing-output", "partial-with-kubeconfig", "collector-error", "collector-timeout", "collector-error-delete", "collector-timeout-delete", "live-failed", "live-failed-preserved"} { + t.Run("fault="+fault, func(t *testing.T) { + collectorFails := strings.HasPrefix(fault, "collector-") + root := t.TempDir() + state := filepath.Join(root, ".scratch", "k8s-execution", "owned") + dir := filepath.Join(root, ".scratch", "ci") + bin := filepath.Join(root, "bin") + for _, path := range []string{state, dir, bin} { + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + } + const cluster = "mecatl-execution-qual-offline" + context := "kind-" + cluster + owner := "fixture" + if fault == "owner" { + owner = "someone-else" + } + if fault == "context" { + context = "ambient" + } + kubeconfig := filepath.Join(state, "kubeconfig") + if fault != "missing-kubeconfig" { + writeFixture(t, kubeconfig, "synthetic", 0o600) + } + ownership := "cluster=" + cluster + "\ncontext=" + context + "\nowner=" + owner + "\nruntime=docker\nprofile=production\nnamespace=execution-qualification\nkubeconfig=" + kubeconfig + "\n" + if fault == "duplicate" { + ownership += "owner=fixture\n" + } + if fault != "missing-ownership" { + writeFixture(t, filepath.Join(state, "ownership"), ownership, 0o600) + } + pointer := filepath.Join(filepath.Dir(state), "current") + writeFixture(t, pointer, state+"\n", 0o600) + if fault == "switched-pointer" { + foreign := filepath.Join(filepath.Dir(state), "foreign") + if err := os.Mkdir(foreign, 0o700); err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(foreign, "ownership"), strings.ReplaceAll(ownership, cluster, "mecatl-execution-qual-foreign"), 0o600) + writeFixture(t, pointer, foreign+"\n", 0o600) + } + capturedState, capturedCluster := state, cluster + if fault == "symlink" { + capturedState = filepath.Join(filepath.Dir(state), "alias") + if err := os.Symlink(state, capturedState); err != nil { + t.Fatal(err) + } + } + if fault == "captured-cluster" { + capturedCluster = "mecatl-execution-qual-other" + } + if fault == "missing-output" { + capturedState, capturedCluster = "", "" + } + writeFixture(t, filepath.Join(dir, "provider-key"), "synthetic", 0o600) + writeFixture(t, filepath.Join(state, "live-summary.json"), "{}\n", 0o600) + writeFixture(t, filepath.Join(bin, "kubectl"), "#!/bin/sh\nprintf '%s\\n' 'kind-"+cluster+"'\n", 0o700) + writeFixture(t, filepath.Join(bin, "docker"), "#!/bin/sh\nif [ \"$FAULT\" = label ]; then exit 1; fi\nprintf '%s\\n' '"+cluster+"'\n", 0o700) + writeFixture(t, filepath.Join(bin, "kind"), "#!/bin/sh\nif [ \"$1\" = delete ]; then\n test \"$*\" = 'delete cluster --name "+cluster+"' || exit 1\n printf '%s\\n' \"$*\" >> \"$MARKER.attempt\"\n case \"$FAULT\" in delete|collector-*-delete) exit 1 ;; esac\n printf deleted > \"$MARKER\"\nelse\n test \"$FAULT\" != list || exit 1\nfi\n", 0o700) + marker := filepath.Join(root, "deleted") + if fault == "delete" || fault == "list" || fault == "partial-with-kubeconfig" || collectorFails || fault == "live-failed" { + scripts := filepath.Join(root, "deploy/mecatl-execution-kind") + if err := os.MkdirAll(scripts, 0o700); err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(scripts, "collect-failure.sh"), "#!/bin/sh\ntest ! -e \"$MARKER.attempt\" || exit 1\nprintf collected > \"$MARKER.collection\"\ncase \"$FAULT\" in collector-error*) exit 17 ;; collector-timeout*) exit 124 ;; esac\nprintf '{\"kind\":\"collection\"}\\n' > \"$3\"\n", 0o700) + } + outcome := "success" + if strings.HasPrefix(fault, "missing-") || fault == "partial-with-kubeconfig" || fault == "delete" || fault == "list" || collectorFails { + outcome = "failure" + } + liveOutcome := "success" + if strings.HasPrefix(fault, "live-failed") { + liveOutcome = "failure" + } + if fault == "live-failed-preserved" { + writeFixture(t, filepath.Join(state, "live-diagnostics.jsonl"), "{\"kind\":\"create_stage\",\"stage\":\"bind_ensure\"}\n", 0o600) + writeFixture(t, filepath.Join(state, "live-diagnostics.status"), "complete\n", 0o600) + } + out, err := runStep(t, root, steps["cleanup"].Run, "PATH="+bin+":"+os.Getenv("PATH"), "NATIVE_CI_DIR="+dir, "GITHUB_WORKSPACE="+root, "USER=fixture", "PRODUCTION_OUTCOME="+outcome, "LIVE_OUTCOME="+liveOutcome, "MECATL_EXECUTION_QUAL_STATE="+capturedState, "NATIVE_CLUSTER="+capturedCluster, "FAULT="+fault, "MARKER="+marker) + wantOK := fault == "" || fault == "switched-pointer" || fault == "partial-with-kubeconfig" || fault == "collector-error" || fault == "collector-timeout" || strings.HasPrefix(fault, "live-failed") + if fault == "live-failed-preserved" { + data, err := os.ReadFile(filepath.Join(dir, "live-diagnostics.jsonl")) + if err != nil || !strings.Contains(string(data), "bind_ensure") { + t.Fatal("pre-restoration real evidence lost") + } + if _, err := os.Stat(filepath.Join(dir, "production-diagnostics.jsonl")); !os.IsNotExist(err) { + t.Fatal("mock fallback should not replace complete real evidence") + } + } + if collectorFails { + if !strings.Contains(string(out), "::warning::Bounded production diagnostics incomplete") { + t.Fatal("collector failure must warn") + } + if data, err := os.ReadFile(marker + ".collection"); err != nil || string(data) != "collected" { + t.Fatal("collector must run before deletion") + } + if data, err := os.ReadFile(marker + ".attempt"); err != nil || string(data) != "delete cluster --name "+cluster+"\n" { + t.Fatal("collector failure must not skip exact owned-cluster deletion") + } + } + if (err == nil) != wantOK { + t.Fatalf("cleanup result: %v: %s", err, out) + } + if _, err := os.Stat(filepath.Join(dir, "provider-key")); !os.IsNotExist(err) { + t.Fatal("CI credential was not removed independently") + } + if fault == "delete" || fault == "list" || fault == "partial-with-kubeconfig" || fault == "live-failed" { + if data, err := os.ReadFile(filepath.Join(dir, "production-diagnostics.jsonl")); err != nil || !strings.Contains(string(data), "collection") { + t.Fatal("failure evidence must be captured before cleanup, even when cleanup fails") + } + } + _, deleted := os.Stat(marker) + if (deleted == nil) != (wantOK || fault == "list") { + t.Fatal("wrong cluster deletion decision") + } + }) + } +} + +func TestLiveImageDigestUsesRuntimeSpecificInspect(t *testing.T) { + data, err := os.ReadFile("live.sh") + if err != nil { + t.Fatal(err) + } + start := strings.Index(string(data), "if [ \"$runtime\" = podman ]; then\n podman image exists") + end := strings.Index(string(data), "\nworkload_tag=") + if start < 0 || end < start { + t.Fatal("image build preparation missing") + } + for _, runtime := range []string{"docker", "podman"} { + t.Run(runtime, func(t *testing.T) { + root := t.TempDir() + writeFixture(t, filepath.Join(root, runtime), "#!/bin/sh\ncase \"$*\" in\n 'image exists docker.io/library/golang:1.27'|'image inspect docker.io/library/golang:1.27') exit 0 ;;\n 'image inspect docker.io/library/golang:1.27 --format {{.Digest}}') test \"$RUNTIME\" = podman || exit 1; printf 'sha256:fake' ;;\n 'image inspect docker.io/library/golang:1.27 --format {{index .RepoDigests 0}}') test \"$RUNTIME\" = docker || exit 1; printf 'golang@sha256:fake' ;;\n *) exit 1 ;;\nesac\n", 0o700) + out, err := runStep(t, root, string(data[start:end])+"\ntest \"$go_image\" = docker.io/library/golang@sha256:fake", "PATH="+root+":"+os.Getenv("PATH"), "runtime="+runtime, "RUNTIME="+runtime) + if err != nil { + t.Fatalf("digest selection: %v: %s", err, out) + } + }) + } +} diff --git a/deploy/mecatl-execution-kind/collect-failure.sh b/deploy/mecatl-execution-kind/collect-failure.sh new file mode 100644 index 0000000000..f464ef1da0 --- /dev/null +++ b/deploy/mecatl-execution-kind/collect-failure.sh @@ -0,0 +1,36 @@ +#!/bin/sh +set -eu +# Called only with the owned fixture's explicit kubeconfig/context. Each request +# is bounded; the caller also bounds the whole collector (including jq). +[ "$#" -eq 3 ] || exit 2 +kubeconfig=$1 +context=$2 +output=$3 +filters=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +umask 077 +set -C +exec 3> "$output" +failed=0 +# Collect logs first so a short termination grace still retains create progress. +# Drain every bounded response: no head/SIGPIPE and no raw log file, even locally. +# Fixed workloads in the already ownership-checked fixture namespace only. +for deployment in mecak8s mecatl-execution; do + if ! { + if ! kubectl --kubeconfig "$kubeconfig" --context "$context" --request-timeout=5s logs "deployment/$deployment" -n execution-qualification --all-containers=true --tail=1000 --limit-bytes=262144 2>/dev/null; then + printf '\n{"collector_unavailable":true}\n' + fi + } | jq -Rnc --arg source "$deployment" -f "$filters/create-stages.jq" >&3 2>/dev/null; then + failed=1 + fi +done +if ! { + for resource in executionenvironments pods persistentvolumeclaims resourcequotas events; do + if ! kubectl --kubeconfig "$kubeconfig" --context "$context" --request-timeout=5s get "$resource" -n execution-qualification -o json 2>/dev/null; then + printf '{"unavailable":true}\n' + fi + done +} | jq -cs -f "$filters/failure-evidence.jq" >&3 2>/dev/null; then + failed=1 + printf '{"kind":"collection","unavailable":true}\n' >&3 +fi +exit "$failed" diff --git a/deploy/mecatl-execution-kind/create-stages.jq b/deploy/mecatl-execution-kind/create-stages.jq new file mode 100644 index 0000000000..bf447fc0d8 --- /dev/null +++ b/deploy/mecatl-execution-kind/create-stages.jq @@ -0,0 +1,12 @@ +# No timestamps, text, stack, endpoint, owner, ref, grant or arbitrary keys survive. +def member($values): . as $v | ($values | index($v)) != null; +def count: type == "number" and . >= 0 and . <= 1000000000000 and floor == .; +[inputs | fromjson? | select(type == "object")] as $rows | +($rows | map(select(.msg == "remote create stage" and .level == "DEBUG") | + select(.stage | member(["http_handler","session_id_probe","bind_ensure","attach_poll","attach_poll_end","engine_factory","session_persist","reference_commit","http_response"])) | + select(.reason | member(["begin","ok","error","cancelled","deadline","returned","pending","ready","not_ready_retryable","not_ready_nonretryable","remote_retryable","remote_terminal","transport_error"])) | + select((.elapsed_ms | count) and (.calls | count)) | + select(.session | type == "string" and test("^([a-f0-9]{32})?$"))) | .[-512:][] | + {kind:"create_stage", source:$source, stage, reason, elapsed_ms, calls, session}), +{kind:"log_collection", source:$source, unavailable:any($rows[]; .collector_unavailable == true)}, +(if any($rows[]; .collector_unavailable == true) then "" | halt_error(1) else empty end) diff --git a/deploy/mecatl-execution-kind/execution-values.yaml b/deploy/mecatl-execution-kind/execution-values.yaml new file mode 100644 index 0000000000..a6ed5a8bc9 --- /dev/null +++ b/deploy/mecatl-execution-kind/execution-values.yaml @@ -0,0 +1,59 @@ +provider: + clientIngressSelectors: + - namespaceLabels: + kubernetes.io/metadata.name: execution-qualification + podLabels: + app.kubernetes.io/name: mecak8s + apiServerCIDRs: [172.16.0.0/12] + dnsCIDRs: [10.96.0.10/32] +profiles: + go: + image: placeholder.invalid/workload@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + storageClass: standard + storageSize: 1Gi + cpuRequest: 50m + memoryRequest: 64Mi + cpuLimit: "1" + memoryLimit: 512Mi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: qualification-runc + maxFileBytes: 5242880 + maxCommandBytes: 1048576 + maxCommandDuration: 5m + maxEnvironments: 20 + quota-cas: + image: placeholder.invalid/workload@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + storageClass: standard + storageSize: 1Gi + cpuRequest: 50m + memoryRequest: 64Mi + cpuLimit: "1" + memoryLimit: 512Mi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: qualification-runc + maxFileBytes: 5242880 + maxCommandBytes: 1048576 + maxCommandDuration: 5m + maxEnvironments: 1 + quota-kube: + image: placeholder.invalid/workload@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + storageClass: standard + storageSize: 1Gi + cpuRequest: 50m + memoryRequest: 64Mi + cpuLimit: "1" + memoryLimit: 512Mi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: qualification-runc + maxFileBytes: 5242880 + maxCommandBytes: 1048576 + maxCommandDuration: 5m + maxEnvironments: 4 +networkPolicy: + workloadProfiles: {} diff --git a/deploy/mecatl-execution-kind/failure-evidence.jq b/deploy/mecatl-execution-kind/failure-evidence.jq new file mode 100644 index 0000000000..1f6924b5cf --- /dev/null +++ b/deploy/mecatl-execution-kind/failure-evidence.jq @@ -0,0 +1,33 @@ +# Only closed vocabulary, booleans and counts may leave the fixture. Names, +# messages, UIDs, spec data, lease identities and timestamps stay in memory. +def known($allowed): . as $v | if ($allowed | index($v)) != null then $v else "other" end; +def finalizers: + [.metadata.finalizers[]? | select(. == "execution.mecatl.dev/retain-workspace" or . == "execution.mecatl.dev/verify-termination" or . == "kubernetes.io/pvc-protection")] | unique; +def quota_keys: + ["pods", "persistentvolumeclaims", "count/executionenvironments.execution.mecatl.dev", "requests.cpu", "requests.memory", "requests.storage", "requests.ephemeral-storage", "limits.cpu", "limits.memory", "limits.ephemeral-storage"]; +[.[] | .items[]?] as $items | +($items | map(select(.kind == "Pod"))) as $pods | +($items | map(select(.kind == "PersistentVolumeClaim"))) as $pvcs | +($items | map(select(.kind == "ExecutionEnvironment"))) as $envs | +{kind:"collection", unavailable:([.[] | select(.unavailable == true)] | length), environments:($envs|length), pods:($pods|length), pvcs:($pvcs|length)}, +($envs[:32][] | . as $env | + ([$pods[] | select(.metadata.name == $env.status.pod.name)][0]) as $pod | + ([$pvcs[] | select(.metadata.name == $env.status.pvc.name)][0]) as $pvc | + {kind:"environment", deleting:(.metadata.deletionTimestamp != null), finalizers:finalizers, + lifecycle:(.status.lifecycleOperation.type | known(["DeleteRetiredEnvironment","RetireEnvironment","ReplaceExecutor"])), + phase:(.status.lifecycleOperation.phase | known(["DeletingPVC","ReleasingSlot","Quiescing","WaitingForTermination","RemovingPodFinalizer","WaitingForPodDeletion","CreatingReplacement"])), + pod_present:($pod != null), pod_uid_matches:($pod != null and .status.pod.uid != null and .status.pod.uid == $pod.metadata.uid), + pvc_present:($pvc != null), pvc_uid_matches:($pvc != null and .status.pvc.uid != null and .status.pvc.uid == $pvc.metadata.uid), + operation_pvc_uid_matches:(.status.lifecycleOperation.expectedPVCUID != null and .status.lifecycleOperation.expectedPVCUID == .status.pvc.uid), + operation_pod_uid_matches:(.status.lifecycleOperation.expectedPodUID != null and .status.lifecycleOperation.expectedPodUID == .status.pod.uid), + lease_present:(.status.activeOperation.expiresAt != null), + lease_expired:((.status.activeOperation.expiresAt // "" | sub("\\.[0-9]+Z$"; "Z") | try fromdateiso8601 catch null) as $expiry | if $expiry == null then null else $expiry <= now end), + conditions:[.status.conditions[:16][]? | {type:(.type | known(["Ready","Retired","ExecutorTerminated","DeletionBlocked"])), status:(.status | known(["True","False","Unknown"])), reason:(.reason | known(["FenceUnknown","ExactLifecycleRequired","WorkspaceRetained","TerminalPodProof","AwaitingTerminalExecutor","Retained","ReplacementStarting","ReplacementReady","PVCUnavailable","ExecutorUnavailable","Reconciled","InvalidProfile","IncompatibleSchema"]))}]}), +($pods[:64][] | {kind:"pod", deleting:(.metadata.deletionTimestamp != null), finalizers:finalizers, + phase:(.status.phase | known(["Pending","Running","Succeeded","Failed","Unknown"])), + containers:([.status.containerStatuses[]?, .status.initContainerStatuses[]?, .status.ephemeralContainerStatuses[]?][:16] | map({terminated:(.state.terminated != null), reason:((.state.terminated.reason // .state.waiting.reason) | known(["Completed","Error","OOMKilled","ContainerStatusUnknown","CrashLoopBackOff","ImagePullBackOff","ErrImagePull","ContainerCreating"]))}))}), +($pvcs[:64][] | {kind:"pvc", deleting:(.metadata.deletionTimestamp != null), finalizers:finalizers}), +($items | map(select(.kind == "ResourceQuota")) | .[:32][] | . as $q | + {kind:"quota", missing_or_mismatched_keys:[quota_keys[] | . as $key | select($q.spec.hard[$key] != null and ($q.status.hard[$key] != $q.spec.hard[$key] or $q.status.used[$key] == null))]}), +($items | map(select(.kind == "Event") | .reason | known(["FailedScheduling","FailedMount","FailedAttachVolume","FailedBinding","ProvisioningFailed","FailedCreate","Failed","BackOff","Killing","Scheduled","Pulled","Created","Started"])) | group_by(.)[] | {kind:"event", reason:.[0], count:length}), +(if any(.[]; .unavailable == true) then "" | halt_error(1) else empty end) diff --git a/deploy/mecatl-execution-kind/failure_evidence_test.go b/deploy/mecatl-execution-kind/failure_evidence_test.go new file mode 100644 index 0000000000..253b4405bb --- /dev/null +++ b/deploy/mecatl-execution-kind/failure_evidence_test.go @@ -0,0 +1,163 @@ +package executionkind_test + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestFailureEvidenceRejectsProducerText(t *testing.T) { + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("jq unavailable") + } + // Even a syntactically valid metadata name or reason is not an output grant. + const hostile = "PRIVATE-URL-TOKEN-COMMAND" + fixture := `{"items":[{"kind":"ExecutionEnvironment","metadata":{"name":"` + hostile + `","deletionTimestamp":"private","finalizers":["` + hostile + `","execution.mecatl.dev/retain-workspace"]},"status":{"pod":{"name":"p","uid":"expected"},"pvc":{"name":"v","uid":"expected"},"activeOperation":{"id":"` + hostile + `","expiresAt":"2000-01-01T00:00:00.123Z"},"lifecycleOperation":{"type":"DeleteRetiredEnvironment","phase":"ReleasingSlot"},"conditions":[{"type":"Ready","status":"False","reason":"FenceUnknown","message":"` + hostile + `"}]}},{"kind":"Pod","metadata":{"name":"p","uid":"foreign","finalizers":["execution.mecatl.dev/verify-termination"]},"status":{"phase":"Failed","containerStatuses":[{"name":"` + hostile + `","state":{"terminated":{"reason":"Error","message":"` + hostile + `"}}}]}},{"kind":"PersistentVolumeClaim","metadata":{"name":"v","uid":"expected","finalizers":["kubernetes.io/pvc-protection"]}},{"kind":"ResourceQuota","spec":{"hard":{"pods":"10","requests.cpu":"10","` + hostile + `":"10"}},"status":{"hard":{"pods":"10","requests.cpu":"20"},"used":{"requests.cpu":"0"}}},{"kind":"Event","reason":"` + hostile + `","message":"` + hostile + `"}]}` + cmd := exec.CommandContext(t.Context(), "jq", "-cs", "-f", "failure-evidence.jq") + cmd.Stdin = strings.NewReader(fixture) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("filter: %v: %s", err, out) + } + if strings.Contains(string(out), hostile) || len(out) > 16384 { + t.Fatal("untrusted evidence escaped") + } + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + var v map[string]any + if err := json.Unmarshal([]byte(line), &v); err != nil { + t.Fatal(err) + } + if v["kind"] == "environment" { + if v["pod_uid_matches"] != false || v["pvc_uid_matches"] != true || v["lease_expired"] != true || v["deleting"] != true || len(v["finalizers"].([]any)) != 1 { + t.Fatalf("lost diagnostic facts: %s", line) + } + } + if v["kind"] == "quota" && len(v["missing_or_mismatched_keys"].([]any)) != 2 { + t.Fatalf("lost quota mismatch: %s", line) + } + } +} + +func TestCreateStageProjectionRejectsRawAndPrivateData(t *testing.T) { + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("jq unavailable") + } + const sentinel = "PRIVATE-URL-TOKEN-OWNER-REF-STACK" + valid := `{"msg":"remote create stage","level":"DEBUG","stage":"attach_poll","reason":"not_ready_nonretryable","elapsed_ms":480000,"calls":4800,"session":"0123456789abcdef0123456789abcdef","error":"` + sentinel + `","grant":"` + sentinel + `"}` + input := sentinel + "\n" + `{"msg":"` + sentinel + `"}` + "\n" + valid + "\n" + for _, field := range []string{"stage", "reason", "session", "elapsed_ms", "calls"} { + var row map[string]any + if err := json.Unmarshal([]byte(valid), &row); err != nil { + t.Fatal(err) + } + row[field] = sentinel + bad, err := json.Marshal(row) + if err != nil { + t.Fatal(err) + } + input += string(bad) + "\n" + } + cmd := exec.CommandContext(t.Context(), "jq", "-Rnc", "--arg", "source", "mecak8s", "-f", "create-stages.jq") + cmd.Stdin = strings.NewReader(input) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("projection failed: %v", err) + } + if strings.Contains(string(out), sentinel) || strings.Contains(string(out), "grant") || strings.Count(string(out), `"kind":"create_stage"`) != 1 || !strings.Contains(string(out), "not_ready_nonretryable") { + t.Fatal("projection leaked or lost discriminating state") + } + cmd = exec.CommandContext(t.Context(), "jq", "-Rnc", "--arg", "source", "mecak8s", "-f", "create-stages.jq") + churn := valid + "\n" + strings.ReplaceAll(valid, "not_ready_nonretryable", "not_ready_retryable") + "\n" + terminal := strings.ReplaceAll(valid, "not_ready_nonretryable", "error") + input = strings.Repeat(churn, 750) + strings.ReplaceAll(terminal, "attach_poll", "attach_poll_end") + "\n" + strings.ReplaceAll(terminal, "attach_poll", "engine_factory") + "\n" + input += sentinel + "\n" + strings.ReplaceAll(valid, "attach_poll", sentinel) + "\n" + cmd.Stdin = strings.NewReader(input) + out, err = cmd.CombinedOutput() + if err != nil || strings.Count(string(out), `"kind":"create_stage"`) != 512 || len(out) > 262144 { + t.Fatal("projection did not drain and bound the input") + } + if !strings.Contains(string(out), `"stage":"attach_poll_end","reason":"error"`) || !strings.Contains(string(out), `"stage":"engine_factory","reason":"error"`) { + t.Fatal("projection lost latest terminal stages") + } + if strings.Contains(string(out), sentinel) || strings.Contains(string(out), "grant") { + t.Fatal("bounded projection leaked raw or private data") + } +} + +func TestFailureCollectorBoundsAndContext(t *testing.T) { + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("jq unavailable") + } + root := t.TempDir() + bin := filepath.Join(root, "bin") + if err := os.Mkdir(bin, 0o700); err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(bin, "kubectl"), `#!/bin/sh +set -eu +test "$1 $2 $3 $4 $5" = '--kubeconfig synthetic --context kind-owned --request-timeout=5s' +if [ "$6" = logs ]; then + test "${8} ${9} ${10} ${11} ${12}" = '-n execution-qualification --all-containers=true --tail=1000 --limit-bytes=262144' + case "$7" in deployment/mecak8s|deployment/mecatl-execution) ;; *) exit 99 ;; esac + printf 'PRIVATE-RAW-STACK\n{"msg":"remote create stage","level":"DEBUG","stage":"bind_ensure","reason":"begin","calls":0,"elapsed_ms":1,"session":"","error":"PRIVATE-ERROR"}\n' + if [ "${LOG_FAILURE:-}" = 1 ]; then printf 'PRIVATE-LOG-ERROR' >&2; exit 17; fi + exit 0 +fi +test "$6 ${8} ${9} ${10} ${11}" = 'get -n execution-qualification -o json' +if [ -n "${EVIDENCE_FIXTURE:-}" ]; then cat "$EVIDENCE_FIXTURE"; exit; fi +case "$7" in +pods) printf 'PRIVATE-ERROR' >&2; exit 1 ;; +*) printf '{"items":[{"kind":"Event","reason":"FailedMount","message":"PRIVATE-DATA"}]}' ;; +esac +`, 0o700) + outPath := filepath.Join(root, "evidence") + script, err := filepath.Abs("collect-failure.sh") + if err != nil { + t.Fatal(err) + } + out, err := runStep(t, root, "sh \"$COLLECTOR\" synthetic kind-owned \"$OUT\"", "COLLECTOR="+script, "OUT="+outPath, "PATH="+bin+":"+os.Getenv("PATH")) + if err == nil { + t.Fatal("collector hid failed resource API") + } + data, err := os.ReadFile(outPath) + if err != nil || len(data) > 16384 || len(data) == 0 || strings.Contains(string(data), "PRIVATE") || strings.Contains(string(out), "PRIVATE") { + t.Fatalf("unsafe/missing evidence: %v", err) + } + if !strings.Contains(string(data), `"unavailable":1`) || strings.Count(string(data), `"kind":"create_stage"`) != 2 { + t.Fatal("API failure or projected workload stages not recorded") + } + if _, err := runStep(t, root, "sh \"$COLLECTOR\" synthetic kind-owned \"$OUT\"", "COLLECTOR="+script, "OUT="+outPath, "PATH="+bin+":"+os.Getenv("PATH")); err == nil { + t.Fatal("collector overwrote earlier real-process evidence") + } + preserved, err := os.ReadFile(outPath) + if err != nil || string(preserved) != string(data) { + t.Fatal("earlier evidence changed") + } + // A large fleet must retain late quota evidence, not exhaust a summary-sized + // cap on environment/Pod rows before reaching the quota and event sections. + container := `{"state":{"terminated":{"reason":"Completed"}}}` + pod := `{"kind":"Pod","status":{"phase":"Failed","containerStatuses":[` + strings.TrimSuffix(strings.Repeat(container+",", 16), ",") + `]}}` + fixture := filepath.Join(root, "fleet.json") + writeFixture(t, fixture, `{"items":[`+strings.Repeat(pod+",", 64)+`{"kind":"ResourceQuota","spec":{"hard":{"pods":"1"}}}]}`, 0o600) + outPath = filepath.Join(root, "fleet-evidence") + out, err = runStep(t, root, "sh \"$COLLECTOR\" synthetic kind-owned \"$OUT\"", "COLLECTOR="+script, "OUT="+outPath, "EVIDENCE_FIXTURE="+fixture, "PATH="+bin+":"+os.Getenv("PATH")) + if err != nil { + t.Fatalf("fleet collector: %v: %s", err, out) + } + data, err = os.ReadFile(outPath) + if err != nil || len(data) > 1<<20 || !strings.Contains(string(data), `"kind":"quota"`) { + t.Fatal("fleet evidence lost quota or exceeded artifact cap") + } + outPath = filepath.Join(root, "log-failure-evidence") + out, err = runStep(t, root, "sh \"$COLLECTOR\" synthetic kind-owned \"$OUT\"", "COLLECTOR="+script, "OUT="+outPath, "EVIDENCE_FIXTURE="+fixture, "LOG_FAILURE=1", "PATH="+bin+":"+os.Getenv("PATH")) + if err == nil || strings.Contains(string(out), "PRIVATE") { + t.Fatal("log API failure hidden or leaked") + } + data, err = os.ReadFile(outPath) + if err != nil || strings.Contains(string(data), "PRIVATE") || strings.Count(string(data), `"unavailable":true`) != 2 || !strings.Contains(string(data), `"kind":"quota"`) { + t.Fatal("log API failure lost safe partial evidence or skipped resources") + } +} diff --git a/deploy/mecatl-execution-kind/images.sh b/deploy/mecatl-execution-kind/images.sh new file mode 100644 index 0000000000..7f9859f582 --- /dev/null +++ b/deploy/mecatl-execution-kind/images.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# The alias name is not proof of its target: compare containerd's canonical +# descriptor before reusing it. Never overwrite a conflicting retained alias. +pin_loaded() { + tagged=$1 + images=$("$runtime" exec "${cluster}-control-plane" ctr -n k8s.io images ls) || return 1 + digest=$(printf '%s\n' "$images" | awk -v ref="$tagged" '$1 == ref {print $3; exit}') + printf '%s\n' "$digest" | grep -Eq '^sha256:[0-9a-f]{64}$' || { echo "loaded image digest unavailable or invalid" >&2; return 1; } + pinned="${tagged%:*}@${digest}" + existing=$(printf '%s\n' "$images" | awk -v ref="$pinned" '$1 == ref {print $3; exit}') + if [ -n "$existing" ]; then + [ "$existing" = "$digest" ] || { echo "loaded image alias conflicts with canonical digest" >&2; return 1; } + else + "$runtime" exec "${cluster}-control-plane" ctr -n k8s.io images tag "$tagged" "$pinned" >/dev/null || return 1 + fi + printf '%s\n' "$pinned" +} diff --git a/deploy/mecatl-execution-kind/live.sh b/deploy/mecatl-execution-kind/live.sh new file mode 100755 index 0000000000..36c9791cbb --- /dev/null +++ b/deploy/mecatl-execution-kind/live.sh @@ -0,0 +1,190 @@ +#!/bin/sh +set -eu + +# Explicit opt-in live qualification against one already-owned Kind cluster. +# Re-exec with only non-secret runtime selectors plus the credential FILE path. +if [ "${MECATL_EXECUTION_LIVE_CLEAN_ENV:-}" != 1 ]; then + exec env -i HOME="$HOME" USER="${USER:-user}" PATH="$PATH" XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" \ + DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-}" CONTAINER_HOST="${CONTAINER_HOST:-}" DOCKER_HOST="${DOCKER_HOST:-}" \ + TOOLBOX_PATH="${TOOLBOX_PATH:-}" MECATL_EXECUTION_DEV_TOOLBOX="${MECATL_EXECUTION_DEV_TOOLBOX:-}" MECATL_EXECUTION_K8S_TOOLBOX="${MECATL_EXECUTION_K8S_TOOLBOX:-}" \ + MECATL_EXECUTION_CREDENTIAL_FILE="${MECATL_EXECUTION_CREDENTIAL_FILE:-}" MECATL_EXECUTION_QUAL_STATE="${MECATL_EXECUTION_QUAL_STATE:-}" \ + MECATL_EXECUTION_LIVE_CLEAN_ENV=1 "$0" "$@" +fi +[ "$#" -eq 0 ] || { echo "usage: MECATL_EXECUTION_CREDENTIAL_FILE=/absolute/file MECATL_EXECUTION_QUAL_STATE=/owned/state $0" >&2; exit 2; } +[ -n "$MECATL_EXECUTION_CREDENTIAL_FILE" ] || { echo "MECATL_EXECUTION_CREDENTIAL_FILE is required" >&2; exit 2; } +[ -n "$MECATL_EXECUTION_QUAL_STATE" ] || { echo "MECATL_EXECUTION_QUAL_STATE is required" >&2; exit 2; } + +root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd -P) +cd "$root" +owned_root="$root/.scratch/k8s-execution" +case "${MECATL_EXECUTION_DEV_TOOLBOX:-}" in ''|dev) ;; *) echo "MECATL_EXECUTION_DEV_TOOLBOX must be empty or dev" >&2; exit 2 ;; esac +case "${MECATL_EXECUTION_K8S_TOOLBOX:-}" in ''|sre) ;; *) echo "MECATL_EXECUTION_K8S_TOOLBOX must be empty or sre" >&2; exit 2 ;; esac +state=$(CDPATH= cd -- "$MECATL_EXECUTION_QUAL_STATE" && pwd -P) || { echo "qualification state unavailable" >&2; exit 1; } +case "$state" in + "$owned_root"/*) ;; + *) echo "refusing qualification state outside owned root" >&2; exit 1 ;; +esac +ownership="$state/ownership" +[ -f "$ownership" ] && [ ! -L "$ownership" ] || { echo "owned qualification record unavailable" >&2; exit 1; } +owned_value() { + awk -F= -v key="$1" ' + $1 == key { value=substr($0, length(key)+2); count++ } + END { if (count != 1 || value == "") exit 1; print value } + ' "$ownership" +} +optional_owned_value() { + awk -F= -v key="$1" ' + $1 == key { value=substr($0, length(key)+2); count++ } + END { if (count > 1) exit 1; if (count == 1) print value } + ' "$ownership" +} +cluster=$(owned_value cluster) || { echo "owned cluster identity unavailable" >&2; exit 1; } +case "$cluster" in + [a-z0-9]|[a-z0-9]*[a-z0-9]) ;; + *) echo "owned cluster identity invalid" >&2; exit 1 ;; +esac +case "$cluster" in *[!a-z0-9-]*) echo "owned cluster identity invalid" >&2; exit 1 ;; esac +[ "${#cluster}" -le 63 ] || { echo "owned cluster identity invalid" >&2; exit 1; } +owned_namespace=$(owned_value namespace) || { echo "owned namespace unavailable" >&2; exit 1; } +[ "$owned_namespace" = execution-qualification ] || { echo "owned namespace mismatch" >&2; exit 1; } +recorded_owner=$(owned_value owner) || { echo "owned user identity unavailable" >&2; exit 1; } +[ "$recorded_owner" = "${USER:-user}" ] || { echo "owned user identity mismatch" >&2; exit 1; } +kubeconfig="$state/kubeconfig" +recorded_kubeconfig=$(owned_value kubeconfig) || { echo "owned kubeconfig identity unavailable" >&2; exit 1; } +[ "$recorded_kubeconfig" = "$kubeconfig" ] || { echo "owned kubeconfig mismatch" >&2; exit 1; } +[ -f "$kubeconfig" ] && [ ! -L "$kubeconfig" ] && [ "$(stat -c '%a' "$kubeconfig")" = 600 ] || { echo "owned kubeconfig must be a private regular file" >&2; exit 1; } +context=$(optional_owned_value context) || { echo "owned context identity invalid" >&2; exit 1; } +[ -n "$context" ] || context="kind-$cluster" +[ "$context" = "kind-$cluster" ] || { echo "owned context mismatch" >&2; exit 1; } +runtime=$(owned_value runtime) || { echo "owned container runtime unavailable" >&2; exit 1; } +[ "$runtime" = docker ] || [ "$runtime" = podman ] || { echo "owned container runtime invalid" >&2; exit 1; } +profile=$(owned_value profile) || { echo "owned qualification profile unavailable" >&2; exit 1; } +[ "$profile" = production ] || { echo "live qualification requires a completed production-state fixture" >&2; exit 1; } +label=$($runtime inspect "${cluster}-control-plane" --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}') +[ "$label" = "$cluster" ] || { echo "owned Kind container label mismatch" >&2; exit 1; } + +if [ "$runtime" = podman ]; then export KIND_EXPERIMENTAL_PROVIDER=podman; else unset KIND_EXPERIMENTAL_PROVIDER; fi +export KUBECONFIG=$kubeconfig +export REGISTRY_AUTH_FILE="$state/registry-auth.json" +dev() { + if [ -n "$MECATL_EXECUTION_DEV_TOOLBOX" ]; then toolbox run -c "$MECATL_EXECUTION_DEV_TOOLBOX" "$@"; else "$@"; fi +} +kube() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" kubectl --kubeconfig "$kubeconfig" --context "$context" "$@"; else kubectl --kubeconfig "$kubeconfig" --context "$context" "$@"; fi +} +helm_kube() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" helm --kubeconfig "$kubeconfig" --kube-context "$context" "$@"; else helm --kubeconfig "$kubeconfig" --kube-context "$context" "$@"; fi +} +[ "$(kube config current-context)" = "$context" ] || { echo "owned kube context mismatch" >&2; exit 1; } +build_ko() { package=$1 repo=$2; dev env KIND_EXPERIMENTAL_PROVIDER="${KIND_EXPERIMENTAL_PROVIDER:-}" KO_DOCKER_REPO="$repo" ko build --local --bare "$package" | tail -n 1; } +. "$root/deploy/mecatl-execution-kind/images.sh" +load_image() { + tagged=$1 + archive="$state/images/live-$(printf '%s' "$tagged" | sha256sum | cut -c1-16).tar" + $runtime save "$tagged" -o "$archive" >/dev/null + kind load image-archive "$archive" --name "$cluster" + pin_loaded "$tagged" +} + +provider_tag=$(build_ko ./cmd/mecatl-execution-provider ko.local/mecatl-execution-provider) +agent_tag=$(build_ko ./cmd/mecak8s ko.local/mecak8s) +if [ "$runtime" = podman ]; then + podman image exists docker.io/library/golang:1.27 || podman pull docker.io/library/golang:1.27 >/dev/null + go_digest=$(podman image inspect docker.io/library/golang:1.27 --format '{{.Digest}}') +else + docker image inspect docker.io/library/golang:1.27 >/dev/null 2>&1 || docker pull docker.io/library/golang:1.27 >/dev/null + go_ref=$(docker image inspect docker.io/library/golang:1.27 --format '{{index .RepoDigests 0}}') + go_digest=${go_ref#*@} +fi +go_image="docker.io/library/golang@${go_digest}" +workload_tag=localhost/mecatl-execution-workload:e2e +$runtime build --build-arg GO_IMAGE="$go_image" -f "$root/build/execution-workload/Dockerfile" -t "$workload_tag" "$root" >/dev/null +provider_image=$(load_image "$provider_tag") +agent_image=$(load_image "$agent_tag") +workload_image=$(load_image "$workload_tag") +printf 'provider=%s\nagent=%s\nworkload=%s\ngo_base=%s\n' "$provider_image" "$agent_image" "$workload_image" "$go_image" >"$state/images/live-proof" + +# Consume the already-qualified storage and synthetic security state. Preserve +# Kind's local-path helper configuration; the non-root workload is not its helper. +# Live mode never regenerates or replaces the execution Secret/keyring, or reads it back. +# A real provider credential is staged only after this deterministic rerun passes. +kube -n execution-qualification create configmap execution-mock --from-file=mock-script.json="$root/deploy/mecatl-execution-kind/mock-script.json" --dry-run=client -o yaml | kube apply -f - +kube -n execution-qualification rollout status deployment/mecatl-execution --timeout=240s +helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ + --set-string image.repository="${agent_image%@*}" --set-string image.tag= --set-string image.digest="${agent_image#*@}" --wait --timeout=5m +kube -n execution-qualification rollout restart deployment/mecak8s +kube -n execution-qualification rollout status deployment/mecak8s --timeout=240s +dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_QUAL_STATE="$state" \ + go test -tags kind_execution_e2e -run '^TestKindExecutionQualification$' -count=1 -timeout=15m ./e2e/k8s_execution + +secret="mecak8s-live-$(date -u +%Y%m%d%H%M%S)-$$" +receipt="$state/live-secret-$secret.receipt.json" +restore_needed=0 +restore() { + status=$? + trap - EXIT INT TERM + cleanup_failed=0 + mock_restored=0 + secret_cleaned=0 + if [ "$status" -ne 0 ] && [ "$restore_needed" -eq 1 ]; then + # Ownership was verified before staging. Capture the real process before + # Helm replaces it, independently of Secret cleanup and mock restoration. + set -- sh "$root/deploy/mecatl-execution-kind/collect-failure.sh" "$kubeconfig" "$context" "$state/live-diagnostics.jsonl" + if [ -n "${MECATL_EXECUTION_K8S_TOOLBOX:-}" ]; then + set -- toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" "$@" + fi + if timeout --kill-after=5s 45s "$@"; then + printf 'complete\n' > "$state/live-diagnostics.status" || echo 'warning: diagnostic status unavailable' >&2 + else + printf 'incomplete\n' > "$state/live-diagnostics.status" || echo 'warning: diagnostic status unavailable' >&2 + echo 'warning: bounded pre-restoration diagnostics incomplete' >&2 + fi + fi + if [ "$restore_needed" -eq 1 ]; then + if helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ + --set-string image.repository="${agent_image%@*}" --set-string image.tag= --set-string image.digest="${agent_image#*@}" --wait --timeout=5m >/dev/null 2>&1; then + mock_restored=1 + else + echo "mock profile restoration failed; cluster retained for inspection" >&2 + cleanup_failed=1 + fi + fi + if [ -f "$receipt" ]; then + if dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" \ + go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/credentialloader delete "$kubeconfig" "$context" "$secret" "$receipt"; then + secret_cleaned=1 + else + echo "UID-pinned run-scoped provider Secret cleanup pending; cluster retained for inspection" >&2 + cleanup_failed=1 + fi + else + secret_cleaned=1 + fi + if [ "$mock_restored" -eq 1 ] && [ "$secret_cleaned" -eq 1 ]; then + echo "cleanup verification passed: mock harness restored; run-scoped Secret receipt cleared" + fi + if [ "$status" -eq 0 ] && [ "$cleanup_failed" -ne 0 ]; then + status=1 + fi + exit "$status" +} +trap restore EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_CREDENTIAL_FILE="$MECATL_EXECUTION_CREDENTIAL_FILE" \ + go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/credentialloader stage "$kubeconfig" "$context" "$secret" "$receipt" +restore_needed=1 +helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ + --set-string image.repository="${agent_image%@*}" --set-string image.tag= --set-string image.digest="${agent_image#*@}" \ + --set mockProvider=false --set security.allowUnsafeRealProvider=true --set defaultProvider=openrouter --set-string model=anthropic/claude-haiku-4.5 --set maxRunTokens=32000 \ + --set-json 'extraArgs=["--log-level=debug","--no-soul","--no-user-model","--permissions-conventional=false","--agents-conventional=false"]' \ + --set extraVolumes=null --set extraVolumeMounts=null \ + --set-string extraEnv[0].name=OPENROUTER_API_KEY --set-string extraEnv[0].valueFrom.secretKeyRef.name="$secret" --set-string extraEnv[0].valueFrom.secretKeyRef.key=OPENROUTER_API_KEY \ + --wait --timeout=5m +kube -n execution-qualification rollout status deployment/mecak8s --timeout=240s +dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_QUAL_STATE="$state" MECATL_EXECUTION_LIVE=1 MECATL_EXECUTION_LIVE_SECRET="$secret" MECATL_EXECUTION_LIVE_CLUSTER="$cluster" \ + go test -tags kind_execution_e2e -run '^TestKindExecutionLiveQualification$' -count=1 -timeout=10m ./e2e/k8s_execution + +echo "live qualification passed; sanitized summary: $state/live-summary.json" +echo "qualification cluster retained: $cluster" diff --git a/deploy/mecatl-execution-kind/local-path.yaml b/deploy/mecatl-execution-kind/local-path.yaml new file mode 100644 index 0000000000..bb7d4991fb --- /dev/null +++ b/deploy/mecatl-execution-kind/local-path.yaml @@ -0,0 +1,33 @@ +# Recovery fixture for a Kind local-path provisioner whose helper image was +# changed during an interrupted qualification setup. Fresh clusters do not need +# this manifest. The workload image supplies /bin/sh, mkdir, and chown. +apiVersion: v1 +kind: ConfigMap +metadata: + name: local-path-config + namespace: local-path-storage +data: + config.json: |- + {"nodePathMap":[{"node":"DEFAULT_PATH_FOR_NON_LISTED_NODES","paths":["/var/local-path-provisioner"]}]} + setup: |- + #!/bin/sh + set -eu + mkdir -m 0770 -p "$VOL_DIR" + chown 65532:65532 "$VOL_DIR" + teardown: |- + #!/bin/sh + echo "retained by execution qualification; no automatic deletion" + helperPod.yaml: |- + apiVersion: v1 + kind: Pod + metadata: + name: helper-pod + spec: + securityContext: {runAsUser: 0, runAsGroup: 0} + priorityClassName: system-node-critical + tolerations: + - operator: Exists + containers: + - name: helper-pod + image: WORKLOAD_IMAGE + imagePullPolicy: IfNotPresent diff --git a/deploy/mecatl-execution-kind/mecak8s-values.yaml b/deploy/mecatl-execution-kind/mecak8s-values.yaml new file mode 100644 index 0000000000..49c04703ef --- /dev/null +++ b/deploy/mecatl-execution-kind/mecak8s-values.yaml @@ -0,0 +1,38 @@ +fullnameOverride: mecak8s +replicaCount: 1 +mockProvider: true +redis: + endpoint: redis:6379 + credentialsSecret: "" + local: + enabled: true +execution: + enabled: true + endpoint: mecatl-execution.execution-qualification.svc.cluster.local:8443 + profile: go + tlsSecret: execution-client-tls + caKey: ca.crt + certKey: tls.crt + keyKey: tls.key +oidc: + enabled: true + issuer: https://oidc-issuer.execution-qualification.svc.cluster.local:8443 + audience: mecatl + allowPrivateHTTPSIssuer: true + caSecret: execution-client-tls + caKey: ca.crt +extraArgs: +- --log-level=debug +- --mock-script=/var/run/mecatl-execution/mock-script.json +- --no-soul +- --no-user-model +- --permissions-conventional=false +- --agents-conventional=false +extraVolumeMounts: +- name: execution-mock + mountPath: /var/run/mecatl-execution + readOnly: true +extraVolumes: +- name: execution-mock + configMap: + name: execution-mock diff --git a/deploy/mecatl-execution-kind/mock-script-reattach.json b/deploy/mecatl-execution-kind/mock-script-reattach.json new file mode 100644 index 0000000000..104f83ecf3 --- /dev/null +++ b/deploy/mecatl-execution-kind/mock-script-reattach.json @@ -0,0 +1,7 @@ +{ + "turns": [ + {"tool_calls":[{"id":"reattach-read","name":"Read","args":{"path":"proof.txt"}}]}, + {"tool_calls":[{"id":"reattach-shell","name":"Shell","args":{"command":"go test ./...","timeout_ms":120000}}]}, + {"text":"REMOTE_EXECUTION_REATTACH_COMPLETE"} + ] +} diff --git a/deploy/mecatl-execution-kind/mock-script.json b/deploy/mecatl-execution-kind/mock-script.json new file mode 100644 index 0000000000..16e59416b7 --- /dev/null +++ b/deploy/mecatl-execution-kind/mock-script.json @@ -0,0 +1,16 @@ +{ + "turns": [ + {"tool_calls":[{"id":"write-proof","name":"Write","args":{"path":"proof.txt","content":"alpha\n"}}]}, + {"tool_calls":[{"id":"read-before-edit","name":"Read","args":{"path":"proof.txt"}}]}, + {"tool_calls":[{"id":"edit-proof","name":"Edit","args":{"path":"proof.txt","old_string":"alpha","new_string":"beta"}}]}, + {"tool_calls":[{"id":"copy-proof","name":"Copy","args":{"source":"proof.txt","destination":"copy.txt"}}]}, + {"tool_calls":[{"id":"move-proof","name":"Move","args":{"source":"copy.txt","destination":"moved.txt"}}]}, + {"tool_calls":[{"id":"glob-proof","name":"Glob","args":{"pattern":"*.txt"}},{"id":"grep-proof","name":"Grep","args":{"pattern":"beta","path":"*.txt"}}]}, + {"tool_calls":[{"id":"write-mod","name":"Write","args":{"path":"go.mod","content":"module example.com/qualification\n\ngo 1.27\n"}}]}, + {"tool_calls":[{"id":"write-test","name":"Write","args":{"path":"remote_test.go","content":"package qualification\n\nimport (\"os\"; \"testing\")\n\nfunc TestRemoteWorkspace(t *testing.T) { b, err := os.ReadFile(\"proof.txt\"); if err != nil { t.Fatal(err) }; if string(b) != \"beta\\n\" { t.Fatalf(\"proof = %q\", b) } }\n"}}]}, + {"tool_calls":[{"id":"shell-go-test","name":"Shell","args":{"command":"go test ./...","timeout_ms":120000}}]}, + {"tool_calls":[{"id":"remove-moved","name":"Remove","args":{"path":"moved.txt"}}]}, + {"tool_calls":[{"id":"read-final","name":"Read","args":{"path":"proof.txt"}}]}, + {"text":"REMOTE_EXECUTION_QUALIFICATION_COMPLETE"} + ] +} diff --git a/deploy/mecatl-execution-kind/oidc.yaml b/deploy/mecatl-execution-kind/oidc.yaml new file mode 100644 index 0000000000..5f9775c216 --- /dev/null +++ b/deploy/mecatl-execution-kind/oidc.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oidc-issuer + namespace: execution-qualification +spec: + replicas: 1 + selector: + matchLabels: {app.kubernetes.io/name: oidc-issuer} + template: + metadata: + labels: {app.kubernetes.io/name: oidc-issuer} + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: {type: RuntimeDefault} + containers: + - name: issuer + image: OIDC_IMAGE + imagePullPolicy: IfNotPresent + args: + - --tls-cert=/identity/tls.crt + - --tls-key=/identity/tls.key + - --signing-key=/identity/jwt-key.pem + - --issuer=https://oidc-issuer.execution-qualification.svc.cluster.local:8443 + - --audience=mecatl + ports: [{name: https, containerPort: 8443}] + readinessProbe: {httpGet: {scheme: HTTPS, path: /healthz, port: https}, periodSeconds: 2} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: ["ALL"]} + resources: + requests: {cpu: 10m, memory: 32Mi} + limits: {cpu: 200m, memory: 128Mi} + volumeMounts: + - {name: identity, mountPath: /identity, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: identity + secret: {secretName: oidc-fixture-identity, defaultMode: 0440} + - name: tmp + emptyDir: {sizeLimit: 16Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: oidc-issuer + namespace: execution-qualification +spec: + selector: {app.kubernetes.io/name: oidc-issuer} + ports: [{name: https, port: 8443, targetPort: https}] diff --git a/deploy/mecatl-execution-kind/repair_test.go b/deploy/mecatl-execution-kind/repair_test.go new file mode 100644 index 0000000000..cba43c798a --- /dev/null +++ b/deploy/mecatl-execution-kind/repair_test.go @@ -0,0 +1,295 @@ +package executionkind_test + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func scriptRange(t *testing.T, file, start, end string) string { + t.Helper() + data, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + body := string(data) + i, j := strings.Index(body, start), strings.Index(body, end) + if i < 0 || j <= i { + t.Fatalf("missing script range in %s", file) + } + return body[i:j] +} + +func TestProductionThenLiveImageAlias(t *testing.T) { + production := scriptRange(t, "run.sh", "for item in", "printf 'provider=%s") + live := scriptRange(t, "live.sh", ". \"$root/deploy/mecatl-execution-kind/images.sh\"", "\nprovider_tag=$(build_ko") + helper, err := os.ReadFile("images.sh") + if err != nil { + t.Fatal(err) + } + for _, runtime := range []string{"docker", "podman"} { + for _, conflict := range []bool{false, true} { + t.Run(runtime+"/conflict="+map[bool]string{false: "false", true: "true"}[conflict], func(t *testing.T) { + root := t.TempDir() + for _, dir := range []string{"bin", "images", "deploy/mecatl-execution-kind"} { + if err := os.MkdirAll(filepath.Join(root, dir), 0o700); err != nil { + t.Fatal(err) + } + } + writeFixture(t, filepath.Join(root, "deploy/mecatl-execution-kind/images.sh"), string(helper), 0o600) + digest := "sha256:" + strings.Repeat("a", 64) + tag := "ko.local/provider:head" + alias := "ko.local/provider@" + digest + db := filepath.Join(root, "db") + var sourceRows strings.Builder + for _, name := range []string{"provider", "agent", "oidc", "netprobe", "workload"} { + sourceRows.WriteString("ko.local/" + name + ":head type " + digest + "\n") + } + writeFixture(t, db, sourceRows.String(), 0o600) + writeFixture(t, filepath.Join(root, "bin", runtime), `#!/bin/sh +set -eu +case "$1" in +save) : > "$4" ;; +exec) + test "$2" = owned-control-plane + shift 5 + test "$1" = images + case "$2" in + ls) cat "$DB" ;; + tag) + test "$#" = 4 + if awk -v ref="$4" '$1 == ref {found=1} END {exit !found}' "$DB"; then + echo 'already exists' >&2; exit 1 + fi + digest=$(awk -v ref="$3" '$1 == ref {print $3}' "$DB") + printf '%s type %s\n' "$4" "$digest" >> "$DB" + printf 'tag\n' >> "$TAGS" + ;; + *) exit 1 ;; + esac ;; +*) exit 1 ;; +esac +`, 0o700) + writeFixture(t, filepath.Join(root, "bin/kind"), "#!/bin/sh\nset -eu\ntest \"$1 $2\" = 'load image-archive'\ntest -f \"$3\"\ntest \"$4 $5\" = '--name owned'\n", 0o700) + env := []string{"PATH=" + filepath.Join(root, "bin") + ":" + os.Getenv("PATH"), "root=" + root, "state=" + root, "runtime=" + runtime, "cluster=owned", "DB=" + db, "TAGS=" + filepath.Join(root, "tags")} + for _, name := range []string{"provider", "agent", "oidc", "netprobe", "workload"} { + env = append(env, name+"_tag=ko.local/"+name+":head") + } + out, err := runStep(t, root, production, env...) + if err != nil { + t.Fatalf("fresh production load: %v: %s", err, out) + } + if conflict { + writeFixture(t, db, tag+" type "+digest+"\n"+alias+" type sha256:"+strings.Repeat("b", 64)+"\n", 0o600) + } + out, err = runStep(t, root, live+"\nload_image \"$provider_tag\"", env...) + if (err == nil) == conflict { + t.Fatalf("retained live load: %v: %s", err, out) + } + if !conflict && strings.TrimSpace(string(out)) != alias { + t.Fatalf("wrong alias: %s", out) + } + tags, err := os.ReadFile(filepath.Join(root, "tags")) + if err != nil || string(tags) != strings.Repeat("tag\n", 5) { + t.Fatalf("must create each alias once, never overwrite/re-tag: %q, %v", tags, err) + } + }) + } + } +} + +func TestLivePreservesQualifiedStorageAndHelmDigestThroughRestoration(t *testing.T) { + live := scriptRange(t, "live.sh", "printf 'provider=%s", "\necho \"live qualification passed") + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "images"), 0o700); err != nil { + t.Fatal(err) + } + marker := filepath.Join(root, "helm-calls") + kubeMarker := filepath.Join(root, "kube-calls") + devMarker := filepath.Join(root, "dev-calls") + digest := "sha256:" + strings.Repeat("a", 64) + out, err := runStep(t, root, ` +set -u +kube() { + case "$*" in + "-n execution-qualification create configmap execution-mock --from-file=mock-script.json=$root/deploy/mecatl-execution-kind/mock-script.json --dry-run=client -o yaml") + printf 'mock-config\n' >> "$KUBE_MARKER" + printf 'synthetic mock config\n' ;; + 'apply -f -') + payload=$(cat) + test "$payload" = 'synthetic mock config' || { echo 'forbidden applied manifest' >&2; return 1; } ;; + '-n execution-qualification rollout status deployment/mecatl-execution --timeout=240s') + printf 'provider-ready\n' >> "$KUBE_MARKER" ;; + '-n execution-qualification rollout restart deployment/mecak8s') + printf 'agent-restart\n' >> "$KUBE_MARKER" ;; + '-n execution-qualification rollout status deployment/mecak8s --timeout=240s') + printf 'agent-ready\n' >> "$KUBE_MARKER" ;; + *) echo "forbidden Kubernetes operation: $*" >&2; return 1 ;; + esac +} +kube_jq() { jq "$@"; } +dev() { + case "$*" in + *'go test -tags kind_execution_e2e -run ^TestKindExecutionQualification$ '*) printf 'mock-test\n' >> "$DEV_MARKER" ;; + *'go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/credentialloader stage '*) printf 'stage\n' >> "$DEV_MARKER" ;; + *'go test -tags kind_execution_e2e -run ^TestKindExecutionLiveQualification$ '*) printf 'live-test\n' >> "$DEV_MARKER" ;; + *) echo "unexpected dev operation: $*" >&2; return 1 ;; + esac +} +helm_kube() { + test "$1 $2 $3" = 'upgrade --install mecak8s' || return 1 + # Model a nonempty inherited tag; apply the actual CLI overrides in order. + tag=e2e digest= repository= profile=mock + while [ "$#" -gt 0 ]; do + case "$1" in + --set-string|--set) + shift + case "$1" in + image.tag=*) tag=${1#*=} ;; + image.digest=*) digest=${1#*=} ;; + image.repository=*) repository=${1#*=} ;; + mockProvider=false) profile=live ;; + esac ;; + esac + shift + done + test -z "$tag" && test "$repository@$digest" = "$agent_image" || return 1 + printf '%s\n' "$profile" >> "$MARKER" +} +`+live, "root="+root, "state="+root, "MARKER="+marker, "KUBE_MARKER="+kubeMarker, "DEV_MARKER="+devMarker, + "agent_image=ko.local/mecak8s@"+digest, "provider_image=synthetic", "workload_image=synthetic", "go_image=synthetic", + "kubeconfig=synthetic", "context=synthetic", + "cluster=owned", "MECATL_EXECUTION_CREDENTIAL_FILE=unused") + if err != nil { + t.Fatalf("live must preserve qualified storage and use digest-only Helm images: %v: %s", err, out) + } + calls, err := os.ReadFile(marker) + if err != nil || string(calls) != "mock\nlive\nmock\n" { + t.Fatalf("expected digest-only mock setup, live upgrade, and mock restoration: %q: %v", calls, err) + } + calls, err = os.ReadFile(kubeMarker) + if err != nil || string(calls) != "mock-config\nprovider-ready\nagent-restart\nagent-ready\nagent-ready\n" { + t.Fatalf("expected only mock configuration and readiness operations: %q: %v", calls, err) + } + calls, err = os.ReadFile(devMarker) + if err != nil || string(calls) != "mock-test\nstage\nlive-test\n" { + t.Fatalf("expected mock qualification before credential staging and live qualification: %q: %v", calls, err) + } +} + +func TestLiveSignalPreservesFailureAndAttemptsCleanup(t *testing.T) { + restore := scriptRange(t, "live.sh", "restore() {", "\ndev env -i HOME=\"$HOME\" PATH=\"$PATH\" KUBECONFIG=\"$kubeconfig\" MECATL_KUBE_CONTEXT=\"$context\" MECATL_EXECUTION_CREDENTIAL_FILE=") + for _, signal := range []struct { + name string + code string + command string + }{{"INT", "130", "kill -INT $$"}, {"TERM", "143", "kill -TERM $$"}, {"live-failed", "1", "exit 1"}, {"collector-failed", "1", "exit 1"}} { + t.Run(signal.name, func(t *testing.T) { + root := t.TempDir() + scripts := filepath.Join(root, "deploy/mecatl-execution-kind") + if err := os.MkdirAll(scripts, 0o700); err != nil { + t.Fatal(err) + } + collector := "#!/bin/sh\nset -eu\ntest \"$1 $2\" = 'synthetic kind-owned'\nprintf 'collect\\n' >> \"$MARKER\"\nprintf '{\"kind\":\"collection\"}\\n' > \"$3\"\n" + if signal.name == "collector-failed" { + collector += "exit 17\n" + } + writeFixture(t, filepath.Join(scripts, "collect-failure.sh"), collector, 0o700) + writeFixture(t, filepath.Join(root, "receipt"), "synthetic", 0o600) + writeFixture(t, filepath.Join(root, "signal.sh"), "#!/bin/sh\nset -eu\n"+` +helm_kube() { printf 'restore\n' >> "$MARKER"; } +dev() { printf 'delete\n' >> "$MARKER"; } +restore_needed=1 +`+restore+"\n"+signal.command+"\nexit 99\n", 0o700) + out, err := runStep(t, root, "if sh ./signal.sh; then exit 1; else test \"$?\" -eq "+signal.code+"; fi", "root="+root, "state="+root, "MARKER="+filepath.Join(root, "cleanup"), "receipt="+filepath.Join(root, "receipt"), "agent_image=synthetic@sha256:fake", "kubeconfig=synthetic", "context=kind-owned", "secret=synthetic") + if err != nil { + t.Fatalf("signal was reported as success: %v: %s", err, out) + } + calls, err := os.ReadFile(filepath.Join(root, "cleanup")) + if err != nil || string(calls) != "collect\nrestore\ndelete\n" { + t.Fatalf("evidence must precede restoration and UID cleanup, even on collection failure: %q: %v", calls, err) + } + status, err := os.ReadFile(filepath.Join(root, "live-diagnostics.status")) + want := "complete\n" + if signal.name == "collector-failed" { + want = "incomplete\n" + } + if err != nil || string(status) != want { + t.Fatal("diagnostic failure status lost") + } + }) + } +} + +func TestProductionPublishesOwnershipBeforePartialCreation(t *testing.T) { + data, err := os.ReadFile("run.sh") + if err != nil { + t.Fatal(err) + } + for _, fault := range []string{"partial", "discovery", "collision"} { + t.Run(fault, func(t *testing.T) { + root := t.TempDir() + scripts := filepath.Join(root, "deploy/mecatl-execution-kind") + bin := filepath.Join(root, "bin") + for _, dir := range []string{scripts, bin} { + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + } + writeFixture(t, filepath.Join(scripts, "run.sh"), string(data), 0o700) + writeFixture(t, filepath.Join(bin, "docker"), "#!/bin/sh\nexit 99\n", 0o700) + writeFixture(t, filepath.Join(root, "fault"), fault+"\n", 0o600) + writeFixture(t, filepath.Join(bin, "kind"), `#!/bin/sh +set -eu +IFS= read -r fault < "$HOME/fault" +case "$1" in +get) + case "$fault" in + discovery) exit 1 ;; + collision) awk -F= '$1 == "cluster" {print $2}' .scratch/k8s-execution/*/ownership ;; + esac ;; +create) + test -s "$MECATL_EXECUTION_QUAL_OUTPUT" + printf 'creation attempted\n' > "$HOME/attempted" + exit 1 ;; +*) exit 99 ;; +esac +`, 0o700) + output := filepath.Join(root, "output") + marker := filepath.Join(root, "attempted") + out, err := runStep(t, root, "./deploy/mecatl-execution-kind/run.sh", "PATH="+bin+":"+os.Getenv("PATH"), "CONTAINER_ENGINE=docker", "MECATL_EXECUTION_QUAL_PROFILE=production", "MECATL_EXECUTION_QUAL_OUTPUT="+output) + if err == nil { + t.Fatalf("partial production unexpectedly succeeded: %s", out) + } + outputs, err := os.ReadFile(output) + if fault != "partial" { + if !os.IsNotExist(err) { + t.Fatal("ownership published before collision/discovery gate") + } + if _, err := os.Stat(marker); !os.IsNotExist(err) { + t.Fatal("creation attempted after failed preflight") + } + return + } + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(string(outputs)), "\n") + if len(lines) != 2 || !strings.HasPrefix(lines[0], "state=") || !strings.HasPrefix(lines[1], "cluster=mecatl-execution-qual-") { + t.Fatalf("invalid ownership outputs: %q", outputs) + } + state := strings.TrimPrefix(lines[0], "state=") + ownership, err := os.ReadFile(filepath.Join(state, "ownership")) + if err != nil || !strings.Contains(string(ownership), lines[1]+"\n") { + t.Fatal("output does not bind the initial ownership record") + } + if _, err := os.Stat(marker); err != nil { + t.Fatal("fixture did not reach partial cluster creation") + } + if _, err := os.Stat(filepath.Join(state, "kubeconfig")); !os.IsNotExist(err) { + t.Fatal("partial fixture unexpectedly has a kubeconfig") + } + }) + } +} diff --git a/deploy/mecatl-execution-kind/run.sh b/deploy/mecatl-execution-kind/run.sh new file mode 100755 index 0000000000..0e69768341 --- /dev/null +++ b/deploy/mecatl-execution-kind/run.sh @@ -0,0 +1,345 @@ +#!/bin/sh +set -eu + +# Re-exec once with a deliberately small environment. In particular, no +# provider/API credential or ambient kube context can reach any subprocess. +if [ "${MECATL_EXECUTION_QUAL_CLEAN_ENV:-}" != 1 ]; then + exec env -i \ + HOME="$HOME" USER="${USER:-user}" PATH="$PATH" XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" \ + DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-}" CONTAINER_HOST="${CONTAINER_HOST:-}" DOCKER_HOST="${DOCKER_HOST:-}" \ + CONTAINER_ENGINE="${CONTAINER_ENGINE:-}" MECATL_EXECUTION_QUAL_CI="${MECATL_EXECUTION_QUAL_CI:-}" MECATL_EXECUTION_QUAL_PROFILE="${MECATL_EXECUTION_QUAL_PROFILE:-}" \ + TOOLBOX_PATH="${TOOLBOX_PATH:-}" MECATL_EXECUTION_DEV_TOOLBOX="${MECATL_EXECUTION_DEV_TOOLBOX:-}" MECATL_EXECUTION_K8S_TOOLBOX="${MECATL_EXECUTION_K8S_TOOLBOX:-}" \ + MECATL_EXECUTION_QUAL_OUTPUT="${MECATL_EXECUTION_QUAL_OUTPUT:-}" \ + MECATL_EXECUTION_QUAL_CLEAN_ENV=1 "$0" "$@" +fi + +[ "$#" -eq 0 ] || { echo "usage: $0" >&2; exit 2; } +case "${MECATL_EXECUTION_DEV_TOOLBOX:-}" in ''|dev) ;; *) echo "MECATL_EXECUTION_DEV_TOOLBOX must be empty or dev" >&2; exit 2 ;; esac +case "${MECATL_EXECUTION_K8S_TOOLBOX:-}" in ''|sre) ;; *) echo "MECATL_EXECUTION_K8S_TOOLBOX must be empty or sre" >&2; exit 2 ;; esac +runtime=${CONTAINER_ENGINE:-docker} +case "$runtime" in docker|podman) command -v "$runtime" >/dev/null 2>&1 ;; *) echo "CONTAINER_ENGINE must be docker or podman" >&2; exit 2 ;; esac +root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd -P) +cd "$root" +run_id=$(date -u +%Y%m%d%H%M%S)-$$ +cluster="mecatl-execution-qual-$run_id" +context="kind-$cluster" +state="$root/.scratch/k8s-execution/$run_id" +kubeconfig="$state/kubeconfig" +mkdir -p "$root/.scratch/k8s-execution" +umask 077 +mkdir "$state" +mkdir "$state/pki" "$state/images" +printf 'cluster=%s\ncontext=%s\nkubeconfig=%s\nnamespace=execution-qualification\nowner=%s\nruntime=%s\nprofile=%s\ncreated_at=%s\n' \ + "$cluster" "$context" "$kubeconfig" "${USER:-user}" "$runtime" "${MECATL_EXECUTION_QUAL_PROFILE:-development}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$state/ownership" + +# Refuse collisions; never delete or reuse any cluster. +if [ "$runtime" = podman ]; then + export KIND_EXPERIMENTAL_PROVIDER=podman +else + unset KIND_EXPERIMENTAL_PROVIDER +fi +clusters=$(kind get clusters) || { echo "qualification cluster discovery failed" >&2; exit 1; } +if printf '%s\n' "$clusters" | grep -Fx "$cluster" >/dev/null; then + echo "qualification cluster already exists: $cluster" >&2 + exit 1 +fi +# Publish the exact owned identity before creation can partially fail. Downstream +# CI steps consume these outputs, never the mutable local convenience pointer. +if [ -n "${MECATL_EXECUTION_QUAL_OUTPUT:-}" ]; then + printf 'state=%s\ncluster=%s\n' "$state" "$cluster" >>"$MECATL_EXECUTION_QUAL_OUTPUT" +fi +printf '%s\n' "$state" >"$root/.scratch/k8s-execution/current" + +if [ "${MECATL_EXECUTION_QUAL_CI:-}" = 1 ]; then + cleanup_cluster() { + status=$? + artifact="$state/production-failure-artifact.txt" + if [ "$status" -ne 0 ]; then + timeout --kill-after=5s 60s sh "$root/deploy/mecatl-execution-kind/collect-failure.sh" \ + "$kubeconfig" "$context" "$artifact" || printf 'qualification_diagnostics_incomplete\n' >&2 + fi + kind delete cluster --name "$cluster" + return "$status" + } + trap cleanup_cluster EXIT +fi +export KUBECONFIG="$kubeconfig" +printf '{}\n' >"$state/registry-auth.json" +chmod 600 "$state/registry-auth.json" +export REGISTRY_AUTH_FILE="$state/registry-auth.json" + +kind_config= +if [ "${MECATL_EXECUTION_QUAL_PROFILE:-development}" = production ]; then + kind_config="$state/kind.yaml" + cat >"$kind_config" <<'EOF' +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + disableDefaultCNI: true + podSubnet: 192.168.0.0/16 + serviceSubnet: 10.96.0.0/12 +EOF +fi +if [ -n "$kind_config" ]; then + kind create cluster --name "$cluster" --image kindest/node:v1.35.8@sha256:07b2536e30b803ed61d1677a79df6115f798ce64c80f9e22f6ed45afd09323c0 --config "$kind_config" --kubeconfig "$kubeconfig" +else + kind create cluster --name "$cluster" --image kindest/node:v1.35.8@sha256:07b2536e30b803ed61d1677a79df6115f798ce64c80f9e22f6ed45afd09323c0 --kubeconfig "$kubeconfig" +fi + +dev() { + if [ -n "$MECATL_EXECUTION_DEV_TOOLBOX" ]; then + (cd "$root" && toolbox run -c "$MECATL_EXECUTION_DEV_TOOLBOX" "$@") + else + (cd "$root" && "$@") + fi +} +kube() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then + toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" kubectl --kubeconfig "$kubeconfig" --context "$context" "$@" + else + kubectl --kubeconfig "$kubeconfig" --context "$context" "$@" + fi +} +helm_kube() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then + toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" helm --kubeconfig "$kubeconfig" --kube-context "$context" "$@" + else + helm --kubeconfig "$kubeconfig" --kube-context "$context" "$@" + fi +} + +if [ "${MECATL_EXECUTION_QUAL_PROFILE:-development}" = production ]; then + cni_manifest="$state/calico-v3.32.2.yaml" + curl --fail --location --proto '=https' --tlsv1.2 \ + https://raw.githubusercontent.com/projectcalico/calico/v3.32.2/manifests/calico.yaml \ + --output "$cni_manifest" + printf '%s %s\n' a8c828a06a87c629a282ebbc424895b77f3a030251993e41ea400a743675bb02 "$cni_manifest" | sha256sum -c - + sed -i \ + -e 's|quay.io/calico/cni:v3.32.2|quay.io/calico/cni@sha256:0ef740bc587f25565905adf1d1f61a7faff0d571c449c6bdd789feed743d3ef7|g' \ + -e 's|quay.io/calico/kube-controllers:v3.32.2|quay.io/calico/kube-controllers@sha256:7870b67ebb13fabc3005252b44fe6e78b21635649bd3072b80afa1684b6565d0|g' \ + -e 's|quay.io/calico/node:v3.32.2|quay.io/calico/node@sha256:99b03fe91e8bfbcb153ae65ef4b701b24ce541ffdd74ff314eb041096008f7fd|g' \ + "$cni_manifest" + kube apply -f "$cni_manifest" + kube -n kube-system rollout status daemonset/calico-node --timeout=5m + kube -n kube-system rollout status deployment/calico-kube-controllers --timeout=5m + kube wait --for=condition=Ready node --all --timeout=3m +fi + +# Synthetic keys are generated by the fixture only and never printed. Resume +# keeps the original PKI so mounted trust and running identities cannot drift. +if [ ! -s "$state/pki/ca.crt" ]; then + dev go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/pki "$state/pki" +fi + +# Build the three static Go images in the development toolbox. ko emits a +# content tag; after loading, the fixture resolves the node's actual manifest +# digest and creates a matching digest alias for Kubernetes. +build_ko() { + package=$1 repo=$2 + dev env KO_DOCKER_REPO="$repo" ko build --local --bare "$package" | tail -n 1 +} +provider_tag=$(build_ko ./cmd/mecatl-execution-provider ko.local/mecatl-execution-provider) +agent_tag=$(build_ko ./cmd/mecak8s ko.local/mecak8s) +oidc_tag=$(dev env GOFLAGS=-tags=kind_execution_e2e KO_DOCKER_REPO=ko.local/mecatl-oidc-fixture ko build --local --bare ./e2e/k8s_execution/fixture/oidcissuer | tail -n 1) +netprobe_tag=$(dev env GOFLAGS=-tags=kind_execution_e2e KO_DOCKER_REPO=ko.local/mecatl-netprobe-fixture ko build --local --bare ./e2e/k8s_execution/fixture/netprobe | tail -n 1) + +# Resolve the public Go base from the local OCI store, then feed its observed +# digest to the production workload recipe without weakening its pin check. +if [ "$runtime" = podman ]; then + podman image exists docker.io/library/golang:1.27 || podman pull docker.io/library/golang:1.27 >/dev/null + go_digest=$(podman image inspect docker.io/library/golang:1.27 --format '{{.Digest}}') +else + docker image inspect docker.io/library/golang:1.27 >/dev/null 2>&1 || docker pull docker.io/library/golang:1.27 >/dev/null + go_ref=$(docker image inspect docker.io/library/golang:1.27 --format '{{index .RepoDigests 0}}') + go_digest=${go_ref#*@} +fi +go_image="docker.io/library/golang@${go_digest}" +workload_tag="localhost/mecatl-execution-workload:e2e" +"$runtime" build --build-arg GO_IMAGE="$go_image" -f "$root/build/execution-workload/Dockerfile" -t "$workload_tag" "$root" >/dev/null + +for item in "$provider_tag" "$agent_tag" "$oidc_tag" "$netprobe_tag" "$workload_tag"; do + archive="$state/images/$(printf '%s' "$item" | sha256sum | cut -c1-16).tar" + "$runtime" save "$item" -o "$archive" >/dev/null + kind load image-archive "$archive" --name "$cluster" +done +. "$root/deploy/mecatl-execution-kind/images.sh" +provider_image=$(pin_loaded "$provider_tag") +agent_image=$(pin_loaded "$agent_tag") +oidc_image=$(pin_loaded "$oidc_tag") +netprobe_image=$(pin_loaded "$netprobe_tag") +workload_image=$(pin_loaded "$workload_tag") +printf 'provider=%s\nagent=%s\noidc=%s\nnetprobe=%s\nworkload=%s\ngo_base=%s\n' "$provider_image" "$agent_image" "$oidc_image" "$netprobe_image" "$workload_image" "$go_image" >"$state/images/proof" + +kube create namespace execution-qualification --dry-run=client -o yaml | kube apply -f - +kube label namespace execution-qualification pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/audit=restricted pod-security.kubernetes.io/warn=restricted --overwrite + +# Kind v0.33 installs its pinned local-path provisioner and default "standard" +# StorageClass as part of cluster creation. The fixture RuntimeClass names Kind's +# existing runc handler so provider startup can exercise the production preflight. +kube apply -f "$root/deploy/mecatl-execution-kind/runtimeclass.yaml" + +if [ "${MECATL_EXECUTION_QUAL_PROFILE:-development}" = production ]; then + cat >"$state/network-fixtures.yaml" <"$legacy_profiles" <&2; exit 1 ;; + :*|*:) echo "missing discovered network-policy endpoint" >&2; exit 1 ;; + esac + runtime_values="$state/production-runtime-values.yaml" + cat >"$runtime_values" < /workspace/migration-sentinel' + # Helm does not upgrade existing CRDs. Preserve the stored legacy references + # until the explicit CRD upgrade; the first production test then migrates them. + kube apply -f "$root/deploy/helm/mecatl-execution/crds/executionenvironment.yaml" + kube wait --for=condition=Established crd/executionenvironments.execution.mecatl.dev --timeout=60s + kube -n execution-qualification scale deployment/mecatl-execution --replicas=2 + kube -n execution-qualification rollout status deployment/mecatl-execution --timeout=4m +fi + +kube -n execution-qualification create configmap execution-mock --from-file=mock-script.json="$root/deploy/mecatl-execution-kind/mock-script.json" --dry-run=client -o yaml | kube apply -f - +helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" \ + --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ + --set-string image.repository="${agent_image%@*}" --set-string image.digest="${agent_image#*@}" --set-string image.tag= \ + --wait --timeout=5m +kube -n execution-qualification rollout restart deployment/mecak8s +kube -n execution-qualification rollout status deployment/mecak8s --timeout=240s + +dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_QUAL_STATE="$state" MECATL_EXECUTION_QUAL_PROFILE="${MECATL_EXECUTION_QUAL_PROFILE:-development}" \ + go test -tags kind_execution_e2e -count=1 -timeout=45m ./e2e/k8s_execution + +if [ "${MECATL_EXECUTION_QUAL_CI:-}" = 1 ]; then + echo "qualification cluster is CI-owned and will be removed: $cluster" +else + echo "qualification cluster retained: $cluster" + echo "kubeconfig: $kubeconfig" + echo "cleanup (requires human confirmation): CONTAINER_ENGINE=$runtime kind delete cluster --name $cluster" +fi diff --git a/deploy/mecatl-execution-kind/runtimeclass.yaml b/deploy/mecatl-execution-kind/runtimeclass.yaml new file mode 100644 index 0000000000..80524fc6c3 --- /dev/null +++ b/deploy/mecatl-execution-kind/runtimeclass.yaml @@ -0,0 +1,5 @@ +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: qualification-runc +handler: runc diff --git a/docs/adr/0027-cloud-native.md b/docs/adr/0027-cloud-native.md index c936159c64..764abe0465 100644 --- a/docs/adr/0027-cloud-native.md +++ b/docs/adr/0027-cloud-native.md @@ -793,6 +793,14 @@ durable artifact survives and is reloaded), or **lost** (gone, possibly leaking) | # | Resource | Owner | Scope | Cleanup today | Re-attach | Evidence | |---|---|---|---|---|---|---| +| 79 | Native-execution reference-intent reconciler ticker and worker | `app.Build` | process | `Built.Close` cancels and joins the bounded worker before closing the provider client; each pass handles at most 64 intents | reconstructible; unresolved intents remain durable in `ExecutionEnvironment.status.references` and are retried after restart | `internal/app/reference_intent_reconcile.go` (`startReferenceIntentReconcile`); `internal/adapter/server/service.go` (`ReconcileReferenceIntents`) | +| 80 | Native-execution provider gRPC connection and bounded RPC limiter identity counters | `executionclient.Provider` / execution-provider process | process / active RPC | provider close releases the connection; limiter reservations release at RPC return and idle per-client entries are removed | reconstructible from operator endpoint and mTLS files; limiter counters reset by design | `internal/adapter/executionclient/client.go` (`Client`, `Provider`, `Close`); `internal/adapter/executioncontroller/security_limit.go` (`RPCLimiter`) | +| 81 | Native-execution active run and operation renewal handles | `executionclient.Provider` / `executioncontroller.Store` | run / operation | context cancellation joins renewal, while terminal receipts or explicit release clear only the matching durable holder; uncertain expiry is retained and fenced | persisted in `ExecutionEnvironment.status.activeRun` and `status.activeOperation`; restart observes rather than adopts an unknown holder | `internal/adapter/executionclient/client.go` (`Provider`); `internal/adapter/executioncontroller/store.go` (`Store`) | +| 82 | Native-execution controller informers, queue, and worker | execution-provider `Reconciler` | process | provider context stops informers and worker; `Run` shuts down the queue | reconstructible from Kubernetes objects after profile-resource preflight and cache synchronization | `internal/adapter/executioncontroller/controller.go` (`Reconciler`, `Initialize`, `Run`) | +| 83 | Security authority immutable snapshot, reload ticker, and durable high-water ConfigMap | execution-provider `SecurityManager` | process snapshot / cluster-durable authority | context stops reload; candidates publish atomically, invalid candidates clear readiness, and no secret bytes enter the ConfigMap | snapshot reconstructible from projected files; generation, canonical digest, and public fingerprints persist in the authority ConfigMap, retained across default Helm uninstall and reused byte-for-byte by a quiesced same-identity reinstall | `internal/adapter/executioncontroller/security.go` (`SecurityManager`, `Reload`, `Run`); `deploy/helm/mecatl-execution/templates/provider.yaml` | +| 84 | Profile-allocation slot CAS ledger | execution-provider `Store` | namespace, per profile | exact final CR deletion releases a slot; conflicts retry with resource-version CAS and invalid state fails closed | persisted in the `mecatl-execution-profile-allocations` ConfigMap and reconciled against retained CRs; default Helm uninstall retains the ledger and same-identity reinstall preserves its data | `internal/adapter/executioncontroller/store.go` (`reserveProfileSlot`, `releaseProfileSlot`); `deploy/helm/mecatl-execution/templates/provider.yaml` | +| 85 | Bounded revocation receipt ledger and lifecycle/migration proof state | `ExecutionEnvironment` status | environment | at most 32 revocation receipts are retained; lifecycle and migration records clear only after exact terminal/UID proof and successful phase completion | persisted in CR status; retries use operation ID and fingerprint, while evicted receipt IDs remain rejected | `internal/adapter/executioncontroller/store_revoke.go` (`RevokeEnvironment`); `internal/adapter/executioncontroller/admin_lifecycle.go`; `internal/adapter/executioncontroller/controller_lifecycle.go` | +| 86 | Native-execution workload NetworkPolicies and retained profile/security configuration | Helm release, namespace-scoped | retained allocation lifetime | default uninstall retains default-deny/profile policies and both configuration ConfigMaps; no deletion hook | persisted; `profiles` ConfigMap `lifetime.json` pins profiles, network configuration, and security Secret name. Same-release/namespace ownership checks permit adoption; incompatible configuration or missing ledgers with retained allocations fails closed. Operator-owned Secret material is retained outside Helm | `deploy/helm/mecatl-execution/templates/network-policy.yaml`; `deploy/helm/mecatl-execution/templates/provider.yaml`; `deploy/helm/mecatl-execution/templates/lifetime-check.yaml` | | 58 | Title coordinator (two workers, 64-item queue, dedupe set, and retry timers) | `server.Service` / `app.Build` | process | `Service.Close` cancels and joins workers before session dependencies close; queue admission is non-blocking and bounded | reset-by-design; durable session attempt records make a crash-interrupted claim terminal rather than rebilling | `internal/adapter/server/title_coordinator.go` (`titleCoordinator`) | | 59 | Provider OIDC bearer sources and runtime registry | `internal/cliconfig.NativeEndpointLoader` / `NativeEndpointRuntime` | process, one per configured OIDC provider | `NativeEndpointLoader.Close` closes every loader-owned runtime; each runtime closes its serving Store handles | reconstructible from explicit operator provider configuration and an exact protected record; access tokens reset and are refreshed from that record | `internal/cliconfig/native_endpoint.go` (`NativeEndpointLoader`, `NativeEndpointRuntime`, `Source`, `Close`) | | 60 | Caller OIDC JWKS validator cache | `authn/oidc.Validator` | process | validator `Close` stops its refresh work during command-root shutdown | reset/refetch: a restart fetches current signing keys; this authenticates callers only and is separate from provider issuer/gateway trust | `authn/oidc/oidc.go` (`Validator`, `Close`) | @@ -981,6 +989,19 @@ rather than papered over. | 72 | Per-generation isolated Redis follow client and bounded connection pool | each redisstore `clientGeneration`, owned by `clientGenerations` | one follow client per live or retiring credential generation; `PoolSize` and `MaxActiveConns` use the configured process-local follow-pool size | a rejected candidate closes both clients; generation retirement closes the pair after its leases release. `Store.Close` may asynchronously force-close only the follow client when an admitted read misses the cooperative join bound, while durability clients retain their lease-safe close rule | reconstructible from Redis connection settings and mounted credential files. A restart builds and probes a new pair; it retains no watch position or session authority | `internal/adapter/redisstore/generation.go` (`clientPair`, `buildClientPair`, `forceCloseFollow`); `internal/adapter/redisstore/reload.go` (`reloadCandidate`); ADR 0330 | | 73 | Redis follower admission, cancellation, and join registry | redisstore `Store` owns one `followerRegistry` | process; one entry per admitted `ReadAfter` iterator with `Follow:true`, capped by `MaxFollowers` | iterator release removes its entry on every exit. `Store.Close` linearizes against admission, rejects new followers, cancels the active snapshot, waits cooperatively, and retains bounded late-cleanup ownership after a follow-client force-close | reset-by-design and reattached by the client. The registry contains no cursor or durable session state; after restart the client resumes with its last processed cursor and unchanged filter | `internal/adapter/redisstore/followers.go` (`followerRegistry`); `internal/adapter/redisstore/cursoreventlog.go` (`ReadAfter`); `internal/adapter/redisstore/redisstore.go` (`Close`); ADR 0330 | | 74 | Prompt-free persisted-ask continuation contexts and detached relay join registry | `internal/adapter/server.Service` owns the post-acceptance context, exact resumed `agent.Run`, recorder/relay goroutine, and `detachedControlWG`; the unary caller owns only work before atomic acceptance | one process-local entry per accepted persisted ordinary-ask resolution | relay admission adds to the Service wait group before the run is published and is fenced before shutdown wait. `Service.Close` first closes admission, marks every already-durable awaiting handoff `preserveDurable` and every cancellation `cancelSignaled` under the persistence barrier, then cancels contexts/runs and waits with the engine-close bound. Graceful drain and lease loss use the same mark-before-signal order. A preserved terminal relay cannot overwrite the awaiting snapshot | **derived from durable state / reset local ownership**: the context, run, recorder, marker bits, and join count disappear on process loss. A replacement acquires the distributed session lease, reloads the authoritative snapshot after acquisition, and repeats exact run/state/ask/origin/placement validation before rehydrating. See List 2 row 47 | `internal/adapter/server/service.go` (`ResolveRunAsk`, `validatePersistedRunAsk`, `reserveDetachedControlRelay`, `relayDetachedControlRun`, `Close`, `GracefulDrain`, `cancelRegisteredRunState`); ADR 0347 | +| 75 | Experimental execution-provider bounded mTLS gRPC server and Kubernetes REST transports | `cmd/mecatl-execution-provider` | provider process | process cancellation gracefully stops the gRPC server (then hard-stops after the shutdown bound); Kubernetes clients own no separate application goroutine | **derive** from explicit flags, mounted mTLS material, in-cluster configuration, and the live Kubernetes API. Disabled `mecak8s` composition creates none of these resources | `cmd/mecatl-execution-provider/main.go`; `internal/adapter/executioncontroller/handler.go` (`Handler`) | +| 76 | ExecutionEnvironment informer, rate-limited workqueue, and controller worker | `executioncontroller.Reconciler` | provider process, namespace-scoped | the Run context stops the informer and worker; `Run` shuts down the queue before return | **derive** by listing and watching the external Kubernetes CRs after restart. Startup marks any persisted active operation `FenceUnknown` before admitting mutations | `internal/adapter/executioncontroller/controller.go` (`Reconciler`, `Initialize`, `Run`) | +| 77 | ExecutionEnvironment CR, retained PVC, executor Pod, and one status-held active-operation claim | Kubernetes API and storage provider; the execution controller reconciles them | logical remote environment | explicit retirement removes a proven-terminal Pod but retains the PVC. CR deletion remains finalizer-blocked until retirement and quiescence; this candidate has no PVC-delete path | **external Kubernetes state**: CR status, PVC data, and current Pod references survive provider or `mecak8s` process restart according to cluster/storage durability. An interrupted active-operation claim becomes fail-closed `FenceUnknown`, not reconstructed authority | `internal/adapter/executioncontroller/controller.go` (`Reconcile`, `reconcileRetiring`, `reconcileDeletion`); `internal/adapter/executioncontroller/store.go` (`execute`) | +| 78 | In-memory grant signer/verifier configuration, short-lived signed grants, and revocation maps | execution-provider `Handler`; each remote `grantContext` caches one grant | provider process / bound session | grants expire; client reattachment refreshes them. Signing and verifier values leave process memory at shutdown. Full durable key rotation and revocation are not implemented | **reset-by-design / derive** from mounted signing configuration and exact external environment state. A restart does not preserve in-memory nonce/key revocation, so this draft is not production rotation or revocation | `internal/adapter/executioncontroller/handler.go` (`GrantSigner`, `HandlerConfig`, `sign`); `internal/executionenv/grant.go`; `internal/adapter/executionclient/client.go` (`grantContext`) | +| 79 | Mecak8s execution-provider bounded mTLS gRPC client connection and per-binding remote Workspace/runner | `executionclient.Client` and the Service placement registry | process transport / session environment | `Client.Close` closes its gRPC connection; session engines and environments follow existing Service teardown. Each authority lookup and file/command call has caller or adapter bounds | **derive** by exact reattachment from the persisted `session.EnvironmentRef` and owner-bound provider state. Missing, changed, or fenced environments fail closed with no local fallback | `internal/adapter/executionclient/client.go` (`Client`, `Provider`, `workspace`, `runner`) | + +**Experimental native-execution re-audit (List 1 / List 2).** List 1 rows +75–79 inventory the provider server and transports, controller informer and queue, +external Kubernetes runtime state, grant state, and `mecak8s` client transport. +List 2 row 48 records the split between the persisted exact environment reference, +external CR/PVC state, and process-local operation/grant state. The implementation +is draft: run-wide ownership, complete successor reference lifecycle, controlled +replacement/PVC deletion, and durable grant-key rotation remain unresolved. **Mecatui status-line re-audit (ADR 0247).** List 1 row 64 owns the local source lifecycle. It contains only display-safe input and generated presentation state, never session authority or transcript data, so List 2 gains no row. Shutdown is bounded rather than silently abandoning a live command tree; a process restart deliberately starts from fresh local status state. @@ -1178,6 +1199,11 @@ what is persisted), **reset-by-design** (documented, acceptable), | # | Item | Where it lives | On restart today | Decision | Phase | |---|---|---|---|---|---| +| 48 | Native-execution pending reference intents | `ExecutionEnvironment.status.references` structured PendingCreate/PendingDelete records | process-local reconciler state resets; complete environment, owner, binding, source binding, operation ID, state, and creation time remain in the CR | **persist-in-CR**: the next bounded reference-intent pass commits, confirms, or cancels from durable state; unknown intent remains retained | native execution | +| 49 | Native-execution run/operation ownership and renewal | `ExecutionEnvironment.status.activeRun` and `status.activeOperation` | renewal goroutines and handles disappear; holder, claim/run IDs, epoch, generation, expiry, and operation fingerprint remain | **persist-in-CR and fail closed**: expiry becomes `FenceUnknown`; another replica never clears unknown ownership by timeout alone | native execution | +| 50 | Security authority and grant revocation | projected security files, authority high-water ConfigMap, and CR `grantGeneration` plus bounded revocation receipts | immutable in-memory snapshot and limiter counts reset; durable generation/digest/fingerprints, environment generation, and receipts remain | **persist authority metadata and derive snapshot**: reload all secret bytes atomically; drift, rollback, expiry, or unavailable ledger keeps readiness false | native execution | +| 51 | Native-execution allocation, lifecycle proof, and schema migration | profile-allocation ConfigMap plus `ExecutionEnvironment` spec/status, including runtime UIDs, termination proof, lifecycle operation, and migration operation | controller queues reset; retained CR/PVC and every operation phase remain observable | **persist-in-Kubernetes**: cache sync and exact UID/CAS checks resume convergence; malformed or unknown schema is retained for explicit administration | native execution | +| 52 | Native-execution Helm lifetime configuration, isolation, and ledgers | retained workload NetworkPolicies, profile/security-manifest ConfigMaps, authority high-water ConfigMap, and capacity ConfigMap | provider absence leaves enforcement and data in Kubernetes; Helm release history may disappear but original ownership metadata remains | **persist-in-Kubernetes and explicitly adopt**: same release/namespace and unchanged lifetime configuration only; live lookup renders actual ledger data while writers are quiesced. Missing history with retained allocations, foreign ownership, and incompatible configuration fail closed; no bootstrap reset. CRDs upgrade manually before compatible provider/schema migration | native execution N2; List 1 rows 83, 84, 86 | | 36 | Title coordinator queue/dedupe and in-flight title call | `server.titleCoordinator` | process | `Service.Close` cancels and joins; pending queued work is discarded | reset-by-design: durable incomplete attempts are reloaded as `interrupted` and exhausted, so no uncertain provider call is retried | title coordinator (SHIPPED) | | 37 | Native endpoint access and refresh tokens | protected `mecatl/provider-oidc/v1` record; transient access material in `nativeBearerSource` | access tokens reset; the encrypted exact-identity record reloads refresh state only | persist-in-record / derive: no OAuth material enters a session snapshot, event, diagnostic, or RPC; an unavailable, corrupt, or identity-drifted record is not-enrolled | ADR 0329 | | 38 | Native endpoint issuer/gateway clients, keyring/store handles, and transaction locks | `NativeEndpointRuntime` and `TransactionLocker` | handles and locks are closed/released; a restart has no live runtime state | explicitly reattach/reset: Build or local lifecycle reopens them only from explicit configuration; issuer/gateway trust stays separate and the next lifecycle operation reacquires its lock | ADR 0329 | @@ -1228,6 +1254,7 @@ what is persisted), **reset-by-design** (documented, acceptable), | 45 | Session-scoped MCP broker incarnation, local attachment, and parked authorization (P10–P11; List 1 rows 68–70) | the snapshot persists `session.Session.ExternalBinding` plus the exact private `PendingAuthorization`; the process-owned broker holds logical state, grants/tokens, callback/replay state, timers, and prepared continuations in memory | same-process controls reattach only when the opaque binding matches exactly. **The binding value itself, not merely its persistence, is what defeats a restart**: it is `bindingPrefix + "." + generation`, where `bindingPrefix` is random per process and `generation` is an in-memory counter (`internal/adapter/mcpbroker/runtime.go`), so a byte-perfect restored value is structurally guaranteed stale the moment the broker process restarts — no persistence fix closes this alone. A process restart loses in-process broker authority and timers; a lease-gated status-unavailable observation pairs the parked call as interrupted and never replays the protected mutation. The one narrow exception is the pre-prompt workspace-enrollment seam (`rebindBrokerAttachment`, `internal/adapter/server/mcp_broker.go`), which adopts the live incarnation ONLY where nothing durable was built on the lost one — it must not widen to authorization control paths, which stay fail-closed. A future remote broker can return pending, granted, or a concrete terminal status through the same control flow | **durable pending / reset runtime / derive control**: the aggregate preserves exact effective and deferred calls, while timers and prepared runs are recreated only from authoritative live state. Public wire and client surfaces remain deferred to P12–P14 | P10, P11 | | 46 | Workspace-enrollment outer correlation versus ToolHive inner authorization storage (ADR 0311; List 1 rows 68–69 and 71) | the snapshot persists the exact `PendingWorkspaceEnrollment`; mecatl's runtime owns its process-local callback state/incarnation mapping, while ToolHive memory or Redis storage owns separate inner upstream authorization/token records | Redis-backed ToolHive records may survive restart, but the outer operation-to-session correlation does not. Rebinding the pre-prompt session to a fresh runtime only discards the exact old pending correlation and permits a fresh enrollment; it is not authority to search ToolHive storage or resume an old operation. A wrong replica has the same correlation loss | **persist aggregate / reset outer correlation / do not infer inner recovery**: terminal observations are repeatable in-process until the aggregate save succeeds; after process loss, discard only the exact old pre-prompt correlation and begin a fresh enrollment. Durable outer correlation, replica routing, and restart recovery remain deferred | ADR 0311 | | 47 | Prompt-free ordinary-ask handoff and acknowledgement-only resumed-run lifecycle (ADR 0347; List 1 row 74) | the existing session snapshot persists `StateAwaiting`, exact `RunID`, `PendingAsk`, mode, and `EnvironmentRef`; the accepting Service owns only the reconstructed run/context/relay described in List 1 | a restart or replica handoff loses local execution and join bookkeeping but retains the last authoritative awaiting or terminal snapshot. An accepting replica acquires the distributed lease, reloads after acquisition, and revalidates exact run, awaiting state, ask ID, non-plan origin, and placement before `ResumeApproval`; a stale pre-lease snapshot cannot replay allow-once work | **persist existing snapshot, reconstruct under fresh lease**: no new durable field is introduced. Before shutdown cancellation reaches a detached continuation, an already-persisted awaiting point is marked for preservation and the relay is joined boundedly; otherwise terminal cancellation is persisted so accepted work is not ambiguously replayed. Plan asks remain on their dedicated resolution path | 0347 | +| 48 | Experimental native Kubernetes environment identity, references, grants, and active operation | the session snapshot persists the exact `EnvironmentRef`; the Kubernetes `ExecutionEnvironment` CR/status and retained PVC hold provider-side identity, references, operation state, and workspace data; grants and client transports are process memory | restart reloads the session reference and exactly reattaches through the provider to the external CR. The controller derives its queue from an informer list/watch. A provider restart that observes an active operation clears the claim only by marking the environment `FenceUnknown`; it does not infer completion or takeover. Short-lived grants and in-memory revocation state reset | **persist-in-snapshot + external Kubernetes state + derive/reset-by-design**: derive the live Workspace/runner from exact reattachment; derive controller work from CRs; reset transports, queue state, and expired grants. Complete successor reference release, run-wide ownership, automated fenced recovery, controlled replacement/PVC deletion, and durable multi-key rotation are not implemented | experimental native execution; List 1 rows 75–79 | Two ledger observations worth stating in prose: diff --git a/docs/architecture.md b/docs/architecture.md index fa9cb8c72a..7b304ee2dc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -70,6 +70,28 @@ This page is the overview and router; the big picture and the layering rule are - **[Session titles & durable token accounting](architecture/domain-model.md#session-titles-and-durable-token-accounting)** — operator renaming, opt-in asynchronous generation, and canonical token usage. - **[Deployment & server hardening](architecture/deployment-and-hardening.md)** +### Native Kubernetes execution + +The opt-in native execution path keeps the controller boundary outside `mecak8s`. +`cmd/mecatl-execution-provider` owns the private authenticated API +(`mecatl.execution.v1.ExecutionProviderService`, protocol `execution-grpc/1`) over +bounded mTLS gRPC and the namespaced `ExecutionEnvironment` reconciler. It +creates a retained PVC and credential-free executor Pod from an operator-defined, +digest-pinned profile. Startup validates each profile's named RuntimeClass and +StorageClass before readiness or Pod creation. `cmd/mecatl-executor` serves +bounded file operations and foreground commands inside `/workspace`; the +host-side `internal/adapter/executionclient` adapts those operations to the +existing `tool.Environment` contract. + +Run claims, reference transactions, replacement, retirement, retained-PVC +deletion, schema migration, and grant revocation use exact CR status identities +and resource-version CAS. Security material reloads as one immutable snapshot and +is checked against a durable generation/digest ledger on every RPC. Unknown +operation ownership or unproven Pod termination leaves the environment fenced. +Disabled composition builds no execution client, and the separate provider chart +has no dependency on the `mecak8s` chart. Remote Git/project ingestion, +background commands, and delegated filesystem execution remain unsupported. + ### Protected-resource discovery Both `mecated` and `mecak8s` use the shared OIDC profile flags. When configured, diff --git a/docs/design/IMPLEMENTATION-NOTES.md b/docs/design/IMPLEMENTATION-NOTES.md index a5521411a2..5f56c0bdbd 100644 --- a/docs/design/IMPLEMENTATION-NOTES.md +++ b/docs/design/IMPLEMENTATION-NOTES.md @@ -6536,6 +6536,74 @@ deny-dominant (the inner fold runs first — a configured Deny or configured Ask reaches the checker). Default `false` is the byte-identical un-routed posture table. See `docs/adr/0080-guardrail-routed-escape-checking.md`. +### Native Kubernetes execution provider + +The opt-in provider composes a separately deployed provider through +`internal/adapter/executionclient/client.go` (`Provider`). The private +`mecatl.execution.v1.ExecutionProviderService` gRPC API uses private protocol +`execution-grpc/1`. It requires mTLS, authenticates exactly one allowlisted URI SAN, binds short-lived +Ed25519 grants to client, owner hash, session binding, environment revision, +epoch, and a closed operation set, and sends credential-free requests to the +workload helper. `internal/adapter/executioncontroller/handler.go` (`Handler`) +owns authentication and grant checks; `internal/adapter/executioncontroller/store.go` +(`execute`) claims the one CR status operation before dispatch and marks +uncertain completion `FenceUnknown`. + +`internal/adapter/executioncontroller/controller.go` (`Reconciler`) watches the +namespaced `ExecutionEnvironment` CR and creates a retained PVC plus a +single executor Pod from an operator-only, digest-pinned profile. Provisioning +errors return alongside any condition-write error so the existing rate-limited +workqueue retries without a finite budget or another Kubernetes event. Repeated +reconciliation never replaces a missing authoritative PVC/Pod or adopts a foreign +resource. The Pod +has no service-account token and contains the fixed helper built by +`build/execution-workload/Dockerfile`. `internal/executionexecutor/executor.go` +(`Executor.Execute`) dispatches bounded file operations and foreground Shell in +`/workspace`. Authority-resource resolution is itself an authenticated +`file.resolve_authority` operation: the executor invokes the existing physical +`osfs` resolver, and the remote Workspace returns only the confined canonical +`/workspace` identity. Resolver, transport, authorization, or confinement errors +fail closed before the external authority evaluator runs; there is no lexical +client-side substitute. + +Composition selects this path only when `Config.RemoteExecution` is set. +`internal/app/project_ingestion.go` (`projectIngestionAdmitted`) rejects host-project +steering for remote deployments even when that local root is trusted. The remote +instruction variant retains user-tier rules; command listing never opens a local +workspace and the remote engine uses the no-op command expander. Operator-global +skills remain available. The +remote catalog excludes local project ingestion, schedules, SkillDraft, +Parallel, and Team, while explicit `profile:"no-fs"` still selects the existing +no-FS placement. The `mecatl-execution` and `mecak8s` charts remain independent. Run claims, +transactional references, replacement, retirement, retained-PVC deletion, +schema migration, and generation-based grant revocation persist their exact +operation identities in CR status and use resource-version CAS across replicas. +Prototype migration is explicitly limited to schema 0/1 objects whose existing +Pod and PVC UIDs are observable and whose runtime resources already satisfy the +current ownership, finalizer, `RestartPolicyNever`, profile, and security shape; +foreign, missing, or legacy-insecure runtime resources are refused rather than +adopted. Operators must reconstruct an incompatible prototype as a new current +allocation after separately preserving its data; migration never force-deletes +or silently replaces it. +The controller validates the configured RuntimeClass and StorageClass names +before readiness, then synchronizes informer caches without clearing peer +operations. Security reload publishes one immutable TLS/client-policy/grant-key +snapshot only after matching the durable authority ConfigMap high-water record. +`internal/adapter/executioncontroller/admin_lifecycle.go` (`adminSubject`) shares +creator-scope, exact owner, and revision admission across all six administrative +RPC families, including `internal/adapter/executioncontroller/store_revoke.go`. +The authenticated request carries `ClientPolicy.AdministratorFor` from the same +security snapshot that verified its peer; it does not re-read policy or substitute +the administrator for the immutable creator. Empty scope retains self-administration. +Sorted creator scopes participate in the authority digest; removal denies subsequent +requests and receipt replay on existing connections, without cancelling already +admitted safe lifecycle reconciliation. Data-plane and owner-attestation checks +are unchanged. +Unproven executor loss remains `FenceUnknown`; only exact built-in terminal proof +or an external platform fencing procedure can recover it. Remote project +sources, background commands, schedules, and delegated filesystem execution +remain deliberately unsupported. + ### Server-owned session placement and worktree successors (ADR 0291) Placement is server-owned across embedded, loopback, remote, and cloud-native composition. diff --git a/docs/design/PRODUCTION-READINESS.md b/docs/design/PRODUCTION-READINESS.md index 681d8d3eb7..5693bd82e7 100644 --- a/docs/design/PRODUCTION-READINESS.md +++ b/docs/design/PRODUCTION-READINESS.md @@ -14,6 +14,7 @@ record; current behaviour is in the linked [architecture](../architecture.md) do | Subsystem | Status | Design record | Architecture | |---|---|---|---| +| Native Kubernetes execution provider | 🔨 Candidate qualification complete; existing [PR #1614](https://github.com/stacklok/mecatl/pull/1614) and plan #1579 remain draft, not approved, merged, or shipped. Tested runtime `2020e59934369e8f771059d0c173fa4cbf964c7e`: [manual exact-head run 35700303737](https://github.com/stacklok/mecatl/actions/runs/35700303737) passed all 11 production Kind+Calico scenarios, including retained-resource Helm lifecycle, then native OpenRouter coding with independent exact-ref typed-gRPC file/test verification and cleanup. The live-only storage-helper rewrite was removed (executor UID 65532 could not mkdir beneath a root-owned 0755 parent); qualified storage is preserved and creation completed in 8 seconds. The verifier now uses read-only binding lookup then exact Attach/run acquisition, not an invalid Ensure replay; provider semantics are unchanged. Current runtime-head PR checks: 31 success / 8 skipped / 0 failed / 0 pending; synthetic merge SHA and full-race CI skips are distinguished in the receipt. Earlier local full-race and latest targeted-race evidence do not imply draft full-race CI ran. ⛔ Human API/schema/security approval, release maturity, final panel/human review and merge remain open; partial test-scope caveats remain. Documentation-only descendants retain the tested-runtime attribution. Historical failures and their resolution remain in the [completion receipt and proof history](../acceptance/native-kubernetes-execution.md#candidate-completion-receipt). | [0350](../adr/0350-native-kubernetes-execution.md) · [acceptance contract](../acceptance/native-kubernetes-execution.md) | [native execution mechanics](./IMPLEMENTATION-NOTES.md#native-kubernetes-execution-provider) | | Multi-provider / multi-model | ✅ P0+P1+live listing & metadata · ✅ same-provider history carryover (issue #20) · ✅ experimental manual `openai-codex` subscription token (ADR 0215) · ✅ ToolHive OpenAI Responses and native Anthropic protocol providers over one gateway identity (ADR 0334) · ✅ internal opaque credential-store substrate consumed by MCP profiles with local encrypted-file Store and explicit read-only environment Reader (issues #519/#542) · ⛔ disk cache (P2) · ⛔ key acquisition/per-client routing/remote stores or Kubernetes Secret `resourceVersion` CAS backend (P3) | [MULTI-PROVIDER.md](../adr/0016-multi-provider.md) · [0215](../adr/0215-openai-subscription-manual-token.md) · [0218](../adr/0218-credential-store.md) · [0221](../adr/0221-read-only-credential-source.md) · [0334](../adr/0334-toolhive-protocol-specific-providers.md) | [providers](../architecture/providers.md) · [credential store](../architecture.md#internal-credential-store) | | Remote mecatui OIDC login, refresh, recovery, and logout | ✅ fixed-callback PKCE enrollment, actual-record-only root-scoped keyring migration, one refresh/enroll/logout target transaction with ambiguous-commit compensation, target-bound encrypted credentials, token-demand-gated proactive refresh, exact structured-rejection cleanup, ownership-checked same-target recovery, CAS-safe local-first logout, one-budget best-effort RFC 7009 revocation, and target-aware TLS defaults with saved-auth verified-TLS enforcement shipped; ✅ offline rotation/rejection/crash-residual/logout/TLS-policy coverage · ⚠️ no cross-store journal: a crash may require login or leave an unenumerable credential-only orphan · ⚠️ legacy zero-padded-port credential identities require one login · ⛔ live Kind qualification remains environment-dependent | [0277](../adr/0277-remote-mecatui-oidc.md) · [0287](../adr/0287-target-aware-mecatui-tls.md) · [0218](../adr/0218-credential-store.md) | [remote login](../tui.md#oidc-connected-server) | | Direct MCP OAuth dynamic client registration | 🔨 Core trusted-profile validation, durable CAS registration, generation-bound no-refresh grant, explicit reset/retry recovery, restart reuse, expiry-to-login-required, protocol/security fixtures, seven named acceptance proofs, and cloud-native resource re-audit complete · ⛔ repeat panel review, implementation PR/merge, and live local-mecatui qualification remain · ⛔ complete resource-bound public-client refresh ([issue #1355](https://github.com/stacklok/mecatl/issues/1355)) | [0325](../adr/0325-direct-mcp-dcr.md) | [credential store](../architecture.md#internal-credential-store) | diff --git a/e2e/k8s_execution/aa_production_migration_test.go b/e2e/k8s_execution/aa_production_migration_test.go new file mode 100644 index 0000000000..86ccae5a5f --- /dev/null +++ b/e2e/k8s_execution/aa_production_migration_test.go @@ -0,0 +1,9 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import "testing" + +func TestKindExecutionProductionCompatiblePrototypeMigration(t *testing.T) { + runKindExecutionProductionCompatiblePrototypeMigration(t) +} diff --git a/e2e/k8s_execution/ab_production_admin_test.go b/e2e/k8s_execution/ab_production_admin_test.go new file mode 100644 index 0000000000..3932ec14fe --- /dev/null +++ b/e2e/k8s_execution/ab_production_admin_test.go @@ -0,0 +1,70 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestKindExecutionProductionScopedAdministrator(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + qualifyDistinctAdministrator(t, ctx, state, kubeconfig, "initial") +} + +func qualifyDistinctAdministrator(t *testing.T, ctx context.Context, state, kubeconfig, phase string) { + t.Helper() + creator, forward := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, creator, "admin-"+phase) + clients := map[string]*executionclient.Client{} + for _, name := range []string{"operations", "wrong-scope", "intruder"} { + client, err := executionclient.New(forward.addr, loadTLS(t, filepath.Join(state, "pki"), name, "mecatl-execution.execution-qualification.svc.cluster.local")) + if err != nil { + t.Fatal("create synthetic admin client") + } + t.Cleanup(func() { client.Close() }) + clients[name] = client + } + rc, release := acquireRun(t, ctx, creator, owner, binding, attached, "admin-data-"+phase) + if _, err := creator.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "admin-sentinel", Data: []byte("creator-only\n")}); err != nil { + t.Fatal("creator data-plane positive:", remoteErrorCode(err)) + } + admin := clients["operations"] + if _, err := admin.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); !isRemoteCode(err, executionenv.CodeNotFound) { + t.Fatal("scoped administrator acquired creator attach privilege:", remoteErrorCode(err)) + } + if _, err := admin.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: "admin-sentinel"}); !isRemoteCode(err, executionenv.CodePermissionDenied) { + t.Fatal("scoped administrator used creator file grant:", remoteErrorCode(err)) + } + if _, err := admin.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "true"}); !isRemoteCode(err, executionenv.CodePermissionDenied) { + t.Fatal("scoped administrator used creator command grant:", remoteErrorCode(err)) + } + release() + before := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + request := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: before.Epoch, ExpectedPodUID: before.PodUID, ExpectedPVCUID: before.PVCUID, OperationID: "admin-replace-" + phase} + for name, code := range map[string]executionenv.ErrorCode{"wrong-scope": executionenv.CodeNotFound, "intruder": executionenv.CodePermissionDenied} { + if err := clients[name].ReplaceExecutor(ctx, request); !isRemoteCode(err, code) { + t.Fatalf("%s admin refusal: %s", name, remoteErrorCode(err)) + } + } + if after := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID); after != before { + t.Fatal("denied admin calls changed runtime state") + } + if err := admin.ReplaceExecutor(ctx, request); err != nil { + t.Fatal("distinct scoped administrator replacement:", remoteErrorCode(err)) + } + after := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Ready && s.PodUID != before.PodUID }) + if after.PVCUID != before.PVCUID || after.Epoch <= before.Epoch { + t.Fatal("scoped replacement failed exact PVC/epoch reconciliation") + } + reattached := waitReady(t, ctx, creator, owner, binding, attached.Environment) + rc, release = acquireRun(t, ctx, creator, owner, binding, reattached, "admin-verify-"+phase) + waitFileContent(t, ctx, creator, rc, "admin-sentinel", "creator-only\n", "scoped replacement") + release() + retireSyntheticEnvironment(t, ctx, creator, kubeconfig, owner, binding, attached.Environment) +} diff --git a/e2e/k8s_execution/file_tools_test.go b/e2e/k8s_execution/file_tools_test.go new file mode 100644 index 0000000000..65b88f56cb --- /dev/null +++ b/e2e/k8s_execution/file_tools_test.go @@ -0,0 +1,61 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/stacklok/mecatl/engine/adapter/fstools" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func qualifyRemoteFileTools(t *testing.T, ctx context.Context, client *executionclient.Client, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) { + t.Helper() + provider, err := executionclient.NewProvider(client, "go") + if err != nil { + t.Fatal(err) + } + handle, err := provider.AcquireRun(ctx, server.ExecutionRunRequest{Ref: session.EnvironmentRef{Kind: "kubernetes", ID: ref.ID, Revision: ref.Revision}, Principal: &session.Principal{Issuer: owner.Issuer, Subject: owner.Subject, GrantType: session.GrantTypeUser}, BindingID: session.SessionID(binding), RunID: "file-tool-matrix"}) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := handle.Release(ctx); err != nil { + t.Error(err) + } + }() + env := handle.Environment() + call := func(body tool.Tool, args string, wantError bool, contains string) { + t.Helper() + result, err := body.Execute(ctx, session.ToolCall{ID: "matrix", Name: body.Spec().Name, Args: json.RawMessage(args)}, env) + if err != nil || result.IsError != wantError || !strings.Contains(result.Content, contains) { + t.Fatalf("%s: result=%+v err=%v", body.Spec().Name, result, err) + } + } + if _, err := env.Workspace().CreateFile(ctx, "matrix/source.go", []byte("package matrix\n// before\n")); err != nil { + t.Fatal(err) + } + call(fstools.EditTool{}, `{"path":"matrix/source.go","old_string":"before","new_string":"after"}`, true, "") + call(fstools.WriteTool{}, `{"path":"matrix/source.go","content":"clobber"}`, true, "") + call(fstools.CopyTool{}, `{"source":"matrix/source.go","destination":"matrix/copied.go"}`, false, "") + call(fstools.MoveTool{}, `{"source":"matrix/copied.go","destination":"matrix/moved.go"}`, false, "") + call(fstools.RemoveTool{}, `{"path":"matrix/moved.go"}`, false, "") + call(fstools.ReadTool{}, `{"path":"matrix/source.go"}`, false, "before") + call(fstools.EditTool{}, `{"path":"matrix/source.go","old_string":"before","new_string":"after"}`, false, "") + call(fstools.WriteTool{}, `{"path":"matrix/source.go","content":"package matrix\n// final\n"}`, false, "") + call(fstools.WriteTool{}, `{"path":"matrix/new.go","content":"package matrix\n"}`, false, "") + call(fstools.ListDirTool{}, `{"path":"matrix"}`, false, "source.go") + call(fstools.GrepTool{}, `{"path":"matrix/*.go","pattern":"final"}`, false, "final") + call(fstools.GlobTool{}, `{"pattern":"matrix/*.go"}`, false, "source.go") + data, err := env.Workspace().Read(ctx, "matrix/source.go") + if err != nil || string(data) != "package matrix\n// final\n" { + t.Fatalf("persisted matrix=%q err=%v", data, err) + } +} diff --git a/e2e/k8s_execution/fixture/credentialloader/main.go b/e2e/k8s_execution/fixture/credentialloader/main.go new file mode 100644 index 0000000000..7f69f1882e --- /dev/null +++ b/e2e/k8s_execution/fixture/credentialloader/main.go @@ -0,0 +1,279 @@ +//go:build kind_execution_e2e + +// Command credentialloader is the only qualification component permitted to read +// a real provider credential. It creates a new, narrowly labelled Secret and +// never reads Secret data back from Kubernetes. +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utilvalidation "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + namespace = "execution-qualification" + secretKey = "OPENROUTER_API_KEY" + maxCredentialBytes = 16 << 10 + maxReceiptBytes = 4 << 10 +) + +type cleanupReceipt struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + UID types.UID `json:"uid"` + Context string `json:"context"` + Kubeconfig string `json:"kubeconfig"` +} + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, errorMessage(err)) + os.Exit(1) + } +} + +func errorMessage(err error) string { + return "credential loader: " + classify(err) +} + +func run(args []string) error { + if len(args) != 5 { + return errors.New("usage") + } + mode, kubeconfig, contextName, name, receiptPath := args[0], args[1], args[2], args[3], args[4] + if (mode != "stage" && mode != "delete") || !strings.HasPrefix(name, "mecak8s-live-") || len(utilvalidation.IsDNS1123Subdomain(name)) != 0 { + return errors.New("invalid arguments") + } + kubeconfig, receiptPath, err := validatePrivatePaths(kubeconfig, receiptPath, name) + if err != nil { + return err + } + + // Resolve kube configuration (including any auth plugin) and prove API access + // before the credential file is opened. + cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, + &clientcmd.ConfigOverrides{CurrentContext: contextName}, + ).ClientConfig() + if err != nil { + return fmt.Errorf("kube configuration: %w", err) + } + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return fmt.Errorf("kube client: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if _, err = client.Discovery().ServerVersion(); err != nil { + return fmt.Errorf("kube access: %w", err) + } + + secrets := client.CoreV1().Secrets(namespace) + if mode == "delete" { + if err := deleteStagedSecret(ctx, secrets, receiptPath, kubeconfig, contextName, name); err != nil { + return err + } + fmt.Println(`{"status":"deleted"}`) + return nil + } + path := os.Getenv("MECATL_EXECUTION_CREDENTIAL_FILE") + credential, err := loadCredential(path) + if err != nil { + return err + } + defer clear(credential) + if err := stageSecret(ctx, secrets, receiptPath, kubeconfig, contextName, name, credential); err != nil { + return err + } + fmt.Println(`{"status":"staged","provider":"openrouter"}`) + return nil +} + +func stageSecret(ctx context.Context, secrets v1.SecretInterface, receiptPath, kubeconfig, contextName, name string, credential []byte) error { + created, err := secrets.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: map[string]string{"app.kubernetes.io/managed-by": "mecatl-execution-live-qualification"}}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{secretKey: credential}, + }, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("secret stage: %w", err) + } + if created == nil || created.UID == "" { + return errors.New("secret staged without UID; cleanup pending") + } + receipt := cleanupReceipt{Name: name, Namespace: namespace, UID: created.UID, Context: contextName, Kubeconfig: kubeconfig} + if err := writeReceipt(receiptPath, receipt); err != nil { + uid := receipt.UID + _ = secrets.Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + return errors.New("cleanup receipt creation failed; cleanup status ambiguous") + } + return nil +} + +func deleteStagedSecret(ctx context.Context, secrets v1.SecretInterface, receiptPath, kubeconfig, contextName, name string) error { + receipt, err := readReceipt(receiptPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("cleanup receipt: %w", err) + } + if receipt.Name != name || receipt.Namespace != namespace || receipt.Context != contextName || receipt.Kubeconfig != kubeconfig { + return errors.New("cleanup receipt identity mismatch") + } + if receipt.UID == "" { + return errors.New("secret staged without UID; cleanup pending") + } + uid := receipt.UID + err = secrets.Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("secret cleanup: %w", err) + } + if err := os.Remove(receiptPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return errors.New("cleanup receipt removal failed") + } + return nil +} + +func validatePrivatePaths(kubeconfig, receiptPath, name string) (string, string, error) { + if !filepath.IsAbs(kubeconfig) || !filepath.IsAbs(receiptPath) { + return "", "", errors.New("state paths must be absolute") + } + kubeconfig, err := filepath.EvalSymlinks(kubeconfig) + if err != nil { + return "", "", errors.New("kubeconfig unavailable") + } + kubeInfo, err := os.Stat(kubeconfig) + if err != nil || !kubeInfo.Mode().IsRegular() || kubeInfo.Mode().Perm()&0o077 != 0 { + return "", "", errors.New("kubeconfig must be a private regular file") + } + stateDir, err := filepath.EvalSymlinks(filepath.Dir(receiptPath)) + if err != nil || stateDir != filepath.Dir(kubeconfig) { + return "", "", errors.New("receipt path must be in kubeconfig state directory") + } + info, err := os.Stat(stateDir) + if err != nil || !info.IsDir() || info.Mode().Perm()&0o077 != 0 { + return "", "", errors.New("state directory must be private") + } + base := filepath.Base(receiptPath) + if base != "live-secret-"+name+".receipt.json" || filepath.Clean(receiptPath) != filepath.Join(stateDir, base) { + return "", "", errors.New("invalid receipt path") + } + return kubeconfig, filepath.Join(stateDir, base), nil +} + +func writeReceipt(path string, receipt cleanupReceipt) error { + blob, err := json.Marshal(receipt) + if err != nil { + return err + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + ok := false + defer func() { + _ = f.Close() + if !ok { + _ = os.Remove(path) + } + }() + if _, err := f.Write(append(blob, '\n')); err != nil { + return err + } + if err := f.Sync(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + ok = true + return nil +} + +func readReceipt(path string) (cleanupReceipt, error) { + var receipt cleanupReceipt + f, err := os.Open(path) + if err != nil { + return receipt, err + } + defer f.Close() + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 || info.Size() > maxReceiptBytes { + return receipt, errors.New("receipt must be a bounded private regular file") + } + dec := json.NewDecoder(io.LimitReader(f, maxReceiptBytes+1)) + dec.DisallowUnknownFields() + if err := dec.Decode(&receipt); err != nil { + return receipt, errors.New("invalid cleanup receipt") + } + var extra any + if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { + return receipt, errors.New("invalid cleanup receipt") + } + return receipt, nil +} + +func loadCredential(path string) ([]byte, error) { + if path == "" || !filepath.IsAbs(path) { + return nil, errors.New("credential file must be an absolute path") + } + f, err := os.Open(path) + if err != nil { + return nil, errors.New("credential file unavailable") + } + defer f.Close() + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() { + return nil, errors.New("credential file must be regular") + } + if info.Mode().Perm()&0o077 != 0 { + return nil, errors.New("credential file permissions must exclude group and others") + } + b, err := io.ReadAll(io.LimitReader(f, maxCredentialBytes+1)) + if err != nil { + return nil, errors.New("credential file read failed") + } + defer clear(b) + if len(b) > maxCredentialBytes { + return nil, errors.New("credential file exceeds limit") + } + trimmed := bytes.TrimSpace(b) + if len(trimmed) == 0 { + return nil, errors.New("credential file is empty") + } + return append([]byte(nil), trimmed...), nil +} + +func classify(err error) string { + s := err.Error() + for _, allowed := range []string{ + "usage", "invalid arguments", "state paths must be absolute", "kubeconfig unavailable", "kubeconfig must be a private regular file", + "receipt path must be in kubeconfig state directory", "state directory must be private", "invalid receipt path", + "credential file must be an absolute path", "credential file unavailable", "credential file must be regular", + "credential file permissions must exclude group and others", "credential file read failed", "credential file exceeds limit", "credential file is empty", + "cleanup receipt creation failed; cleanup status ambiguous", "secret staged without UID; cleanup pending", "cleanup receipt identity mismatch", + } { + if s == allowed { + return allowed + } + } + return "operation failed (details redacted)" +} diff --git a/e2e/k8s_execution/fixture/credentialloader/main_test.go b/e2e/k8s_execution/fixture/credentialloader/main_test.go new file mode 100644 index 0000000000..3a49282812 --- /dev/null +++ b/e2e/k8s_execution/fixture/credentialloader/main_test.go @@ -0,0 +1,188 @@ +//go:build kind_execution_e2e + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" +) + +func TestLoadCredentialAndRedaction(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "credential") + sentinel := "synthetic-provider-secret-never-print" + if err := os.WriteFile(path, []byte(" "+sentinel+"\n"), 0o600); err != nil { + t.Fatal(err) + } + got, err := loadCredential(path) + if err != nil { + t.Fatal(err) + } + if string(got) != sentinel { + t.Fatal("credential was not trimmed") + } + clear(got) + if out := classify(errors.New("request rejected: " + sentinel)); strings.Contains(out, sentinel) || out != "operation failed (details redacted)" { + t.Fatalf("classification was not strictly redacted: %q", out) + } + if stderr := errorMessage(errors.New("provider body: " + sentinel)); strings.Contains(stderr, sentinel) || stderr != "credential loader: operation failed (details redacted)" { + t.Fatalf("stderr was not strictly redacted: %q", stderr) + } +} + +func TestLoadCredentialRejectsBroadPermissions(t *testing.T) { + path := filepath.Join(t.TempDir(), "credential") + if err := os.WriteFile(path, []byte("synthetic"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := loadCredential(path); err == nil { + t.Fatal("broad permissions accepted") + } +} + +func TestDeletePinsReceiptUIDAndLeavesReplacement(t *testing.T) { + state, receipt, kubeconfig := privateState(t) + _ = state + const name = "mecak8s-live-replacement-test" + originalUID := types.UID("original-synthetic-uid") + client := fake.NewSimpleClientset() + client.PrependReactor("create", "secrets", func(action ktesting.Action) (bool, runtime.Object, error) { + created := action.(ktesting.CreateAction).GetObject().(*corev1.Secret).DeepCopy() + created.UID = originalUID + return true, created, nil + }) + if err := stageSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", name, []byte("synthetic-sentinel")); err != nil { + t.Fatal(err) + } + + replacementUID := types.UID("replacement-synthetic-uid") + client.PrependReactor("delete", "secrets", func(action ktesting.Action) (bool, runtime.Object, error) { + deleteAction := action.(ktesting.DeleteAction) + preconditions := deleteAction.GetDeleteOptions().Preconditions + if preconditions == nil || preconditions.UID == nil || *preconditions.UID != originalUID { + t.Fatal("cleanup did not pin the staged UID") + } + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "secrets"}, name, errors.New("UID precondition does not match")) + }) + if err := deleteStagedSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", name); err == nil { + t.Fatal("replacement UID mismatch was accepted") + } + if _, err := os.Stat(receipt); err != nil { + t.Fatal("receipt was removed despite UID mismatch") + } + _ = replacementUID // The simulated replacement identity must never be targeted. + assertNoSecretReads(t, client.Actions()) +} + +func TestDeleteTreatsNotFoundAsCleaned(t *testing.T) { + _, receipt, kubeconfig := privateState(t) + const name = "mecak8s-live-not-found-test" + uid := types.UID("staged-synthetic-uid") + if err := writeReceipt(receipt, cleanupReceipt{Name: name, Namespace: namespace, UID: uid, Context: "kind-owned", Kubeconfig: kubeconfig}); err != nil { + t.Fatal(err) + } + client := fake.NewSimpleClientset() + client.PrependReactor("delete", "secrets", func(action ktesting.Action) (bool, runtime.Object, error) { + preconditions := action.(ktesting.DeleteAction).GetDeleteOptions().Preconditions + if preconditions == nil || preconditions.UID == nil || *preconditions.UID != uid { + t.Fatal("cleanup did not pin the staged UID") + } + return true, nil, apierrors.NewNotFound(schema.GroupResource{Resource: "secrets"}, name) + }) + if err := deleteStagedSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", name); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(receipt); !errors.Is(err, os.ErrNotExist) { + t.Fatal("successful cleanup retained its receipt") + } + assertNoSecretReads(t, client.Actions()) +} + +func TestReceiptIsPrivateAndCreateOnly(t *testing.T) { + path := filepath.Join(t.TempDir(), "receipt") + receipt := cleanupReceipt{Name: "mecak8s-live-receipt-test", Namespace: namespace, UID: "synthetic-uid", Context: "kind-owned", Kubeconfig: "/private/kubeconfig"} + if err := writeReceipt(path, receipt); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatal("receipt was not created with mode 0600") + } + if err := writeReceipt(path, receipt); err == nil { + t.Fatal("receipt overwrite was accepted") + } +} + +func TestStageFailureDoesNotDeleteOrWriteReceipt(t *testing.T) { + _, receipt, kubeconfig := privateState(t) + client := fake.NewSimpleClientset() + client.PrependReactor("create", "secrets", func(ktesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewAlreadyExists(schema.GroupResource{Resource: "secrets"}, "mecak8s-live-stage-failure") + }) + err := stageSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", "mecak8s-live-stage-failure", []byte("synthetic-sentinel")) + if err == nil { + t.Fatal("stage failure was accepted") + } + if _, err := os.Stat(receipt); !errors.Is(err, os.ErrNotExist) { + t.Fatal("stage failure wrote a cleanup receipt") + } + for _, action := range client.Actions() { + if action.GetVerb() == "delete" { + t.Fatal("stage failure attempted deletion") + } + } + assertNoSecretReads(t, client.Actions()) +} + +func TestMissingUIDDoesNotWriteReceiptOrDeleteByName(t *testing.T) { + _, receipt, kubeconfig := privateState(t) + client := fake.NewSimpleClientset(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "unrelated", Namespace: namespace}}) + name := "mecak8s-live-ambiguous-test" + if err := stageSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", name, []byte("synthetic-sentinel")); err == nil || err.Error() != "secret staged without UID; cleanup pending" { + t.Fatalf("missing UID result = %v", err) + } + if _, err := os.Stat(receipt); !errors.Is(err, os.ErrNotExist) { + t.Fatal("missing UID wrote an invalid cleanup receipt") + } + for _, action := range client.Actions() { + if action.GetVerb() == "delete" { + t.Fatal("missing UID triggered name-only deletion") + } + } + assertNoSecretReads(t, client.Actions()) +} + +func privateState(t *testing.T) (string, string, string) { + t.Helper() + state := t.TempDir() + if err := os.Chmod(state, 0o700); err != nil { + t.Fatal(err) + } + kubeconfig := filepath.Join(state, "kubeconfig") + if err := os.WriteFile(kubeconfig, []byte("synthetic"), 0o600); err != nil { + t.Fatal(err) + } + return state, filepath.Join(state, "live-secret-test.receipt.json"), kubeconfig +} + +func assertNoSecretReads(t *testing.T, actions []ktesting.Action) { + t.Helper() + for _, action := range actions { + if action.GetResource().Resource == "secrets" && (action.GetVerb() == "get" || action.GetVerb() == "list") { + t.Fatalf("unexpected Secret %s action", action.GetVerb()) + } + } +} diff --git a/e2e/k8s_execution/fixture/legacyfixture/crd.yaml b/e2e/k8s_execution/fixture/legacyfixture/crd.yaml new file mode 100644 index 0000000000..a8fe3be98e --- /dev/null +++ b/e2e/k8s_execution/fixture/legacyfixture/crd.yaml @@ -0,0 +1,34 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: executionenvironments.execution.mecatl.dev +spec: + group: execution.mecatl.dev + scope: Namespaced + names: + plural: executionenvironments + singular: executionenvironment + kind: ExecutionEnvironment + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + properties: + references: + type: array + maxItems: 64 + items: + type: string diff --git a/e2e/k8s_execution/fixture/legacyfixture/main.go b/e2e/k8s_execution/fixture/legacyfixture/main.go new file mode 100644 index 0000000000..45833ecf9c --- /dev/null +++ b/e2e/k8s_execution/fixture/legacyfixture/main.go @@ -0,0 +1,52 @@ +//go:build kind_execution_e2e + +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" +) + +func main() { + if len(os.Args) != 4 { + fmt.Fprintln(os.Stderr, "usage: legacyfixture NAMESPACE PROFILES OUTPUT") + os.Exit(2) + } + config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) + if err != nil { + fail(err) + } + d, err := dynamic.NewForConfig(config) + if err != nil { + fail(err) + } + kube, err := kubernetes.NewForConfig(config) + if err != nil { + fail(err) + } + seed, err := executioncontroller.SeedLegacyMigrationFixture(context.Background(), d, kube, os.Args[1], os.Args[2]) + if err != nil { + fail(err) + } + encoded, err := json.Marshal(seed) + if err != nil { + fail(err) + } + encoded = append(encoded, '\n') + if err := os.WriteFile(os.Args[3], encoded, 0o600); err != nil { + fail(err) + } +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/e2e/k8s_execution/fixture/netprobe/main.go b/e2e/k8s_execution/fixture/netprobe/main.go new file mode 100644 index 0000000000..fae09de4cb --- /dev/null +++ b/e2e/k8s_execution/fixture/netprobe/main.go @@ -0,0 +1,37 @@ +//go:build kind_execution_e2e + +package main + +import ( + "flag" + "fmt" + "net" + "net/http" + "os" + "time" +) + +func main() { + listen := flag.String("listen", "", "HTTP listen address") + target := flag.String("target", "", "TCP endpoint to probe") + timeout := flag.Duration("timeout", 3*time.Second, "dial timeout") + flag.Parse() + if *listen != "" { + srv := &http.Server{Addr: *listen, Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }), ReadHeaderTimeout: 2 * time.Second} + if err := srv.ListenAndServe(); err != nil { + fmt.Fprintln(os.Stderr, "listen failed") + os.Exit(1) + } + } + if *target == "" { + fmt.Fprintln(os.Stderr, "target is required") + os.Exit(2) + } + conn, err := net.DialTimeout("tcp", *target, *timeout) + if err != nil { + fmt.Fprintln(os.Stderr, "connect denied or unavailable") + os.Exit(42) + } + _ = conn.Close() + fmt.Println("connected") +} diff --git a/e2e/k8s_execution/fixture/oidcissuer/main.go b/e2e/k8s_execution/fixture/oidcissuer/main.go new file mode 100644 index 0000000000..ddd1c3ce8c --- /dev/null +++ b/e2e/k8s_execution/fixture/oidcissuer/main.go @@ -0,0 +1,91 @@ +//go:build kind_execution_e2e + +// Command oidcissuer is a synthetic OIDC issuer for the offline Kind qualification. +package main + +import ( + "crypto/rsa" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "flag" + "fmt" + "log" + "math/big" + "net/http" + "os" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +func main() { + var certFile, keyFile, signingFile, issuer, audience string + flag.StringVar(&certFile, "tls-cert", "", "TLS certificate") + flag.StringVar(&keyFile, "tls-key", "", "TLS key") + flag.StringVar(&signingFile, "signing-key", "", "RSA PKCS8 signing key") + flag.StringVar(&issuer, "issuer", "", "exact issuer URL") + flag.StringVar(&audience, "audience", "mecatl", "token audience") + flag.Parse() + if certFile == "" || keyFile == "" || signingFile == "" || issuer == "" { + log.Fatal("all identity flags are required") + } + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + must(err) + raw, err := os.ReadFile(signingFile) + must(err) + block, _ := pem.Decode(raw) + if block == nil { + log.Fatal("invalid signing key") + } + parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) + must(err) + key, ok := parsed.(*rsa.PrivateKey) + if !ok { + log.Fatal("signing key is not RSA") + } + kidRaw := sha256.Sum256(x509.MarshalPKCS1PublicKey(&key.PublicKey)) + kid := base64.RawURLEncoding.EncodeToString(kidRaw[:12]) + mux := http.NewServeMux() + mux.HandleFunc("GET /.well-known/openid-configuration", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, map[string]any{"issuer": issuer, "jwks_uri": issuer + "/keys", "id_token_signing_alg_values_supported": []string{"RS256"}}) + }) + mux.HandleFunc("GET /keys", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, map[string]any{"keys": []any{map[string]any{"kty": "RSA", "use": "sig", "alg": "RS256", "kid": kid, "n": base64.RawURLEncoding.EncodeToString(key.N.Bytes()), "e": base64.RawURLEncoding.EncodeToString(big.NewInt(int64(key.E)).Bytes())}}}) + }) + mux.HandleFunc("GET /token", func(w http.ResponseWriter, r *http.Request) { + sub := r.URL.Query().Get("sub") + if sub != "alice" && sub != "bob" { + http.Error(w, "unknown fixture subject", http.StatusBadRequest) + return + } + now := time.Now().UTC() + tok := jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{"iss": issuer, "aud": audience, "sub": sub, "iat": now.Unix(), "exp": now.Add(10 * time.Minute).Unix()}) + tok.Header["kid"] = kid + signed, err := tok.SignedString(key) + if err != nil { + http.Error(w, "signing failed", 500) + return + } + writeJSON(w, map[string]string{"id_token": signed}) + }) + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + server := &http.Server{Addr: ":8443", Handler: mux, ReadHeaderTimeout: 5 * time.Second, TLSConfig: &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{cert}}} + ln, err := tls.Listen("tcp", server.Addr, server.TLSConfig) + must(err) + must(server.Serve(ln)) +} +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(v); err != nil { + log.Print("response encoding failed") + } +} +func must(err error) { + if err != nil { + log.Fatal(fmt.Errorf("fixture startup: %w", err)) + } +} diff --git a/e2e/k8s_execution/fixture/pki/main.go b/e2e/k8s_execution/fixture/pki/main.go new file mode 100644 index 0000000000..3a2c2205e7 --- /dev/null +++ b/e2e/k8s_execution/fixture/pki/main.go @@ -0,0 +1,107 @@ +//go:build kind_execution_e2e + +// Command pki generates synthetic, short-lived qualification identities. +package main + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/json" + "encoding/pem" + "math/big" + "net/url" + "os" + "path/filepath" + "time" +) + +func main() { + if len(os.Args) != 2 { + panic("usage: pki OUTPUT_DIRECTORY") + } + dir := os.Args[1] + if err := os.MkdirAll(dir, 0o700); err != nil { + panic(err) + } + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + must(err) + now := time.Now().UTC() + ca := &x509.Certificate{SerialNumber: serial(), Subject: pkix.Name{CommonName: "mecatl execution qualification CA"}, NotBefore: now.Add(-time.Minute), NotAfter: now.Add(6 * time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) + must(err) + write(filepath.Join(dir, "ca.crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), 0o600) + must(issue(dir, "provider", ca, caKey, []string{"mecatl-execution", "mecatl-execution.execution-qualification.svc", "mecatl-execution.execution-qualification.svc.cluster.local"}, "", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth})) + must(issue(dir, "mecak8s", ca, caKey, nil, "spiffe://mecatl.test/client/mecak8s", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})) + must(issue(dir, "intruder", ca, caKey, nil, "spiffe://mecatl.test/client/intruder", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})) + for _, name := range []string{"operations", "wrong-scope"} { + must(issue(dir, name, ca, caKey, nil, "spiffe://mecatl.test/client/"+name, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})) + } + grantPub, grantKey, err := ed25519.GenerateKey(rand.Reader) + must(err) + grantDER, err := x509.MarshalPKCS8PrivateKey(grantKey) + must(err) + write(filepath.Join(dir, "grant-key.pem"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: grantDER}), 0o600) + fingerprint := sha256.Sum256(grantPub) + manifest, err := json.Marshal(map[string]any{ + "version": 1, "generation": 1, "issuer": "https://mecatl.execution.test", "audience": "mecatl-execution", "activeKeyID": "k1", "grantTTL": "1m", "clockSkew": "5s", + "keys": []any{map[string]any{"id": "k1", "version": 1, "file": "grant-k1.pem", "publicKeySHA256": hex.EncodeToString(fingerprint[:]), "activateAt": now.Add(-time.Minute).Format(time.RFC3339), "verifyUntil": now.Add(5 * time.Hour).Format(time.RFC3339), "state": "active"}}, + "tls": map[string]any{"certificateFile": "tls.crt", "privateKeyFile": "tls.key", "clientCAFile": "clients.pem"}, + "clients": []any{ + map[string]any{"uri": "spiffe://mecatl.test/client/mecak8s", "mayAttestOwner": true, "administrator": true}, + map[string]any{"uri": "spiffe://mecatl.test/client/intruder", "mayAttestOwner": true, "administrator": false}, + map[string]any{"uri": "spiffe://mecatl.test/client/operations", "mayAttestOwner": true, "administrator": true, "administratorFor": []string{"spiffe://mecatl.test/client/mecak8s"}}, + map[string]any{"uri": "spiffe://mecatl.test/client/wrong-scope", "mayAttestOwner": true, "administrator": true, "administratorFor": []string{"spiffe://mecatl.test/client/intruder"}}, + }, + }) + must(err) + write(filepath.Join(dir, "manifest.json"), append(manifest, '\n'), 0o600) + must(issue(dir, "oidc", ca, caKey, []string{"oidc-issuer", "oidc-issuer.execution-qualification.svc", "oidc-issuer.execution-qualification.svc.cluster.local"}, "", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth})) + jwtKey, err := rsa.GenerateKey(rand.Reader, 2048) + must(err) + jwtDER, err := x509.MarshalPKCS8PrivateKey(jwtKey) + must(err) + write(filepath.Join(dir, "oidc-jwt-key.pem"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: jwtDER}), 0o600) +} + +func issue(dir, name string, ca *x509.Certificate, caKey *rsa.PrivateKey, dns []string, uri string, usages []x509.ExtKeyUsage) error { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + cert := &x509.Certificate{SerialNumber: serial(), Subject: pkix.Name{CommonName: name}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(6 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, ExtKeyUsage: usages, DNSNames: dns} + if uri != "" { + u, err := url.Parse(uri) + if err != nil { + return err + } + cert.URIs = []*url.URL{u} + } + der, err := x509.CreateCertificate(rand.Reader, cert, ca, &key.PublicKey, caKey) + if err != nil { + return err + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return err + } + write(filepath.Join(dir, name+".crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600) + write(filepath.Join(dir, name+".key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600) + return nil +} + +func serial() *big.Int { + n, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + must(err) + return n +} +func write(path string, data []byte, mode os.FileMode) { must(os.WriteFile(path, data, mode)) } +func must(err error) { + if err != nil { + panic(err) + } +} diff --git a/e2e/k8s_execution/fixture/rotation/main.go b/e2e/k8s_execution/fixture/rotation/main.go new file mode 100644 index 0000000000..3eb0c748fe --- /dev/null +++ b/e2e/k8s_execution/fixture/rotation/main.go @@ -0,0 +1,140 @@ +//go:build kind_execution_e2e + +// Command rotation creates synthetic execution-provider rotation candidates. +// It consumes only fixture material generated for the current qualification run. +package main + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/json" + "encoding/pem" + "math/big" + "net/url" + "os" + "path/filepath" + "time" +) + +func main() { + if len(os.Args) != 3 { + panic("usage: rotation INITIAL_PKI OUTPUT_DIRECTORY") + } + generate(os.Args[1], os.Args[2]) +} + +func generate(initial, out string) { + must(os.MkdirAll(out, 0o700)) + now := time.Now().UTC() + oldCA := read(filepath.Join(initial, "ca.crt")) + oldGrant := readGrant(filepath.Join(initial, "grant-key.pem")) + + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + must(err) + ca := &x509.Certificate{SerialNumber: serial(), Subject: pkix.Name{CommonName: "mecatl execution qualification rotation CA"}, NotBefore: now.Add(-time.Minute), NotAfter: now.Add(6 * time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) + must(err) + newCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}) + write(filepath.Join(out, "new-ca.crt"), newCA) + write(filepath.Join(out, "client-roots.pem"), append(append([]byte{}, oldCA...), newCA...)) + write(filepath.Join(out, "bridge-clients.pem"), append(append([]byte{}, oldCA...), newCA...)) + write(filepath.Join(out, "final-clients.pem"), newCA) + issue(out, "provider-new", ca, caKey, []string{"mecatl-execution", "mecatl-execution.execution-qualification.svc", "mecatl-execution.execution-qualification.svc.cluster.local"}, "", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}) + issue(out, "mecak8s-new", ca, caKey, nil, "spiffe://mecatl.test/client/mecak8s", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}) + + pub2, key2, err := ed25519.GenerateKey(rand.Reader) + must(err) + der2, err := x509.MarshalPKCS8PrivateKey(key2) + must(err) + write(filepath.Join(out, "grant-k2.pem"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der2})) + + writeManifest(filepath.Join(out, "invalid-same-generation.json"), manifest(1, "changed-policy.invalid", "k1", []keyEntry{key("k1", 1, "grant-k1.pem", oldGrant.Public().(ed25519.PublicKey), "active", now.Add(5*time.Hour))}, "tls.crt", "tls.key", "clients.pem")) + keys := []keyEntry{ + key("k1", 1, "grant-k1.pem", oldGrant.Public().(ed25519.PublicKey), "active", now.Add(5*time.Hour)), + key("k2", 2, "grant-k2.pem", pub2, "active", now.Add(5*time.Hour)), + } + writeManifest(filepath.Join(out, "bridge.json"), manifest(2, "mecatl-execution", "k2", keys, "tls.crt", "tls.key", "bridge-clients.pem")) + keys[0].State = "revoked" + writeManifest(filepath.Join(out, "final.json"), manifest(3, "mecatl-execution", "k2", keys, "provider-new.crt", "provider-new.key", "final-clients.pem")) + writeManifest(filepath.Join(out, "restore-fixture-clients.json"), manifest(4, "mecatl-execution", "k2", keys, "provider-new.crt", "provider-new.key", "bridge-clients.pem")) +} + +type keyEntry struct { + ID, File, Fingerprint, State string + Version uint64 + ActivateAt, VerifyUntil time.Time +} + +func key(id string, version uint64, file string, pub ed25519.PublicKey, state string, until time.Time) keyEntry { + sum := sha256.Sum256(pub) + return keyEntry{id, file, hex.EncodeToString(sum[:]), state, version, time.Now().UTC().Add(-time.Minute), until} +} + +func manifest(generation uint64, audience, active string, keys []keyEntry, cert, privateKey, clients string) map[string]any { + entries := make([]any, 0, len(keys)) + for _, k := range keys { + entries = append(entries, map[string]any{"id": k.ID, "version": k.Version, "file": k.File, "publicKeySHA256": k.Fingerprint, "activateAt": k.ActivateAt.Format(time.RFC3339), "verifyUntil": k.VerifyUntil.Format(time.RFC3339), "state": k.State}) + } + return map[string]any{ + "version": 1, "generation": generation, "issuer": "https://mecatl.execution.test", "audience": audience, "activeKeyID": active, "grantTTL": "1m", "clockSkew": "5s", "keys": entries, + "tls": map[string]any{"certificateFile": cert, "privateKeyFile": privateKey, "clientCAFile": clients}, + "clients": []any{ + map[string]any{"uri": "spiffe://mecatl.test/client/mecak8s", "mayAttestOwner": true, "administrator": true}, + map[string]any{"uri": "spiffe://mecatl.test/client/intruder", "mayAttestOwner": true, "administrator": false}, + map[string]any{"uri": "spiffe://mecatl.test/client/operations", "mayAttestOwner": true, "administrator": true, "administratorFor": []string{"spiffe://mecatl.test/client/mecak8s"}}, + map[string]any{"uri": "spiffe://mecatl.test/client/wrong-scope", "mayAttestOwner": true, "administrator": true, "administratorFor": []string{"spiffe://mecatl.test/client/intruder"}}, + }, + } +} + +func writeManifest(path string, value any) { + data, err := json.Marshal(value) + must(err) + write(path, append(data, '\n')) +} +func readGrant(path string) ed25519.PrivateKey { + block, _ := pem.Decode(read(path)) + if block == nil { + panic("invalid synthetic grant key") + } + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + must(err) + out, ok := key.(ed25519.PrivateKey) + if !ok { + panic("unexpected synthetic grant key type") + } + return out +} +func issue(dir, name string, ca *x509.Certificate, caKey *rsa.PrivateKey, dns []string, uri string, usages []x509.ExtKeyUsage) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + must(err) + cert := &x509.Certificate{SerialNumber: serial(), Subject: pkix.Name{CommonName: name}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(6 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, ExtKeyUsage: usages, DNSNames: dns} + if uri != "" { + u, err := url.Parse(uri) + must(err) + cert.URIs = []*url.URL{u} + } + der, err := x509.CreateCertificate(rand.Reader, cert, ca, &key.PublicKey, caKey) + must(err) + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + must(err) + write(filepath.Join(dir, name+".crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + write(filepath.Join(dir, name+".key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})) +} +func serial() *big.Int { + n, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + must(err) + return n +} +func read(path string) []byte { data, err := os.ReadFile(path); must(err); return data } +func write(path string, data []byte) { must(os.WriteFile(path, data, 0o600)) } +func must(err error) { + if err != nil { + panic(err) + } +} diff --git a/e2e/k8s_execution/fixture/rotation/main_test.go b/e2e/k8s_execution/fixture/rotation/main_test.go new file mode 100644 index 0000000000..bbbd6a5f9b --- /dev/null +++ b/e2e/k8s_execution/fixture/rotation/main_test.go @@ -0,0 +1,127 @@ +//go:build kind_execution_e2e + +package main + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "math/big" + "net" + "os" + "path/filepath" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +func TestForwardRestoreTrustBundleSupportsFixtureClient(t *testing.T) { + now := time.Now().UTC() + initial := filepath.Join(t.TempDir(), "initial") + out := filepath.Join(t.TempDir(), "rotation") + if err := os.MkdirAll(initial, 0o700); err != nil { + t.Fatal(err) + } + + oldCAKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + oldCA := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "fixture CA"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign} + oldCADER, err := x509.CreateCertificate(rand.Reader, oldCA, oldCA, &oldCAKey.PublicKey, oldCAKey) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(initial, "ca.crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: oldCADER}), 0o600); err != nil { + t.Fatal(err) + } + _, grant, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + grantDER, err := x509.MarshalPKCS8PrivateKey(grant) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(initial, "grant-key.pem"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: grantDER}), 0o600); err != nil { + t.Fatal(err) + } + generate(initial, out) + for _, tc := range []struct { + file, cert, key, ca string + }{ + {"bridge.json", "tls.crt", "tls.key", "bridge-clients.pem"}, + {"final.json", "provider-new.crt", "provider-new.key", "final-clients.pem"}, + {"restore-fixture-clients.json", "provider-new.crt", "provider-new.key", "bridge-clients.pem"}, + } { + var manifest struct { + TLS struct { + CertificateFile, PrivateKeyFile, ClientCAFile string + } + } + if err := json.Unmarshal(read(filepath.Join(out, tc.file)), &manifest); err != nil { + t.Fatal(err) + } + if manifest.TLS.CertificateFile != tc.cert || manifest.TLS.PrivateKeyFile != tc.key || manifest.TLS.ClientCAFile != tc.ca { + t.Errorf("%s must reference immutable material names", tc.file) + } + } + + clientKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + clientTemplate := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "fixture client"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, oldCA, &clientKey.PublicKey, oldCAKey) + if err != nil { + t.Fatal(err) + } + clientCert := tls.Certificate{Certificate: [][]byte{clientDER}, PrivateKey: clientKey} + serverCert, err := tls.LoadX509KeyPair(filepath.Join(out, "provider-new.crt"), filepath.Join(out, "provider-new.key")) + if err != nil { + t.Fatal(err) + } + bundle, err := os.ReadFile(filepath.Join(out, "client-roots.pem")) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(bundle) { + t.Fatal("generated forward trust bundle contained no certificates") + } + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{serverCert}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: roots}))) + healthpb.RegisterHealthServer(server, health.NewServer()) + serveDone := make(chan error, 1) + go func() { serveDone <- server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + _ = listener.Close() + <-serveDone + }) + + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, ServerName: "mecatl-execution.execution-qualification.svc.cluster.local", RootCAs: roots, Certificates: []tls.Certificate{clientCert}}))) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + if _, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}); err != nil { + t.Fatal("forward-restored TLS gRPC handshake failed") + } +} diff --git a/e2e/k8s_execution/holder_loss_helpers_test.go b/e2e/k8s_execution/holder_loss_helpers_test.go new file mode 100644 index 0000000000..c1ff8aa4ed --- /dev/null +++ b/e2e/k8s_execution/holder_loss_helpers_test.go @@ -0,0 +1,61 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "errors" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" +) + +func terminateOwnedExecutor(ctx context.Context, kube kubernetes.Interface, environmentID, environmentUID, podUID string) error { + pods, err := kube.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "execution.mecatl.dev/environment=" + environmentID}) + if err != nil { + return err + } + if len(pods.Items) != 1 || podUID == "" || environmentUID == "" { + return errors.New("exact owned executor unavailable") + } + pod := pods.Items[0] + if string(pod.UID) != podUID || len(pod.OwnerReferences) != 1 { + return errors.New("executor identity changed") + } + owner := pod.OwnerReferences[0] + if owner.APIVersion != "execution.mecatl.dev/v1alpha1" || owner.Kind != "ExecutionEnvironment" || owner.Name != environmentID || string(owner.UID) != environmentUID || owner.Controller == nil || !*owner.Controller { + return errors.New("executor owner changed") + } + uid, grace := types.UID(podUID), int64(30) + return kube.CoreV1().Pods(namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &grace, Preconditions: &metav1.Preconditions{UID: &uid}}) +} + +func executorTerminalProof(pod *corev1.Pod, uid string) bool { + if pod == nil || string(pod.UID) != uid || uid == "" || (pod.Status.Phase != corev1.PodFailed && pod.Status.Phase != corev1.PodSucceeded) { + return false + } + names := make(map[string]bool) + for _, c := range pod.Spec.Containers { + names[c.Name] = false + } + for _, c := range pod.Spec.InitContainers { + names[c.Name] = false + } + for _, c := range pod.Spec.EphemeralContainers { + names[c.Name] = false + } + statuses := append(append(append([]corev1.ContainerStatus{}, pod.Status.ContainerStatuses...), pod.Status.InitContainerStatuses...), pod.Status.EphemeralContainerStatuses...) + if len(pod.Spec.Containers) == 0 || len(statuses) != len(names) { + return false + } + for _, s := range statuses { + done, found := names[s.Name] + if !found || done || s.State.Terminated == nil { + return false + } + names[s.Name] = true + } + return true +} diff --git a/e2e/k8s_execution/holder_loss_test.go b/e2e/k8s_execution/holder_loss_test.go new file mode 100644 index 0000000000..f4525d930f --- /dev/null +++ b/e2e/k8s_execution/holder_loss_test.go @@ -0,0 +1,68 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" +) + +func TestHolderLossTerminalStimulusPinsOwnedPod(t *testing.T) { + for _, changed := range []string{"", "uid", "owner", "absent"} { + t.Run(changed, func(t *testing.T) { + controlled := true + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "executor", Namespace: namespace, UID: "pod-uid", Labels: map[string]string{"execution.mecatl.dev/environment": "env"}, OwnerReferences: []metav1.OwnerReference{{APIVersion: "execution.mecatl.dev/v1alpha1", Kind: "ExecutionEnvironment", Name: "env", UID: "env-uid", Controller: &controlled}}}} + if changed == "uid" { + pod.UID = "foreign" + } + if changed == "owner" { + pod.OwnerReferences[0].UID = "foreign" + } + k := kubefake.NewClientset(pod) + if changed == "absent" { + k = kubefake.NewClientset() + } + deletes := 0 + k.PrependReactor("delete", "pods", func(a ktesting.Action) (bool, runtime.Object, error) { + deletes++ + opts := a.(ktesting.DeleteAction).GetDeleteOptions() + if opts.GracePeriodSeconds == nil || *opts.GracePeriodSeconds != 30 || opts.Preconditions == nil || opts.Preconditions.UID == nil || *opts.Preconditions.UID != "pod-uid" { + t.Fatal("deletion must pin UID with normal grace") + } + return true, nil, nil + }) + err := terminateOwnedExecutor(context.Background(), k, "env", "env-uid", "pod-uid") + if (err == nil) != (changed == "") || deletes != map[bool]int{true: 1, false: 0}[changed == ""] { + t.Fatalf("deletes=%d err=%v", deletes, err) + } + for _, a := range k.Actions() { + if a.GetVerb() != "list" && a.GetVerb() != "delete" { + t.Fatal("stimulus must not patch finalizers") + } + } + }) + } +} + +func TestHolderLossTerminalProofRequiresExactCompleteEvidence(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{UID: "pod-uid"}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "executor"}}, InitContainers: []corev1.Container{{Name: "init"}}}, Status: corev1.PodStatus{Phase: corev1.PodFailed, ContainerStatuses: []corev1.ContainerStatus{{Name: "executor", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{}}}}, InitContainerStatuses: []corev1.ContainerStatus{{Name: "init", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{}}}}}} + if !executorTerminalProof(pod, "pod-uid") { + t.Fatal("terminal proof rejected") + } + for _, mutate := range []func(*corev1.Pod){func(p *corev1.Pod) { p.UID = "foreign" }, func(p *corev1.Pod) { p.Status.Phase = corev1.PodRunning }, func(p *corev1.Pod) { p.Status.InitContainerStatuses = nil }, func(p *corev1.Pod) { p.Status.ContainerStatuses[0].Name = "other" }, func(p *corev1.Pod) { p.Status.ContainerStatuses[0].State.Terminated = nil }} { + candidate := pod.DeepCopy() + mutate(candidate) + if executorTerminalProof(candidate, "pod-uid") { + t.Fatal("incomplete or foreign evidence accepted") + } + } + if executorTerminalProof(nil, "pod-uid") { + t.Fatal("absence is not terminal proof") + } +} diff --git a/e2e/k8s_execution/live_qualification_test.go b/e2e/k8s_execution/live_qualification_test.go new file mode 100644 index 0000000000..b32f3c626b --- /dev/null +++ b/e2e/k8s_execution/live_qualification_test.go @@ -0,0 +1,400 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptrace" + "os" + "path/filepath" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + liveModel = "anthropic/claude-haiku-4.5" + liveCredentialKey = "OPENROUTER_API_KEY" +) + +func TestKindExecutionLiveQualification(t *testing.T) { + if os.Getenv("MECATL_EXECUTION_LIVE") != "1" { + t.Skip("explicit live qualification only") + } + state := os.Getenv("MECATL_EXECUTION_QUAL_STATE") + kubeconfig := filepath.Join(state, "kubeconfig") + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute) + defer cancel() + + tokenForward := portForward(t, ctx, kubeconfig, "service/oidc-issuer", 8443) + alice := fixtureToken(t, ctx, tokenForward.addr, filepath.Join(state, "pki"), "alice") + tokenForward.stop() + agentForward := portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + defer agentForward.stop() + + preflightCtx, preflightCancel := context.WithTimeout(ctx, 15*time.Second) + t.Log("live stage=authenticated_http_preflight reason=begin") + preflightStatus, _ := request(t, preflightCtx, http.MethodGet, "http://"+agentForward.addr+"/v1/info", alice, nil) + preflightCancel() + if preflightStatus != http.StatusOK { + t.Fatalf("live stage=authenticated_http_preflight reason=rejected status=%d", preflightStatus) + } + t.Log("live stage=authenticated_http_preflight reason=ok") + created := createLiveSession(t, ctx, agentForward.addr, alice) + secretName := os.Getenv("MECATL_EXECUTION_LIVE_SECRET") + if !strings.HasPrefix(secretName, "mecak8s-live-") { + t.Fatal("run-scoped provider Secret name is unavailable") + } + assertCredentialScope(t, ctx, kubeconfig, secretName) + if created.ResolvedModel == nil || created.ResolvedModel.ProviderId != "openrouter" || created.ResolvedModel.ModelId != liveModel { + t.Fatal("live session resolved an unexpected provider or model") + } + nonce := fmt.Sprintf("qual_%d", time.Now().UnixNano()) + promptText := "Create a small Go 1.27 module with arithmetic/sum.go defining Sum(a, b int) int and a table-driven arithmetic/sum_test.go covering positive, negative, and zero inputs. Include the comment // qualification: " + nonce + " in sum.go. Use the Write tool to create the files, then use Shell with the exact command `go test ./...`. Fix any failures and finish with a concise result only after that exact command passes. Do not add external dependencies." + events := livePrompt(t, ctx, agentForward.addr, created.SessionId, alice, promptText) + + calls := map[string]bool{} + modelShellPassed := modelRanExactShell(events, "go test ./...") + var usage *mecatlv1.Usage + stop := "" + for _, ev := range events { + if ev.ToolCall != nil { + calls[ev.ToolCall.Name] = true + } + if ev.ToolResult != nil { + if ev.ToolResult.IsError { + t.Fatal("live coding smoke had a tool error") + } + } + if ev.Result != nil { + stop = ev.Result.Stop + usage = ev.Result.Usage + } + } + if stop != string(session.StopEndTurn) { + t.Fatalf("real provider run did not finish successfully (stop=%s)", stop) + } + if !calls["Write"] { + t.Fatal("real provider did not call a filesystem write tool") + } + if !calls["Shell"] { + t.Fatal("real provider did not call Shell") + } + if !modelShellPassed { + t.Fatal("real provider Shell result did not report exit code 0") + } + if usage == nil || usage.InputTokens <= 0 || usage.OutputTokens <= 0 { + t.Fatal("real provider returned no positive token usage") + } + + t.Logf("live stage=model_coding reason=ok input_tokens=%d output_tokens=%d shell_exit_code=0", usage.InputTokens, usage.OutputTokens) + + t.Log("live stage=independent_typed_verification reason=begin") + providerForward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + defer providerForward.stop() + client, err := executionclient.New(providerForward.addr, loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local")) + if err != nil { + t.Fatal("typed execution client setup failed") + } + defer client.Close() + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "alice"} + lookup := environmentForBinding(t, ctx, kubeconfig, created.SessionId) + attached := waitReady(t, ctx, client, owner, created.SessionId, lookup) + if attached.Environment != lookup { + t.Fatal("typed execution reattachment returned a different environment") + } + rc, release := acquireRun(t, ctx, client, owner, created.SessionId, attached, fmt.Sprintf("live-verify-%d", time.Now().UnixNano())) + defer release() + artifactCount := 0 + file, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: "arithmetic/sum.go"}) + if err != nil || !strings.Contains(string(file.Data), "return a + b") || !strings.Contains(string(file.Data), nonce) { + t.Fatal("typed gRPC file verification failed") + } + artifactCount++ + testFile, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: "arithmetic/sum_test.go"}) + if err != nil || len(testFile.Data) == 0 { + t.Fatal("typed gRPC test-file verification failed") + } + artifactCount++ + command, err := client.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go test ./...", TimeoutMillis: 120000}) + if err != nil || command.State != executionenv.CommandSucceeded || command.Result.ExitCode != 0 { + t.Fatal("typed gRPC independent go test failed") + } + t.Logf("live stage=independent_typed_verification reason=ok artifact_count=%d command_exit_code=%d", artifactCount, command.Result.ExitCode) + + observedTools := make([]string, 0, 4) + for _, name := range []string{"Write", "Read", "Edit", "Shell"} { + if calls[name] { + observedTools = append(observedTools, name) + } + } + cluster := os.Getenv("MECATL_EXECUTION_LIVE_CLUSTER") + if cluster == "" { + t.Fatal("verified qualification cluster identity is unavailable") + } + summary := liveSummary{ + Cluster: cluster, + Provider: created.ResolvedModel.ProviderId, + Model: created.ResolvedModel.ModelId, + InputTokens: usage.InputTokens, + OutputTokens: usage.OutputTokens, + ToolNames: observedTools, + ArtifactCount: artifactCount, + ExitCodes: []int{command.Result.ExitCode}, + Timestamp: time.Now().UTC().Format(time.RFC3339), + } + blob, err := json.MarshalIndent(summary, "", " ") + if err != nil { + t.Fatal("marshal sanitized summary failed") + } + if err := os.WriteFile(filepath.Join(state, "live-summary.json"), append(blob, '\n'), 0o600); err != nil { + t.Fatal("write sanitized summary failed") + } +} + +func modelRanExactShell(events []*mecatlv1.Event, command string) bool { + matching := map[string]struct{}{} + for _, ev := range events { + if ev.ToolCall != nil && ev.ToolCall.Name == "Shell" { + var args map[string]json.RawMessage + if json.Unmarshal([]byte(ev.ToolCall.Args), &args) != nil { + continue + } + var requested string + if json.Unmarshal(args["command"], &requested) == nil && strings.TrimSpace(requested) == command { + matching[ev.ToolCall.Id] = struct{}{} + } + } + if ev.ToolResult != nil { + if _, ok := matching[ev.ToolResult.CallId]; ok && !ev.ToolResult.IsError && strings.Contains(ev.ToolResult.Content, "[exit code: 0]") { + return true + } + } + } + return false +} + +type liveSummary struct { + Cluster string `json:"cluster"` + Provider string `json:"provider"` + Model string `json:"model"` + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + ToolNames []string `json:"tool_names"` + ArtifactCount int `json:"artifact_count"` + ExitCodes []int `json:"exit_codes"` + Timestamp string `json:"timestamp"` +} + +func assertCredentialScope(t *testing.T, ctx context.Context, kubeconfig, secretName string) { + t.Helper() + var agent appsv1.Deployment + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "deployment/mecak8s", "-n", namespace, "-o", "json"), &agent); err != nil { + t.Fatal("decode mecak8s credential-scope metadata failed") + } + assertRealProviderArgs(t, agent.Spec.Template.Spec) + want := secretReference{Container: "agent", Source: "env", Name: secretName, Key: liveCredentialKey, EnvName: liveCredentialKey} + refs := referencesNamed(secretReferences(agent.Spec.Template.Spec), secretName) + if len(refs) != 1 || refs[0] != want { + t.Fatal("run-scoped provider credential is not confined to the mecak8s agent environment") + } + + var provider appsv1.Deployment + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "deployment/mecatl-execution", "-n", namespace, "-o", "json"), &provider); err != nil { + t.Fatal("decode provider credential-scope metadata failed") + } + var executors corev1.PodList + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "execution.mecatl.dev/environment", "-o", "json"), &executors); err != nil { + t.Fatal("decode executor credential-scope metadata failed") + } + if containsSecretReference(secretReferences(provider.Spec.Template.Spec), secretName) { + t.Fatal("provider credential escaped into the execution provider") + } + for _, pod := range executors.Items { + if containsSecretReference(secretReferences(pod.Spec), secretName) { + t.Fatal("provider credential escaped into an executor pod") + } + } +} + +func assertRealProviderArgs(t *testing.T, spec corev1.PodSpec) { + t.Helper() + if err := realProviderConfigError(spec); err != nil { + t.Fatal(err) + } +} + +func realProviderConfigError(spec corev1.PodSpec) error { + for _, volume := range spec.Volumes { + if volume.Name == "execution-mock" || (volume.ConfigMap != nil && volume.ConfigMap.Name == "execution-mock") { + return errors.New("live deployment retained the execution-mock volume") + } + if volume.Projected != nil { + for _, source := range volume.Projected.Sources { + if source.ConfigMap != nil && source.ConfigMap.Name == "execution-mock" { + return errors.New("live deployment retained a projected execution-mock source") + } + } + } + } + for _, container := range spec.Containers { + for _, mount := range container.VolumeMounts { + if mount.Name == "execution-mock" { + return errors.New("live deployment retained the execution-mock mount") + } + } + if container.Name != "agent" { + continue + } + seen := map[string]bool{} + for _, arg := range container.Args { + seen[arg] = true + if arg == "--mock" || strings.HasPrefix(arg, "--mock-script=") { + return errors.New("live deployment retained mock-provider arguments") + } + if strings.Contains(arg, "base-url") { + return errors.New("live deployment overrides the provider endpoint") + } + } + for _, required := range []string{"--default-provider=openrouter", "--model=" + liveModel, "--max-run-tokens=32000"} { + if !seen[required] { + return fmt.Errorf("live deployment is missing required argument %s", required) + } + } + return nil + } + return errors.New("live deployment has no agent container") +} + +type secretReference struct { + Container string + Source string + Name string + Key string + EnvName string +} + +func secretReferences(spec corev1.PodSpec) []secretReference { + var refs []secretReference + collectContainer := func(container corev1.Container, kind string) { + for _, env := range container.Env { + if env.ValueFrom != nil && env.ValueFrom.SecretKeyRef != nil { + refs = append(refs, secretReference{Container: kind + container.Name, Source: "env", Name: env.ValueFrom.SecretKeyRef.Name, Key: env.ValueFrom.SecretKeyRef.Key, EnvName: env.Name}) + } + } + for _, envFrom := range container.EnvFrom { + if envFrom.SecretRef != nil { + refs = append(refs, secretReference{Container: kind + container.Name, Source: "envFrom", Name: envFrom.SecretRef.Name}) + } + } + } + for _, container := range spec.Containers { + collectContainer(container, "") + } + for _, container := range spec.InitContainers { + collectContainer(container, "init:") + } + for _, volume := range spec.Volumes { + if volume.Secret != nil { + refs = append(refs, secretReference{Source: "volume", Name: volume.Secret.SecretName}) + } + if volume.Projected != nil { + for _, source := range volume.Projected.Sources { + if source.Secret != nil { + refs = append(refs, secretReference{Source: "projected", Name: source.Secret.Name}) + } + } + } + } + for _, pullSecret := range spec.ImagePullSecrets { + refs = append(refs, secretReference{Source: "imagePullSecret", Name: pullSecret.Name}) + } + return refs +} + +func referencesNamed(refs []secretReference, name string) []secretReference { + matched := make([]secretReference, 0, 1) + for _, ref := range refs { + if ref.Name == name { + matched = append(matched, ref) + } + } + return matched +} + +func containsSecretReference(refs []secretReference, name string) bool { + return len(referencesNamed(refs, name)) != 0 +} + +type liveCreateResponse struct { + SessionId string `json:"session_id"` + ResolvedModel *mecatlv1.ResolvedModel `json:"resolved_model"` +} + +func createLiveSession(t *testing.T, ctx context.Context, addr, token string) liveCreateResponse { + t.Helper() + t.Log("live stage=http_create reason=begin") + ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ + WroteHeaders: func() { t.Log("live stage=http_create reason=headers_sent") }, + GotFirstResponseByte: func() { t.Log("live stage=http_create reason=response_started") }, + }) + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions", token, []byte(`{"mode":"default","limits":{"max_turns":8,"max_tool_calls":20,"max_consecutive_failures":3}}`)) + if status != http.StatusCreated { + t.Fatalf("live session creation failed (HTTP %d)", status) + } + var out liveCreateResponse + if json.Unmarshal(body, &out) != nil || out.SessionId == "" { + t.Fatal("live session response invalid") + } + return out +} + +func livePrompt(t *testing.T, ctx context.Context, addr, id, token, text string) []*mecatlv1.Event { + t.Helper() + raw, _ := json.Marshal(map[string]string{"text": text}) + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+id+"/prompt", strings.NewReader(string(raw))) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "text/event-stream") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal("live prompt transport failed") + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 2<<20)) + t.Fatalf("live prompt rejected (HTTP %d; body redacted)", resp.StatusCode) + } + var events []*mecatlv1.Event + s := bufio.NewScanner(io.LimitReader(resp.Body, 4<<20)) + s.Buffer(make([]byte, 64<<10), 1<<20) + for s.Scan() { + line := s.Text() + if !strings.HasPrefix(line, "data: ") { + continue + } + var ev mecatlv1.Event + if json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &ev) != nil { + t.Fatal("invalid live SSE event") + } + events = append(events, &ev) + } + if s.Err() != nil { + t.Fatal("live SSE stream failed") + } + return events +} diff --git a/e2e/k8s_execution/live_qualification_unit_test.go b/e2e/k8s_execution/live_qualification_unit_test.go new file mode 100644 index 0000000000..944d80e31d --- /dev/null +++ b/e2e/k8s_execution/live_qualification_unit_test.go @@ -0,0 +1,118 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestReplacementProofRequestsUsePostClaimEpochForPositiveRequest(t *testing.T) { + ref := executionenv.EnvironmentRef{ID: "fixture", Revision: "revision"} + owner := executionenv.Owner{Issuer: "issuer", Subject: "subject"} + staleStatus := executionStatus{Epoch: 4, PodUID: "pod", PVCUID: "pvc"} + current := executionStatus{Epoch: 6, PodUID: "pod", PVCUID: "pvc"} + stale, wrong, exact := replacementProofRequests(ref, owner, staleStatus, current) + if stale.ExpectedEpoch != staleStatus.Epoch || stale.ExpectedPodUID != staleStatus.PodUID { + t.Fatal("stale negative request did not preserve the pre-claim authority") + } + if wrong.ExpectedEpoch != current.Epoch || wrong.ExpectedPodUID == current.PodUID { + t.Fatal("wrong-UID negative request did not use current authority") + } + if exact.ExpectedEpoch != current.Epoch || exact.ExpectedPodUID != current.PodUID || exact.ExpectedPVCUID != current.PVCUID { + t.Fatal("positive exact request did not use post-release authoritative state") + } +} + +func TestModelRanExactShellRequiresDecodedCommandAndMatchingCallID(t *testing.T) { + call := func(id, args string) *mecatlv1.Event { + return &mecatlv1.Event{ToolCall: &mecatlv1.ToolCall{Id: id, Name: "Shell", Args: args}} + } + result := func(id string) *mecatlv1.Event { + return &mecatlv1.Event{ToolResult: &mecatlv1.ToolResult{CallId: id, Content: "[exit code: 0]"}} + } + for name, events := range map[string][]*mecatlv1.Event{ + "echo": {call("shell", `{"command":"echo ok"}`), result("shell")}, + "invalid JSON": {call("shell", `{"command":`), result("shell")}, + "wrong result ID": {call("shell", `{"command":"go test ./..."}`), result("other")}, + "unrelated success": {call("echo", `{"command":"echo ok"}`), result("echo"), call("test", `{"command":"go test ./..."}`)}, + } { + t.Run(name, func(t *testing.T) { + if modelRanExactShell(events, "go test ./...") { + t.Fatal("unqualified Shell call counted") + } + }) + } + if !modelRanExactShell([]*mecatlv1.Event{call("test", `{"command":" go test ./... "}`), result("test")}, "go test ./...") { + t.Fatal("matching successful Shell call was not counted") + } +} + +func TestRealProviderArgsRejectExecutionMockSources(t *testing.T) { + tests := []corev1.PodSpec{ + {Volumes: []corev1.Volume{{Name: "execution-mock"}}}, + {Volumes: []corev1.Volume{{Name: "other", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "execution-mock"}}}}}}, + { + Volumes: []corev1.Volume{{ + Name: "other", + VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{Sources: []corev1.VolumeProjection{{ + ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "execution-mock"}}, + }}}}, + }}, + }, + {Containers: []corev1.Container{{Name: "sidecar", VolumeMounts: []corev1.VolumeMount{{Name: "execution-mock"}}}}}, + } + for i, spec := range tests { + if err := realProviderConfigError(spec); err == nil { + t.Fatalf("case %d did not reject execution-mock source", i) + } + } +} + +func TestSecretReferencesFindEveryCredentialEscape(t *testing.T) { + const sentinel = "synthetic-live-secret-sentinel" + tests := map[string]corev1.PodSpec{ + "sidecar env": { + Containers: []corev1.Container{{Name: "sidecar", Env: []corev1.EnvVar{{Name: "TOKEN", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}, Key: "token"}}}}}}, + }, + "sidecar envFrom": { + Containers: []corev1.Container{{Name: "sidecar", EnvFrom: []corev1.EnvFromSource{{SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}}}}}}, + }, + "init container env": { + InitContainers: []corev1.Container{{Name: "setup", Env: []corev1.EnvVar{{Name: "TOKEN", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}, Key: "token"}}}}}}, + }, + "secret volume": { + Volumes: []corev1.Volume{{Name: "credentials", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: sentinel}}}}, + }, + "projected secret volume": { + Volumes: []corev1.Volume{{Name: "credentials", VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{Sources: []corev1.VolumeProjection{{Secret: &corev1.SecretProjection{LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}}}}}}}}, + }, + } + for name, spec := range tests { + t.Run(name, func(t *testing.T) { + if !containsSecretReference(secretReferences(spec), sentinel) { + t.Fatal("credential escape was not detected") + } + }) + } +} + +func TestExpectedHarnessCredentialReferenceIsExact(t *testing.T) { + const sentinel = "synthetic-live-secret-sentinel" + spec := corev1.PodSpec{Containers: []corev1.Container{{ + Name: "agent", + Env: []corev1.EnvVar{{Name: liveCredentialKey, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}, + Key: liveCredentialKey, + }}}}, + }}} + want := secretReference{Container: "agent", Source: "env", Name: sentinel, Key: liveCredentialKey, EnvName: liveCredentialKey} + refs := secretReferences(spec) + if len(refs) != 1 || refs[0] != want { + t.Fatal("expected harness credential reference was not projected exactly") + } +} diff --git a/e2e/k8s_execution/production_qualification_test.go b/e2e/k8s_execution/production_qualification_test.go new file mode 100644 index 0000000000..86f0e51ad8 --- /dev/null +++ b/e2e/k8s_execution/production_qualification_test.go @@ -0,0 +1,1504 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func requireProduction(t *testing.T) (string, string, context.Context, context.CancelFunc) { + t.Helper() + if os.Getenv("MECATL_EXECUTION_QUAL_PROFILE") != "production" { + t.Skip("production qualification profile only") + } + state := os.Getenv("MECATL_EXECUTION_QUAL_STATE") + if state == "" { + t.Fatal("MECATL_EXECUTION_QUAL_STATE is required") + } + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + return state, filepath.Join(state, "kubeconfig"), ctx, cancel +} + +func productionClient(t *testing.T, ctx context.Context, state, kubeconfig string) (*executionclient.Client, *forward) { + t.Helper() + f := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + tlsConfig := loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") + client, err := executionclient.New(f.addr, tlsConfig) + if err != nil { + t.Fatal(err) + } + waitProviderRPCReady(t, ctx, client, f.addr, tlsConfig) + t.Cleanup(func() { client.Close() }) + return client, f +} + +func productionReplicaClients(t *testing.T, ctx context.Context, state, kubeconfig string) ([2]*executionclient.Client, [2]string) { + t.Helper() + var pods corev1.PodList + raw := runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "app.kubernetes.io/name=mecatl-execution", "-o", "json") + if err := json.Unmarshal(raw, &pods); err != nil || len(pods.Items) != 2 { + t.Fatalf("decode two provider replicas: pods=%d err=%v", len(pods.Items), err) + } + if pods.Items[0].UID == "" || pods.Items[1].UID == "" || pods.Items[0].UID == pods.Items[1].UID { + t.Fatal("provider replica Pod UIDs are missing or identical") + } + var clients [2]*executionclient.Client + var podUIDs [2]string + for i := range clients { + forward := portForward(t, ctx, kubeconfig, "pod/"+pods.Items[i].Name, 8443) + client, err := executionclient.New(forward.addr, loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local")) + if err != nil { + t.Fatal(err) + } + clients[i] = client + podUIDs[i] = string(pods.Items[i].UID) + t.Cleanup(func() { client.Close() }) + } + return clients, podUIDs +} + +func createProductionEnvironment(t *testing.T, ctx context.Context, c *executionclient.Client, suffix string) (executionenv.Owner, string, executionenv.AttachEnvironmentResponse) { + t.Helper() + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "production-" + suffix} + binding := fmt.Sprintf("production-%s-%d", suffix, time.Now().UnixNano()) + op := "ensure-" + binding + ensured, err := c.Ensure(ctx, binding, "go", owner, op) + if err != nil { + t.Fatalf("ensure production environment: %v", err) + } + if err := c.CommitReference(ctx, executionenv.ReferenceRequest{Environment: ensured.Environment, Owner: owner, BindingID: binding, OperationID: op}); err != nil { + t.Fatalf("commit production reference: %v", err) + } + return owner, binding, waitReady(t, ctx, c, owner, binding, ensured.Environment) +} + +func TestKindExecutionProductionNetworkPolicyEnforced(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + client, _ := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, client, "network") + rc, release := acquireRun(t, ctx, client, owner, binding, attached, fmt.Sprintf("network-%d", time.Now().UnixNano())) + defer release() + + assertProductionExecutorPod(t, ctx, kubeconfig, attached.Environment.ID) + probeSource := []byte("package main\nimport (\"net\";\"os\";\"time\")\nfunc main(){c,e:=net.DialTimeout(\"tcp\",os.Args[1],2*time.Second);if e!=nil{os.Exit(42)};c.Close()}\n") + if _, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "network_probe.go", Data: probeSource}); err != nil { + t.Fatalf("install network probe in owned workspace: %v", err) + } + built, err := client.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go build -o network-probe network_probe.go", TimeoutMillis: 30000}) + if err != nil || built.State != executionenv.CommandSucceeded || built.Result.ExitCode != 0 { + t.Fatalf("build network probe: state=%q exit=%d err=%v", built.State, built.Result.ExitCode, err) + } + fixtureIP := kubeValue(t, ctx, kubeconfig, "get", "pod/network-fixture", "-n", namespace, "-o", "jsonpath={.status.podIP}") + if out := runOwnedProbe(t, ctx, client, rc, fixtureIP+":8080"); out.State != executionenv.CommandSucceeded || out.Result.ExitCode != 0 { + t.Fatal("profile-specific package endpoint was not reachable") + } + providerIP := kubeValue(t, ctx, kubeconfig, "get", "service/mecatl-execution", "-n", namespace, "-o", "jsonpath={.spec.clusterIP}") + peerIP := kubeValue(t, ctx, kubeconfig, "get", "pod/network-intruder", "-n", namespace, "-o", "jsonpath={.status.podIP}") + for name, endpoint := range map[string]string{ + "provider": providerIP + ":8443", "api": "10.96.0.1:443", "peer": peerIP + ":8080", + "dns": "10.96.0.10:53", "metadata": "169.254.169.254:80", "internet": "1.1.1.1:443", + } { + out := runOwnedProbe(t, ctx, client, rc, endpoint) + if out.State != executionenv.CommandFailed || out.Result.State != executionenv.CommandFailed || out.Result.ExitCode != 42 { + t.Fatalf("default-denied executor probe %s state=%q result_state=%q exit=%d, want failed/failed/42", name, out.State, out.Result.State, out.Result.ExitCode) + } + } + + out, err := command(ctx, kubeconfig, "exec", "-n", namespace, "pod/network-intruder", "--", "/ko-app/netprobe", "-target", providerIP+":8443", "-timeout", "2s").CombinedOutput() + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 42 { + t.Fatalf("unrelated pod denial exit proof=%q err=%v, want 42", strings.TrimSpace(string(out)), err) + } + if got := resourceCount(t, ctx, kubeconfig, "pods -l app.kubernetes.io/name=mecatl-execution"); got != 2 { + t.Fatalf("ready provider replicas=%d, want 2", got) + } +} + +func runOwnedProbe(t *testing.T, ctx context.Context, c *executionclient.Client, rc executionenv.RequestContext, endpoint string) executionenv.CommandStartResponse { + t.Helper() + out, err := c.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "./network-probe " + endpoint, TimeoutMillis: 15000}) + if err != nil { + t.Fatalf("network probe RPC failed: %v", err) + } + return out +} + +func assertProductionExecutorPod(t *testing.T, ctx context.Context, kubeconfig, environmentID string) { + t.Helper() + var pods corev1.PodList + raw := runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "execution.mecatl.dev/environment="+environmentID, "-o", "json") + if err := json.Unmarshal(raw, &pods); err != nil || len(pods.Items) != 1 { + t.Fatal("decode exact executor pod") + } + pod := pods.Items[0] + if pod.Spec.RuntimeClassName == nil || *pod.Spec.RuntimeClassName != "qualification-runc" { + t.Fatal("executor does not use the qualified RuntimeClass") + } + if pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { + t.Fatal("executor received a service-account token") + } + if len(pod.Spec.Containers) != 1 { + t.Fatal("executor container shape drifted") + } + c := pod.Spec.Containers[0] + if c.SecurityContext == nil || c.SecurityContext.AllowPrivilegeEscalation == nil || *c.SecurityContext.AllowPrivilegeEscalation || c.SecurityContext.Capabilities == nil || len(c.SecurityContext.Capabilities.Drop) != 1 || c.SecurityContext.Capabilities.Drop[0] != "ALL" { + t.Fatal("executor capability confinement drifted") + } + for _, name := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory, corev1.ResourceEphemeralStorage} { + request, requestOK := c.Resources.Requests[name] + limit, limitOK := c.Resources.Limits[name] + if !requestOK || !limitOK || request.IsZero() || limit.IsZero() { + t.Fatalf("executor resource %s is unbounded", name) + } + } + for _, volume := range pod.Spec.Volumes { + if volume.Name == "tmp" && volume.EmptyDir != nil && volume.EmptyDir.SizeLimit != nil && !volume.EmptyDir.SizeLimit.IsZero() { + return + } + } + t.Fatal("executor /tmp is unbounded") +} + +func TestKindExecutionProductionSecurityRotation(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + oldClient, forward := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, oldClient, "rotation") + oldClaim, err := oldClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-old-grant", OperationID: "acquire-rotation-old-grant", TTL: time.Minute}) + if err != nil { + t.Fatal(err) + } + oldRun := executionenv.RequestContext{Environment: oldClaim.Environment, Owner: owner, BindingID: binding, RunID: oldClaim.RunID, ClaimID: oldClaim.ClaimID, Epoch: oldClaim.Epoch, GrantGeneration: oldClaim.GrantGeneration, Grant: oldClaim.Grant} + if _, err := oldClient.File(ctx, executionenv.FileRequest{Context: oldRun, Operation: executionenv.OpFileCreate, Path: "rotation-sentinel.txt", Data: []byte("old-authority\n")}); err != nil { + t.Fatal(err) + } + + rotationDir := filepath.Join(state, "rotation") + generator := exec.CommandContext(ctx, "go", "run", "-tags", "kind_execution_e2e", "./e2e/k8s_execution/fixture/rotation", filepath.Join(state, "pki"), rotationDir) + generator.Dir = repoRoot(t) + generator.Env = cleanEnv() + if out, err := generator.CombinedOutput(); err != nil { + t.Fatalf("generate synthetic rotation material: %v: %s", err, out) + } + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cleanupCancel() + restoreFixtureSecurity(t, cleanupCtx, kubeconfig, rotationDir) + }) + + // A changed policy at the already-authoritative generation is rejected by + // both replicas. The old snapshot cannot continue authorizing RPCs while the + // mounted candidate disagrees with the durable authority ledger. + applySecurityCandidate(t, ctx, kubeconfig, filepath.Join(state, "pki"), filepath.Join(rotationDir, "invalid-same-generation.json"), "initial") + waitProviderReadyReplicas(t, ctx, kubeconfig, 0) + if _, err := oldClient.File(ctx, executionenv.FileRequest{Context: oldRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err == nil { + t.Fatal("same-generation changed policy continued authorizing an existing gRPC connection") + } + applySecurityCandidate(t, ctx, kubeconfig, filepath.Join(state, "pki"), filepath.Join(state, "pki", "manifest.json"), "initial") + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + recovered, err := oldClient.RenewRun(ctx, executionenv.RunClaimRequest{Environment: oldRun.Environment, Owner: owner, BindingID: binding, RunID: oldRun.RunID, ClaimID: oldRun.ClaimID, Epoch: oldRun.Epoch, GrantGeneration: oldRun.GrantGeneration, OperationID: "renew-rotation-recovered-grant", TTL: time.Minute}) + if isRemoteCode(err, executionenv.CodeConflict) { + recovered, err = oldClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-recovered-grant", OperationID: "acquire-rotation-recovered-grant", TTL: time.Minute}) + } + if err != nil { + t.Fatalf("establish fresh claim after authority recovery: code=%s", remoteErrorCode(err)) + } + recoveredRun := executionenv.RequestContext{Environment: recovered.Environment, Owner: owner, BindingID: binding, RunID: recovered.RunID, ClaimID: recovered.ClaimID, Epoch: recovered.Epoch, GrantGeneration: recovered.GrantGeneration, Grant: recovered.Grant} + waitFileContent(t, ctx, oldClient, recoveredRun, "rotation-sentinel.txt", "old-authority\n", "valid authority recovery") + if err := oldClient.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: recovered.Environment, Owner: owner, BindingID: binding, RunID: recovered.RunID, ClaimID: recovered.ClaimID, Epoch: recovered.Epoch, GrantGeneration: recovered.GrantGeneration, OperationID: "rotation-release-recovered"}); err != nil { + t.Fatalf("release recovery-phase claim: code=%s", remoteErrorCode(err)) + } + + // Generation 2 bridges client trust before changing the server certificate. + // The old connection remains usable, while a new-CA client can establish its + // own connection and receives grants from k2. + applySecurityCandidate(t, ctx, kubeconfig, rotationDir, filepath.Join(rotationDir, "bridge.json"), "bridge") + waitSecurityGeneration(ctx, t, kubeconfig, 2) + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + newTLS, err := executionclient.LoadTLSConfig(executionclient.TLSFiles{CA: filepath.Join(rotationDir, "client-roots.pem"), Cert: filepath.Join(rotationDir, "mecak8s-new.crt"), Key: filepath.Join(rotationDir, "mecak8s-new.key")}) + if err != nil { + t.Fatal(err) + } + newTLS.ServerName = "mecatl-execution.execution-qualification.svc.cluster.local" + newClient, err := executionclient.New(forward.addr, newTLS) + if err != nil { + t.Fatal(err) + } + defer newClient.Close() + newAttached := waitReady(t, ctx, newClient, owner, binding, attached.Environment) + if _, err := newClient.RenewRun(ctx, executionenv.RunClaimRequest{Environment: recoveredRun.Environment, Owner: owner, BindingID: binding, RunID: recoveredRun.RunID, ClaimID: recoveredRun.ClaimID, Epoch: recoveredRun.Epoch, GrantGeneration: recoveredRun.GrantGeneration, OperationID: "rotation-renew-released-phase-claim", TTL: time.Minute}); !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("released recovery-phase claim renewal code=%s, want conflict", remoteErrorCode(err)) + } + renewed, err := newClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: newAttached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-k2-grant", OperationID: "rotation-acquire-k2", TTL: time.Minute}) + if err != nil { + t.Fatalf("acquire bridge-authority claim: code=%s", remoteErrorCode(err)) + } + newRun := executionenv.RequestContext{Environment: renewed.Environment, Owner: owner, BindingID: binding, RunID: renewed.RunID, ClaimID: renewed.ClaimID, Epoch: renewed.Epoch, GrantGeneration: renewed.GrantGeneration, Grant: renewed.Grant} + waitFileContent(t, ctx, newClient, newRun, "rotation-sentinel.txt", "old-authority\n", "bridge authority") + if err := newClient.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: renewed.Environment, Owner: owner, BindingID: binding, RunID: renewed.RunID, ClaimID: renewed.ClaimID, Epoch: renewed.Epoch, GrantGeneration: renewed.GrantGeneration, OperationID: "rotation-release-k2-bridge"}); err != nil { + t.Fatalf("release bridge-phase claim: code=%s", remoteErrorCode(err)) + } + + // Generation 3 switches server TLS, removes the old client CA, and revokes + // k1. Authorization is checked on every RPC, so the established old-client + // HTTP/2 connection is denied rather than passing until reconnect. + applySecurityCandidate(t, ctx, kubeconfig, rotationDir, filepath.Join(rotationDir, "final.json"), "final") + waitSecurityGeneration(ctx, t, kubeconfig, 3) + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + finalAttached := waitReady(t, ctx, newClient, owner, binding, attached.Environment) + finalRequest := executionenv.RunClaimRequest{Environment: finalAttached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-final-grant", OperationID: "rotation-acquire-final", TTL: time.Minute} + finalClaim, err := acquireCurrentAuthorityRun(ctx, finalRequest, newClient.AcquireRun, 500*time.Millisecond) + if err != nil { + var remote *executionenv.Error + t.Fatalf("acquire final-authority claim: errorCode=%s retryable=%t", remoteErrorCode(err), errors.As(err, &remote) && remote.Retryable) + } + finalRun := executionenv.RequestContext{Environment: finalClaim.Environment, Owner: owner, BindingID: binding, RunID: finalClaim.RunID, ClaimID: finalClaim.ClaimID, Epoch: finalClaim.Epoch, GrantGeneration: finalClaim.GrantGeneration, Grant: finalClaim.Grant} + waitFileContent(t, ctx, newClient, finalRun, "rotation-sentinel.txt", "old-authority\n", "final authority") + // Both fixture certificates attest the same URI. Only the CA differs; the + // current claim below has just succeeded and has not been released. + if _, err := oldClient.File(ctx, executionenv.FileRequest{Context: finalRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); !isRemoteCode(err, executionenv.CodeUnauthenticated) { + var remote *executionenv.Error + t.Fatalf("removed client CA rejection: errorCode=%s retryable=%t", remoteErrorCode(err), errors.As(err, &remote) && remote.Retryable) + } + assertRetiredFixtureKeyDenied(ctx, t, newClient, state, finalRun, "rotation-sentinel.txt", "old-authority\n") + freshClient, err := executionclient.New(forward.addr, newTLS) + if err != nil { + t.Fatal(err) + } + defer freshClient.Close() + if _, err := freshClient.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); err != nil { + t.Fatalf("fresh new-CA client failed after TLS switch: %v", err) + } + + if err := newClient.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: finalClaim.Environment, Owner: owner, BindingID: binding, RunID: finalClaim.RunID, ClaimID: finalClaim.ClaimID, Epoch: finalClaim.Epoch, GrantGeneration: finalClaim.GrantGeneration, OperationID: "rotation-release-final"}); err != nil { + t.Fatal(err) + } + generation, err := newClient.RevokeEnvironment(ctx, attached.Environment, owner, newAttached.GrantGeneration, "rotation-revoke-replay") + if err != nil { + t.Fatal(err) + } + replayed, err := freshClient.RevokeEnvironment(ctx, attached.Environment, owner, newAttached.GrantGeneration, "rotation-revoke-replay") + if err != nil || replayed != generation { + t.Fatalf("durable revoke receipt replay generation=%d want=%d err=%v", replayed, generation, err) + } + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=240s") + postRestartForward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + postRestartClient, err := executionclient.New(postRestartForward.addr, newTLS) + if err != nil { + t.Fatal(err) + } + defer postRestartClient.Close() + postRestartAttached := waitReady(t, ctx, postRestartClient, owner, binding, attached.Environment) + if postRestartAttached.GrantGeneration != generation || generation != newAttached.GrantGeneration+1 { + t.Fatalf("persisted revocation generation=%d want=%d", postRestartAttached.GrantGeneration, generation) + } + replayed, err = postRestartClient.RevokeEnvironment(ctx, attached.Environment, owner, newAttached.GrantGeneration, "rotation-revoke-replay") + if err != nil || replayed != generation { + t.Fatalf("post-restart revoke receipt generation=%d want=%d code=%s", replayed, generation, remoteErrorCode(err)) + } + currentRun, releaseCurrent := acquireRun(t, ctx, postRestartClient, owner, binding, postRestartAttached, "rotation-post-restart") + defer releaseCurrent() + if currentRun.GrantGeneration != generation { + t.Fatalf("post-restart claim generation=%d want=%d", currentRun.GrantGeneration, generation) + } + // Keep the live claim's identity and lifetime, signing with the trusted k2. + // Only the generation is stale: neither a released claim nor retired k1 can + // explain the denial. Successful reads bracket the probe on this connection. + stale := resignFixtureGrant(t, currentRun, filepath.Join(rotationDir, "grant-k2.pem"), "k2", newAttached.GrantGeneration) + waitFileContent(t, ctx, postRestartClient, currentRun, "rotation-sentinel.txt", "old-authority\n", "post-restart revocation positive control") + if _, err := postRestartClient.File(ctx, executionenv.FileRequest{Context: stale, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("revoked claim generation rejection: code=%s, want conflict", remoteErrorCode(err)) + } + waitFileContent(t, ctx, postRestartClient, currentRun, "rotation-sentinel.txt", "old-authority\n", "post-restart revocation positive control after probe") + // Restore fixture client compatibility through a higher generation; this is + // another forward rotation, never a high-water-mark rollback. + restoreFixtureSecurity(t, ctx, kubeconfig, rotationDir) + qualifyDistinctAdministrator(t, ctx, state, kubeconfig, "rotated") +} + +func acquireCurrentAuthorityRun(ctx context.Context, req executionenv.RunClaimRequest, acquire func(context.Context, executionenv.RunClaimRequest) (executionenv.RunClaim, error), retryDelay time.Duration) (executionenv.RunClaim, error) { + const maxAttempts = 8 + + var lastErr error + for attempt := 1; attempt <= maxAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return executionenv.RunClaim{}, err + } + claim, err := acquire(ctx, req) + if err == nil { + return claim, nil + } + if !isRetryableAuthorityConvergence(err) { + return executionenv.RunClaim{}, err + } + lastErr = err + if attempt == maxAttempts { + break + } + if retryDelay <= 0 { + continue + } + timer := time.NewTimer(retryDelay) + select { + case <-ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return executionenv.RunClaim{}, ctx.Err() + case <-timer.C: + } + } + return executionenv.RunClaim{}, fmt.Errorf("final-authority acquire did not converge after %d attempts: %w", maxAttempts, lastErr) +} + +func isRetryableAuthorityConvergence(err error) bool { + var remote *executionenv.Error + return errors.As(err, &remote) && remote.Code == executionenv.CodeNotReady && remote.Retryable +} + +func TestAcquireCurrentAuthorityRunRetriesOnlyRetryableNotReady(t *testing.T) { + req := executionenv.RunClaimRequest{Environment: executionenv.EnvironmentRef{ID: "environment", Revision: "revision"}, Owner: executionenv.Owner{Issuer: "issuer", Subject: "subject"}, BindingID: "binding", RunID: "run", OperationID: "same-operation", TTL: time.Minute} + success := executionenv.RunClaim{Environment: req.Environment, BindingID: req.BindingID, RunID: req.RunID, ClaimID: "claim"} + + for _, tc := range []struct { + name string + errs []error + wantCalls int + wantErr bool + }{ + {name: "retryable not ready", errs: []error{&executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true}}, wantCalls: 2}, + {name: "not ready without retryability", errs: []error{&executionenv.Error{Code: executionenv.CodeNotReady}}, wantCalls: 1, wantErr: true}, + {name: "retryable unauthenticated", errs: []error{&executionenv.Error{Code: executionenv.CodeUnauthenticated, Retryable: true}}, wantCalls: 1, wantErr: true}, + {name: "permission denied", errs: []error{&executionenv.Error{Code: executionenv.CodePermissionDenied}}, wantCalls: 1, wantErr: true}, + {name: "internal", errs: []error{&executionenv.Error{Code: executionenv.CodeInternal}}, wantCalls: 1, wantErr: true}, + {name: "malformed error", errs: []error{errors.New("unexpected failure")}, wantCalls: 1, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + claim, err := acquireCurrentAuthorityRun(t.Context(), req, func(_ context.Context, got executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + calls++ + if got != req { + t.Fatalf("retry request changed: got %+v want %+v", got, req) + } + if calls <= len(tc.errs) { + return executionenv.RunClaim{}, tc.errs[calls-1] + } + return success, nil + }, 0) + if calls != tc.wantCalls { + t.Fatalf("calls=%d want=%d", calls, tc.wantCalls) + } + if tc.wantErr { + if err == nil { + t.Fatal("expected terminal error") + } + return + } + if err != nil || claim != success { + t.Fatalf("claim=%+v err=%v, want %+v", claim, err, success) + } + }) + } +} + +func TestAcquireCurrentAuthorityRunStopsAtBoundAndHonorsContext(t *testing.T) { + req := executionenv.RunClaimRequest{OperationID: "same-operation"} + t.Run("bound", func(t *testing.T) { + calls := 0 + _, err := acquireCurrentAuthorityRun(t.Context(), req, func(context.Context, executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + calls++ + return executionenv.RunClaim{}, &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true} + }, 0) + if calls != 8 || !isRemoteCode(err, executionenv.CodeNotReady) { + t.Fatalf("calls=%d code=%s, want 8/not_ready", calls, remoteErrorCode(err)) + } + }) + t.Run("cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + calls := 0 + _, err := acquireCurrentAuthorityRun(ctx, req, func(context.Context, executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + calls++ + return executionenv.RunClaim{}, nil + }, 0) + if calls != 0 || !errors.Is(err, context.Canceled) { + t.Fatalf("calls=%d err=%v, want 0/context canceled", calls, err) + } + }) +} + +func waitFileContent(t *testing.T, ctx context.Context, client *executionclient.Client, rc executionenv.RequestContext, path, want, proof string) { + t.Helper() + if err := readCurrentAuthorityContent(ctx, func(ctx context.Context) (executionenv.FileResponse, error) { + return client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: path}) + }, want, 500*time.Millisecond); err != nil { + var remote *executionenv.Error + retryable := errors.As(err, &remote) && remote.Retryable + t.Fatalf("%s did not authorize preserved data: errorCode=%s retryable=%t", proof, remoteErrorCode(err), retryable) + } +} + +func readCurrentAuthorityContent(ctx context.Context, read func(context.Context) (executionenv.FileResponse, error), want string, delay time.Duration) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + const maxAttempts = 60 + for attempt := 0; ; attempt++ { + if err := ctx.Err(); err != nil { + return err + } + got, err := read(ctx) + if err == nil { + if string(got.Data) != want { + return errors.New("authority read returned unexpected content") + } + return nil + } + if !isRetryableAuthorityConvergence(err) || attempt == maxAttempts-1 { + return err + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} + +func TestReadCurrentAuthorityContentRetriesOnlyPreDispatchLag(t *testing.T) { + for _, tc := range []struct { + name string + err error + content string + wantCalls int + wantErr bool + }{ + {"success", nil, "sentinel", 1, false}, + {"authority lag", &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true}, "", 2, false}, + {"wrong content", nil, "wrong", 1, true}, + {"store not ready", &executionenv.Error{Code: executionenv.CodeNotReady}, "", 1, true}, + {"expired client", &executionenv.Error{Code: executionenv.CodeUnauthenticated, Retryable: true}, "", 1, true}, + {"revoked key", &executionenv.Error{Code: executionenv.CodePermissionDenied}, "", 1, true}, + {"unknown error", errors.New("not_ready"), "", 1, true}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + err := readCurrentAuthorityContent(t.Context(), func(context.Context) (executionenv.FileResponse, error) { + calls++ + if calls == 1 { + return executionenv.FileResponse{Data: []byte(tc.content)}, tc.err + } + return executionenv.FileResponse{Data: []byte("sentinel")}, nil + }, "sentinel", 0) + if calls != tc.wantCalls || (err != nil) != tc.wantErr { + t.Fatalf("calls=%d error=%t", calls, err != nil) + } + }) + } +} + +func TestReadCurrentAuthorityContentBoundAndCancellation(t *testing.T) { + t.Run("persistent lag", func(t *testing.T) { + calls := 0 + err := readCurrentAuthorityContent(t.Context(), func(context.Context) (executionenv.FileResponse, error) { + calls++ + return executionenv.FileResponse{}, &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true} + }, "sentinel", 0) + if calls != 60 || !isRetryableAuthorityConvergence(err) { + t.Fatalf("calls=%d errorCode=%s", calls, remoteErrorCode(err)) + } + }) + for _, cancelInCall := range []bool{false, true} { + ctx, cancel := context.WithCancel(t.Context()) + if !cancelInCall { + cancel() + } + calls := 0 + err := readCurrentAuthorityContent(ctx, func(ctx context.Context) (executionenv.FileResponse, error) { + calls++ + if _, ok := ctx.Deadline(); !ok { + t.Fatal("read has no bounded deadline") + } + cancel() + return executionenv.FileResponse{}, &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true} + }, "sentinel", time.Hour) + cancel() + wantCalls := 0 + if cancelInCall { + wantCalls = 1 + } + if !errors.Is(err, context.Canceled) || calls != wantCalls { + t.Fatalf("calls=%d cancelled=%t", calls, errors.Is(err, context.Canceled)) + } + } +} + +func applySecurityCandidate(t *testing.T, ctx context.Context, kubeconfig, materialDir, manifest, mode string) { + t.Helper() + pki := filepath.Join(os.Getenv("MECATL_EXECUTION_QUAL_STATE"), "pki") + // Retain every previously referenced name with its original bytes. Secret + // and manifest projections may arrive in either order at each replica. + secretArgs := []string{"create", "secret", "generic", "execution-security", "-n", namespace, + "--from-file=grant-k1.pem=" + filepath.Join(pki, "grant-key.pem"), + "--from-file=tls.crt=" + filepath.Join(pki, "provider.crt"), + "--from-file=tls.key=" + filepath.Join(pki, "provider.key"), + "--from-file=clients.pem=" + filepath.Join(pki, "ca.crt")} + switch mode { + case "initial": + case "bridge", "final", "restore": + for _, name := range []string{"grant-k2.pem", "provider-new.crt", "provider-new.key", "bridge-clients.pem", "final-clients.pem"} { + secretArgs = append(secretArgs, "--from-file="+name+"="+filepath.Join(materialDir, name)) + } + default: + t.Fatalf("unknown security candidate mode %q", mode) + } + secretArgs = append(secretArgs, "--dry-run=client", "-o", "yaml") + applyKubectlInput(t, ctx, kubeconfig, runKubectl(t, ctx, kubeconfig, secretArgs...)) + config := runKubectl(t, ctx, kubeconfig, "create", "configmap", "mecatl-execution-security-manifest", "-n", namespace, "--from-file=manifest.json="+manifest, "--dry-run=client", "-o", "yaml") + applyKubectlInput(t, ctx, kubeconfig, config) +} + +func restoreFixtureSecurity(t *testing.T, ctx context.Context, kubeconfig, rotationDir string) { + t.Helper() + applySecurityCandidate(t, ctx, kubeconfig, rotationDir, filepath.Join(rotationDir, "restore-fixture-clients.json"), "restore") + waitSecurityGeneration(ctx, t, kubeconfig, 4) + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + + pki := filepath.Join(os.Getenv("MECATL_EXECUTION_QUAL_STATE"), "pki") + roots, err := os.ReadFile(filepath.Join(rotationDir, "client-roots.pem")) + if err != nil { + t.Fatal("read synthetic provider trust bundle") + } + if err := os.WriteFile(filepath.Join(pki, "provider-roots.pem"), roots, 0o600); err != nil { + t.Fatal("publish synthetic provider trust bundle") + } + clientSecret := runKubectl(t, ctx, kubeconfig, "create", "secret", "generic", "execution-client-tls", "-n", namespace, + "--from-file=ca.crt="+filepath.Join(rotationDir, "client-roots.pem"), "--from-file=tls.crt="+filepath.Join(pki, "mecak8s.crt"), "--from-file=tls.key="+filepath.Join(pki, "mecak8s.key"), "--dry-run=client", "-o", "yaml") + applyKubectlInput(t, ctx, kubeconfig, clientSecret) + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecak8s", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecak8s", "-n", namespace, "--timeout=240s") +} + +func applyKubectlInput(t *testing.T, ctx context.Context, kubeconfig string, input []byte) { + t.Helper() + cmd := command(ctx, kubeconfig, "apply", "-f", "-") + cmd.Stdin = bytes.NewReader(input) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("kubectl apply synthetic fixture: %v: %s", err, out) + } +} + +// PodReady can still describe the prior generation. Observe the nonsecret +// ledger first; the pinned replica's RPC barrier then proves local convergence. +func waitSecurityGeneration(ctx context.Context, t *testing.T, kubeconfig string, want uint64) { + t.Helper() + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + for { + cm := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-security-authority") + var ledger struct{ Generation uint64 } + if err := json.Unmarshal([]byte(cm.Data["state.json"]), &ledger); err != nil || ledger.Generation > want { + t.Fatal("unexpected authority ledger generation") + } + if ledger.Generation == want { + return + } + timer := time.NewTimer(500 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + t.Fatalf("authority ledger did not reach generation %d", want) + case <-timer.C: + } + } +} + +func waitProviderReadyReplicas(t *testing.T, ctx context.Context, kubeconfig string, want int) { + t.Helper() + deadline := time.Now().Add(2 * time.Minute) + for time.Now().Before(deadline) { + var pods corev1.PodList + raw := runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "app.kubernetes.io/name=mecatl-execution", "-o", "json") + if json.Unmarshal(raw, &pods) == nil { + ready := 0 + for _, pod := range pods.Items { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + ready++ + } + } + } + if ready == want { + return + } + } + time.Sleep(time.Second) + } + t.Fatalf("provider Ready replicas did not converge to %d", want) +} + +func TestKindExecutionProductionReplicaLifecycle(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + replicas, replicaUIDs := productionReplicaClients(t, ctx, state, kubeconfig) + client := replicas[0] + owner, binding, attached := createProductionEnvironment(t, ctx, client, "lifecycle") + first := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + if first.SpecSchema != 2 || first.StatusSchema != 2 || first.PodUID == "" || first.PVCUID == "" { + t.Fatal("schema-v2 exact runtime identities were not persisted") + } + + type acquireResult struct { + claim executionenv.RunClaim + err error + } + startAcquire := make(chan struct{}) + acquired := make(chan acquireResult, 2) + for i := range replicas { + i := i + go func() { + <-startAcquire + claim, err := replicas[i].AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: fmt.Sprintf("run-%d", i), OperationID: fmt.Sprintf("acquire-%d", i), TTL: time.Minute}) + acquired <- acquireResult{claim: claim, err: err} + }() + } + close(startAcquire) + var claim1 executionenv.RunClaim + conflicts := 0 + for range replicas { + result := <-acquired + if result.err == nil { + claim1 = result.claim + } else if isRemoteCode(result.err, executionenv.CodeConflict) { + conflicts++ + } else { + t.Fatalf("cross-replica run acquisition returned unexpected error: %v", result.err) + } + } + if claim1.ClaimID == "" || conflicts != 1 { + t.Fatalf("cross-replica run acquisition pod_uids=%v winner=%t conflicts=%d, want one each", replicaUIDs, claim1.ClaimID != "", conflicts) + } + statusWithClaim := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + if statusWithClaim.ActiveGrantGeneration != claim1.GrantGeneration { + t.Fatal("CRD pruned activeRun.grantGeneration") + } + if _, err := client.RenewRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: claim1.RunID, ClaimID: "stale", Epoch: claim1.Epoch, GrantGeneration: claim1.GrantGeneration, OperationID: "renew-stale", TTL: time.Minute}); err == nil { + t.Fatal("stale renewal unexpectedly succeeded") + } + rc := executionenv.RequestContext{Environment: claim1.Environment, Owner: owner, BindingID: binding, RunID: claim1.RunID, ClaimID: claim1.ClaimID, Epoch: claim1.Epoch, GrantGeneration: claim1.GrantGeneration, Grant: claim1.Grant} + if _, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "lifecycle-proof.txt", Data: []byte("preserved\n")}); err != nil { + t.Fatal(err) + } + if err := client.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: claim1.Environment, Owner: owner, BindingID: binding, RunID: claim1.RunID, ClaimID: claim1.ClaimID, Epoch: claim1.Epoch, GrantGeneration: claim1.GrantGeneration, OperationID: "release-a"}); err != nil { + t.Fatal(err) + } + + current := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + stale, wrong, exact := replacementProofRequests(attached.Environment, owner, first, current) + if err := client.ReplaceExecutor(ctx, stale); err == nil { + t.Fatal("stale-epoch replacement unexpectedly succeeded") + } + if err := client.ReplaceExecutor(ctx, wrong); err == nil { + t.Fatal("wrong Pod UID replacement unexpectedly succeeded") + } + if got := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID).PodUID; got != first.PodUID { + t.Fatal("wrong-UID replacement changed executor") + } + if err := client.ReplaceExecutor(ctx, exact); err != nil { + t.Fatalf("replace exact executor: %v", err) + } + second := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Ready && s.PodUID != "" && s.PodUID != first.PodUID }) + if second.PVCUID != first.PVCUID || second.Epoch <= first.Epoch { + t.Fatal("replacement did not preserve exact PVC and advance epoch") + } + reattached := waitReady(t, ctx, client, owner, binding, attached.Environment) + rc2, release2 := acquireRun(t, ctx, client, owner, binding, reattached, "verify-replacement") + proof, err := client.File(ctx, executionenv.FileRequest{Context: rc2, Operation: executionenv.OpFileRead, Path: "lifecycle-proof.txt"}) + if err != nil || string(proof.Data) != "preserved\n" { + t.Fatal("replacement lost PVC data") + } + release2() + + revokeResults := make(chan struct { + generation uint64 + err error + }, 2) + startRevoke := make(chan struct{}) + for i := range replicas { + i := i + go func() { + <-startRevoke + generation, err := replicas[i].RevokeEnvironment(ctx, attached.Environment, owner, reattached.GrantGeneration, "revoke-lifecycle") + revokeResults <- struct { + generation uint64 + err error + }{generation: generation, err: err} + }() + } + close(startRevoke) + var newGeneration uint64 + for range replicas { + result := <-revokeResults + if result.err != nil || result.generation <= reattached.GrantGeneration { + t.Fatalf("cross-replica revoke failed: generation=%d err=%v", result.generation, result.err) + } + if newGeneration != 0 && result.generation != newGeneration { + t.Fatalf("cross-replica revoke replay diverged: got=%d want=%d", result.generation, newGeneration) + } + newGeneration = result.generation + } + if _, err := client.File(ctx, executionenv.FileRequest{Context: rc2, Operation: executionenv.OpFileRead, Path: "lifecycle-proof.txt"}); err == nil { + t.Fatal("revoked grant remained usable") + } + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=240s") + client, _ = productionClient(t, ctx, state, kubeconfig) + if refreshed := waitReady(t, ctx, client, owner, binding, attached.Environment); refreshed.GrantGeneration != newGeneration { + t.Fatal("revocation generation was not durable across provider restart") + } + + refReq := executionenv.ReferenceRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, OperationID: "delete-ref-lifecycle"} + if err := client.PrepareReferenceDelete(ctx, refReq); err != nil { + t.Fatal(err) + } + if err := client.ConfirmReferenceDelete(ctx, refReq); err != nil { + t.Fatal(err) + } + retireStatus := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + retire := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: retireStatus.Epoch, ExpectedPodUID: retireStatus.PodUID, ExpectedPVCUID: retireStatus.PVCUID, OperationID: "retire-lifecycle"} + if err := client.RetireEnvironment(ctx, retire); err != nil { + t.Fatalf("retire exact environment: %v", err) + } + retired := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Retired && s.ExecutorTerminated }) + if resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+attached.Environment.ID) != 1 || resourceCount(t, ctx, kubeconfig, "pods -l execution.mecatl.dev/environment="+attached.Environment.ID) != 0 { + t.Fatal("retirement did not stop executor while retaining PVC") + } + if err := client.DeleteRetiredEnvironment(ctx, attached.Environment, owner, retired.PVCUID, "delete-retired-lifecycle"); err != nil { + t.Fatalf("delete retired environment: %v", err) + } + waitResourceAbsent(t, ctx, kubeconfig, "executionenvironment", attached.Environment.ID) + if resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+attached.Environment.ID) != 0 { + t.Fatal("explicit retired deletion retained its owned synthetic PVC") + } +} + +type executionStatus struct { + SpecSchema, StatusSchema int + Epoch uint64 + PodUID, PVCUID string + ActiveOperationID, FenceState string + ActiveGrantGeneration uint64 + Ready, Retired, ExecutorTerminated bool + ReadyReason string +} + +func replacementProofRequests(ref executionenv.EnvironmentRef, owner executionenv.Owner, staleStatus, current executionStatus) (executionenv.RetireEnvironmentRequest, executionenv.RetireEnvironmentRequest, executionenv.RetireEnvironmentRequest) { + stale := executionenv.RetireEnvironmentRequest{Environment: ref, Owner: owner, ExpectedEpoch: staleStatus.Epoch, ExpectedPodUID: staleStatus.PodUID, ExpectedPVCUID: staleStatus.PVCUID, OperationID: "replace-stale-epoch"} + wrong := executionenv.RetireEnvironmentRequest{Environment: ref, Owner: owner, ExpectedEpoch: current.Epoch, ExpectedPodUID: "wrong-pod-uid", ExpectedPVCUID: current.PVCUID, OperationID: "replace-wrong"} + exact := wrong + exact.ExpectedPodUID = current.PodUID + exact.OperationID = "replace-exact" + return stale, wrong, exact +} + +func readExecutionStatus(t *testing.T, ctx context.Context, kubeconfig, name string) executionStatus { + t.Helper() + var object map[string]any + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "executionenvironment", name, "-n", namespace, "-o", "json"), &object); err != nil { + t.Fatal(err) + } + nested := func(path ...string) any { + var v any = object + for _, p := range path { + m, ok := v.(map[string]any) + if !ok { + return nil + } + v = m[p] + } + return v + } + number := func(path ...string) uint64 { + if v, ok := nested(path...).(float64); ok { + return uint64(v) + } + return 0 + } + text := func(path ...string) string { v, _ := nested(path...).(string); return v } + condition := func(kind string) bool { + items, _ := nested("status", "conditions").([]any) + for _, item := range items { + m, _ := item.(map[string]any) + if m["type"] == kind && m["status"] == "True" { + return true + } + } + return false + } + conditionReason := func(kind string) string { + items, _ := nested("status", "conditions").([]any) + for _, item := range items { + m, _ := item.(map[string]any) + if m["type"] == kind { + reason, _ := m["reason"].(string) + return reason + } + } + return "" + } + return executionStatus{SpecSchema: int(number("spec", "schemaVersion")), StatusSchema: int(number("status", "schemaVersion")), Epoch: number("status", "epoch"), PodUID: text("status", "pod", "uid"), PVCUID: text("status", "pvc", "uid"), ActiveOperationID: text("status", "activeOperation", "id"), FenceState: text("status", "fenceState"), ActiveGrantGeneration: number("status", "activeRun", "grantGeneration"), Ready: condition("Ready"), Retired: condition("Retired"), ExecutorTerminated: condition("ExecutorTerminated"), ReadyReason: conditionReason("Ready")} +} + +func waitExecutionStatus(t *testing.T, ctx context.Context, kubeconfig, name string, accept func(executionStatus) bool) executionStatus { + t.Helper() + deadline := time.Now().Add(4 * time.Minute) + var last executionStatus + for time.Now().Before(deadline) { + last = readExecutionStatus(t, ctx, kubeconfig, name) + if accept(last) { + return last + } + time.Sleep(time.Second) + } + t.Fatalf("execution status did not reach required lifecycle state: environment=%q ready=%t ready_reason=%q fence_state=%q active_operation=%t", name, last.Ready, last.ReadyReason, last.FenceState, last.ActiveOperationID != "") + return executionStatus{} +} + +func waitActiveOperationOrCommandError(t *testing.T, ctx context.Context, kubeconfig, name string, commandDone <-chan error) { + t.Helper() + deadline := time.NewTimer(4 * time.Minute) + defer deadline.Stop() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case err := <-commandDone: + t.Fatalf("command ended before active-operation publication: code=%s", remoteErrorCode(err)) + case <-ticker.C: + if readExecutionStatus(t, ctx, kubeconfig, name).ActiveOperationID != "" { + return + } + case <-deadline.C: + t.Fatal("active-operation publication timed out while command remained in flight") + case <-ctx.Done(): + t.Fatal("active-operation publication context ended") + } + } +} + +func waitResourceAbsent(t *testing.T, ctx context.Context, kubeconfig, resource, name string) { + t.Helper() + deadline := time.Now().Add(3 * time.Minute) + for time.Now().Before(deadline) { + cmd := command(ctx, kubeconfig, "get", resource, name, "-n", namespace, "-o", "name") + if err := cmd.Run(); err != nil { + return + } + time.Sleep(time.Second) + } + t.Fatal("owned synthetic resource was not deleted") +} + +func kubeValue(t *testing.T, ctx context.Context, kubeconfig string, args ...string) string { + t.Helper() + return strings.TrimSpace(string(runKubectl(t, ctx, kubeconfig, args...))) +} + +func TestKindExecutionProductionHolderLossFencesActiveOperation(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + replicas, replicaUIDs := productionReplicaClients(t, ctx, state, kubeconfig) + holderClient, survivingClient := replicas[0], replicas[1] + owner, binding, attached := createProductionEnvironment(t, ctx, holderClient, "holder-loss") + rc, _ := acquireRun(t, ctx, holderClient, owner, binding, attached, "holder-loss-run") + initial := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + + var providers corev1.PodList + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "app.kubernetes.io/name=mecatl-execution", "-o", "json"), &providers); err != nil || len(providers.Items) != 2 { + t.Fatal("holder-loss proof requires two provider replicas") + } + holderPod := "" + for i := range providers.Items { + if string(providers.Items[i].UID) == replicaUIDs[0] { + holderPod = providers.Items[i].Name + } + } + if holderPod == "" { + t.Fatalf("holder replica UID %s no longer identifies a provider Pod", replicaUIDs[0]) + } + + commandDone := make(chan error, 1) + go func() { + _, commandErr := holderClient.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: `i=0; trap '' HUP TERM; while [ $i -lt 90 ]; do i=$((i+1)); printf '%s\n' "$i" > holder-loss-nonce; sleep 1; done`, TimeoutMillis: 110000}) + commandDone <- commandErr + }() + waitActiveOperationOrCommandError(t, ctx, kubeconfig, attached.Environment.ID, commandDone) + runKubectl(t, ctx, kubeconfig, "delete", "pod/"+holderPod, "-n", namespace, "--wait=true", "--timeout=90s") + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=180s") + + if _, err := survivingClient.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "second-writer", Data: []byte("must-not-run")}); err == nil { + t.Fatal("surviving replica admitted an overlapping writer after holder loss") + } + if _, err := survivingClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "overlap-run", OperationID: "overlap-run-acquire", TTL: time.Minute}); err == nil { + t.Fatal("surviving replica admitted a new run while old operation ownership was unresolved") + } + fenced := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { + return s.FenceState == "FenceUnknown" && s.ActiveOperationID != "" && !s.Ready + }) + if fenced.ReadyReason != "FenceUnknown" { + t.Fatalf("holder loss reason=%q, want bounded FenceUnknown", fenced.ReadyReason) + } + select { + case <-commandDone: + case <-time.After(5 * time.Second): + // A disconnected exec stream is permitted to remain locally blocked; the + // durable operation identity above, not RPC completion, is the proof. + } + + // Exec disconnect cancellation kills the command process group and reaped + // descendants, so a child timer cannot reliably terminate PID 1. Only after + // proving unresolved ownership above, ask the kubelet to stop this exact Pod. + contextName := os.Getenv("MECATL_KUBE_CONTEXT") + if contextName == "" { + t.Fatal("explicit fixture Kubernetes context required") + } + config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, &clientcmd.ConfigOverrides{CurrentContext: contextName}).ClientConfig() + if err != nil { + t.Fatal("load explicit fixture Kubernetes configuration") + } + config.Timeout = 15 * time.Second + kube, err := kubernetes.NewForConfig(config) + if err != nil { + t.Fatal("construct fixture Kubernetes client") + } + envUID := kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") + if err := terminateOwnedExecutor(ctx, kube, attached.Environment.ID, envUID, initial.PodUID); err != nil { + t.Fatal("exact owned executor termination request failed") + } + waitExecutorTerminal(ctx, t, kubeconfig, attached.Environment.ID, initial.PodUID) + recovery := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: initial.Epoch, ExpectedPodUID: initial.PodUID, ExpectedPVCUID: initial.PVCUID, OperationID: "recover-holder-loss"} + if err := survivingClient.RecoverEnvironment(ctx, recovery); err != nil { + t.Fatalf("recover exact terminal executor: %v", err) + } + if err := survivingClient.ReplaceExecutor(ctx, executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: initial.Epoch, ExpectedPodUID: initial.PodUID, ExpectedPVCUID: initial.PVCUID, OperationID: "replace-holder-loss"}); err != nil { + t.Fatalf("replace recovered executor: %v", err) + } + replaced := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Ready && s.PodUID != "" && s.PodUID != initial.PodUID }) + if replaced.PVCUID != initial.PVCUID { + t.Fatal("holder-loss recovery replaced the retained workspace") + } + reattached := waitReady(t, ctx, survivingClient, owner, binding, attached.Environment) + verify, release := acquireRun(t, ctx, survivingClient, owner, binding, reattached, "holder-loss-verify") + if _, err := survivingClient.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "holder-loss-nonce"}); err != nil { + t.Fatalf("post-recovery sentinel unavailable: %v", err) + } + if _, err := survivingClient.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "second-writer"}); err == nil { + t.Fatal("denied overlapping writer nevertheless reached the executor") + } + release() +} + +func waitExecutorTerminal(ctx context.Context, t *testing.T, kubeconfig, environmentID, podUID string) { + t.Helper() + deadline := time.Now().Add(2 * time.Minute) + for time.Now().Before(deadline) { + var pods corev1.PodList + raw := runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "execution.mecatl.dev/environment="+environmentID, "-o", "json") + if json.Unmarshal(raw, &pods) == nil && len(pods.Items) == 1 && executorTerminalProof(&pods.Items[0], podUID) { + return + } + time.Sleep(time.Second) + } + t.Fatal("test-owned executor did not produce terminal kubelet evidence") +} + +func TestKindExecutionProductionQuotaSaturation(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + replicas, replicaUIDs := productionReplicaClients(t, ctx, state, kubeconfig) + client := replicas[0] + + // First prove the provider's profile-wide CAS authority under concurrent + // requests. Exactly one distinct allocation may consume quota-cas's sole slot. + type ensureResult struct { + owner executionenv.Owner + binding string + out executionenv.EnsureEnvironmentResponse + err error + } + start := make(chan struct{}) + results := make(chan ensureResult, 2) + for i := 0; i < 2; i++ { + i := i + go func() { + <-start + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: fmt.Sprintf("quota-cas-%d", i)} + binding := fmt.Sprintf("quota-cas-%d-%d", i, time.Now().UnixNano()) + out, err := replicas[i].Ensure(ctx, binding, "quota-cas", owner, "ensure-"+binding) + results <- ensureResult{owner: owner, binding: binding, out: out, err: err} + }() + } + close(start) + var accepted *ensureResult + denied := 0 + for range 2 { + result := <-results + if result.err == nil { + copy := result + accepted = © + } else if isRemoteCode(result.err, executionenv.CodeResourceExhausted) { + denied++ + } else { + t.Fatalf("quota CAS returned unexpected error: %v", result.err) + } + } + if accepted == nil || denied != 1 { + t.Fatalf("quota CAS pod_uids=%v accepted=%v resource_exhausted=%d, want one each", replicaUIDs, accepted != nil, denied) + } + if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: accepted.out.Environment, Owner: accepted.owner, BindingID: accepted.binding, OperationID: "ensure-" + accepted.binding}); err != nil { + t.Fatal(err) + } + waitReady(t, ctx, client, accepted.owner, accepted.binding, accepted.out.Environment) + + // Then exercise Kubernetes admission itself. Tighten only the fixture-owned + // namespace's PVC count to one additional claim and restore the known chart + // value before converging and explicitly deleting these synthetic allocations. + pvcBaseline := resourceCount(t, ctx, kubeconfig, "pvc") + runKubectl(t, ctx, kubeconfig, "patch", "resourcequota/mecatl-execution", "-n", namespace, "--type=merge", "-p", fmt.Sprintf(`{"spec":{"hard":{"persistentvolumeclaims":"%d"}}}`, pvcBaseline+1)) + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute) + defer cleanupCancel() + _ = command(cleanupCtx, kubeconfig, "patch", "resourcequota/mecatl-execution", "-n", namespace, "--type=merge", "-p", `{"spec":{"hard":{"persistentvolumeclaims":"100"}}}`).Run() + }) + + allocations := make([]ensureResult, 0, 2) + for i := 0; i < 2; i++ { + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: fmt.Sprintf("quota-kube-%d", i)} + binding := fmt.Sprintf("quota-kube-%d-%d", i, time.Now().UnixNano()) + out, err := client.Ensure(ctx, binding, "quota-kube", owner, "ensure-"+binding) + if err != nil { + t.Fatalf("create quota admission candidate: %v", err) + } + allocations = append(allocations, ensureResult{owner: owner, binding: binding, out: out}) + } + deadline := time.Now().Add(2 * time.Minute) + var blocked ensureResult + for time.Now().Before(deadline) { + for _, allocation := range allocations { + status := readExecutionStatus(t, ctx, kubeconfig, allocation.out.Environment.ID) + if !status.Ready && status.ReadyReason == "PVCUnavailable" && resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+allocation.out.Environment.ID) == 0 { + blocked = allocation + } + } + if blocked.binding != "" { + break + } + time.Sleep(time.Second) + } + if blocked.binding == "" { + t.Fatal("Kubernetes PVC quota did not leave one allocation NotReady with reason PVCUnavailable") + } + if resourceCount(t, ctx, kubeconfig, "pvc") != pvcBaseline+1 { + t.Fatal("PVC quota admitted more than one additional workspace") + } + runKubectl(t, ctx, kubeconfig, "patch", "resourcequota/mecatl-execution", "-n", namespace, "--type=merge", "-p", `{"spec":{"hard":{"persistentvolumeclaims":"100"}}}`) + waitExecutionStatus(t, ctx, kubeconfig, blocked.out.Environment.ID, func(s executionStatus) bool { return s.Ready }) + + all := append(allocations, *accepted) + for _, allocation := range all { + if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: allocation.out.Environment, Owner: allocation.owner, BindingID: allocation.binding, OperationID: "ensure-" + allocation.binding}); err != nil { + t.Fatal(err) + } + waitReady(t, ctx, client, allocation.owner, allocation.binding, allocation.out.Environment) + retireSyntheticEnvironment(t, ctx, client, kubeconfig, allocation.owner, allocation.binding, allocation.out.Environment) + } +} + +func retireSyntheticEnvironment(t *testing.T, ctx context.Context, client *executionclient.Client, kubeconfig string, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) { + t.Helper() + deleteRef := executionenv.ReferenceRequest{Environment: ref, Owner: owner, BindingID: binding, OperationID: "delete-ref-" + binding} + if err := client.PrepareReferenceDelete(ctx, deleteRef); err != nil { + t.Fatal(err) + } + if err := client.ConfirmReferenceDelete(ctx, deleteRef); err != nil { + t.Fatal(err) + } + status := readExecutionStatus(t, ctx, kubeconfig, ref.ID) + request := executionenv.RetireEnvironmentRequest{Environment: ref, Owner: owner, ExpectedEpoch: status.Epoch, ExpectedPodUID: status.PodUID, ExpectedPVCUID: status.PVCUID, OperationID: "retire-" + binding} + if err := client.RetireEnvironment(ctx, request); err != nil { + t.Fatal(err) + } + retired := waitExecutionStatus(t, ctx, kubeconfig, ref.ID, func(s executionStatus) bool { return s.Retired && s.ExecutorTerminated }) + if err := client.DeleteRetiredEnvironment(ctx, ref, owner, retired.PVCUID, "delete-retired-"+binding); err != nil { + t.Fatal(err) + } + waitResourceAbsent(t, ctx, kubeconfig, "executionenvironment", ref.ID) +} + +func TestKindExecutionProductionPendingDeleteOutageRecovery(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + client, _ := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, client, "pending-delete") + request := executionenv.ReferenceRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, OperationID: "pending-delete-outage"} + if err := client.PrepareReferenceDelete(ctx, request); err != nil { + t.Fatal(err) + } + if got := executionReferences(t, ctx, kubeconfig, attached.Environment.ID)[binding]; got != string(executionenv.ReferencePendingDelete) { + t.Fatalf("prepared reference state=%q", got) + } + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=240s") + client, _ = productionClient(t, ctx, state, kubeconfig) + intents, err := client.ListReferenceIntents(ctx, owner) + if err != nil { + t.Fatal(err) + } + found := false + for _, intent := range intents { + if intent.Environment == attached.Environment && intent.BindingID == binding && intent.State == executionenv.ReferencePendingDelete && intent.OperationID == request.OperationID { + found = true + } + } + if !found { + exact, exactErr := client.FindReferenceIntent(ctx, owner, attached.Environment, binding) + t.Fatalf("pending-delete intent was not durable across provider outage: listed=%d exact_code=%s exact_state=%q exact_operation_match=%t", len(intents), remoteErrorCode(exactErr), exact.State, exact.OperationID == request.OperationID) + } + intruderTLS := loadTLS(t, filepath.Join(state, "pki"), "intruder", "mecatl-execution.execution-qualification.svc.cluster.local") + forward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + intruder, err := executionclient.New(forward.addr, intruderTLS) + if err != nil { + t.Fatal(err) + } + defer intruder.Close() + private, err := intruder.ListReferenceIntents(ctx, owner) + if err != nil { + t.Fatal(err) + } + if len(private) != 0 { + t.Fatal("pending reference intent crossed the mTLS client privacy boundary") + } + if err := client.ConfirmReferenceDelete(ctx, request); err != nil { + t.Fatal(err) + } + waitReferenceSet(t, ctx, kubeconfig, attached.Environment.ID) + if resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+attached.Environment.ID) != 1 { + t.Fatal("pending-delete recovery removed retained storage") + } + status := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + retire := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: status.Epoch, ExpectedPodUID: status.PodUID, ExpectedPVCUID: status.PVCUID, OperationID: "retire-pending-delete"} + if err := client.RetireEnvironment(ctx, retire); err != nil { + t.Fatal(err) + } + retired := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Retired && s.ExecutorTerminated }) + if err := client.DeleteRetiredEnvironment(ctx, attached.Environment, owner, retired.PVCUID, "delete-pending-delete"); err != nil { + t.Fatal(err) + } +} + +type legacyMigrationFixture struct { + Environment executionenv.EnvironmentRef `json:"environment"` + Owner executionenv.Owner `json:"owner"` + Binding string `json:"binding"` + PodUID string `json:"pod_uid"` + PVCUID string `json:"pvc_uid"` + Malformed executionenv.EnvironmentRef `json:"malformed_environment"` + MalformedPodUID string `json:"malformed_pod_uid"` + MalformedPVCUID string `json:"malformed_pvc_uid"` + Insecure executionenv.EnvironmentRef `json:"insecure_environment"` + InsecurePodUID string `json:"insecure_pod_uid"` + InsecurePVCUID string `json:"insecure_pvc_uid"` +} + +func runKindExecutionProductionCompatiblePrototypeMigration(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + client, _ := productionClient(t, ctx, state, kubeconfig) + var fixture legacyMigrationFixture + raw, err := os.ReadFile(filepath.Join(state, "legacy-migration.json")) + if err != nil || json.Unmarshal(raw, &fixture) != nil { + t.Fatalf("load pre-upgrade legacy API fixture: %v", err) + } + if fixture.Environment.ID == "" || fixture.Environment.Revision == "" || fixture.PodUID == "" || fixture.PVCUID == "" || fixture.Binding == "" { + t.Fatal("pre-upgrade legacy API fixture identities are incomplete") + } + + var stored map[string]any + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "executionenvironment", fixture.Environment.ID, "-n", namespace, "-o", "json"), &stored); err != nil { + t.Fatal(err) + } + status, _ := stored["status"].(map[string]any) + references, _ := status["references"].([]any) + if len(references) != 1 || references[0] != fixture.Binding { + t.Fatalf("legacy string references were not persisted through the real API upgrade: %#v", references) + } + before := readExecutionStatus(t, ctx, kubeconfig, fixture.Environment.ID) + if before.SpecSchema != 1 || before.StatusSchema != 1 || before.PodUID != fixture.PodUID || before.PVCUID != fixture.PVCUID { + t.Fatal("pre-upgrade fixture did not preserve schema-1 exact runtime identity") + } + if _, err := client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: fixture.Environment, Owner: fixture.Owner, BindingID: fixture.Binding}, Purpose: executionenv.PurposeSession}); err == nil { + t.Fatal("prototype schema attached before explicit migration") + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, "foreign-pod-uid", before.PVCUID, "migration-wrong-uid"); err == nil { + t.Fatal("migration adopted a foreign Pod UID") + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, "", before.PVCUID, "migration-missing-uid"); err == nil { + t.Fatal("migration accepted a missing runtime UID") + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 2, before.PodUID, before.PVCUID, "migration-unknown-version"); err == nil { + t.Fatal("migration accepted an unrecognized source schema") + } + if err := client.MigrateEnvironment(ctx, fixture.Malformed, fixture.Owner, 1, fixture.MalformedPodUID, fixture.MalformedPVCUID, "migration-malformed-shape"); err == nil || !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("migration accepted a legacy reference shape outside the recognized prototype: %v", err) + } + if err := client.MigrateEnvironment(ctx, fixture.Insecure, fixture.Owner, 1, fixture.InsecurePodUID, fixture.InsecurePVCUID, "migration-insecure-runtime"); err == nil || !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("migration accepted an insecure legacy executor: %v", err) + } + if got := readExecutionStatus(t, ctx, kubeconfig, fixture.Insecure.ID); got.SpecSchema != 1 || got.StatusSchema != 1 { + t.Fatal("rejected insecure legacy executor was rewritten") + } + // Normalize the deliberately rejected test-owned object through the API so it + // cannot poison later client-scoped reference reconciliation in this retained cluster. + normalized := fmt.Sprintf(`[{"bindingID":"legacy-malformed-binding","state":"Published","operationID":"migration-fixture-normalize","createdAt":%q}]`, time.Now().UTC().Format(time.RFC3339Nano)) + runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+fixture.Malformed.ID, "-n", namespace, "--subresource=status", "--type=merge", "-p", `{"status":{"references":`+normalized+`}}`) + insecureNormalized := fmt.Sprintf(`[{"bindingID":"legacy-insecure-binding","state":"Published","operationID":"migration-insecure-fixture-normalize","createdAt":%q}]`, time.Now().UTC().Format(time.RFC3339Nano)) + runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+fixture.Insecure.ID, "-n", namespace, "--subresource=status", "--type=merge", "-p", `{"status":{"references":`+insecureNormalized+`}}`) + if err := client.MigrateEnvironment(ctx, fixture.Malformed, fixture.Owner, 1, fixture.MalformedPodUID, fixture.MalformedPVCUID, "migration-normalized-shape"); err != nil { + t.Fatalf("normalize rejected migration fixture: %v", err) + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, before.PodUID, before.PVCUID, "migration-compatible-v1"); err != nil { + t.Fatalf("migrate compatible prototype: %v", err) + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, before.PodUID, before.PVCUID, "migration-compatible-v1"); err != nil { + t.Fatalf("exact migration replay: %v", err) + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 0, before.PodUID, before.PVCUID, "migration-compatible-v1"); !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatal("migration receipt accepted changed source schema:", remoteErrorCode(err)) + } + if got := kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", fixture.Environment.ID, "-n", namespace, "-o", "jsonpath={.status.lastMigrationFromSchema}"); got != "1" { + t.Fatal("API server pruned the exact migration receipt") + } + runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+fixture.Environment.ID, "-n", namespace, "--subresource=status", "--type=merge", "--dry-run=server", "-p", `{"status":{"lastMigrationFromSchema":0}}`) + invalid := command(ctx, kubeconfig, "patch", "executionenvironment/"+fixture.Environment.ID, "-n", namespace, "--subresource=status", "--type=merge", "--dry-run=server", "-p", `{"status":{"lastMigrationFromSchema":2}}`) + if out, err := invalid.CombinedOutput(); err == nil || !bytes.Contains(out, []byte("lastMigrationFromSchema")) || !bytes.Contains(out, []byte("Unsupported value")) { + t.Fatal("API server did not reject an unsupported migration receipt schema") + } + after := waitExecutionStatus(t, ctx, kubeconfig, fixture.Environment.ID, func(s executionStatus) bool { return s.Ready && s.SpecSchema == 2 && s.StatusSchema == 2 }) + if after.PodUID != before.PodUID || after.PVCUID != before.PVCUID || after.Epoch != before.Epoch+1 { + t.Fatal("migration changed runtime identity or failed to advance only the fence epoch") + } + if got := executionReferences(t, ctx, kubeconfig, fixture.Environment.ID); got[fixture.Binding] != string(executionenv.ReferencePublished) { + t.Fatal("migration did not convert the persisted legacy string reference") + } + reattached := waitReady(t, ctx, client, fixture.Owner, fixture.Binding, fixture.Environment) + verify, verifyRelease := acquireRun(t, ctx, client, fixture.Owner, fixture.Binding, reattached, "migration-verify") + got, err := client.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "migration-sentinel"}) + if err != nil || string(got.Data) != "prototype-data\n" { + t.Fatal("migration lost prototype workspace data") + } + verifyRelease() +} + +func TestKindExecutionProductionClearForkLifecycle(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + provider, _ := productionClient(t, ctx, state, kubeconfig) + + tokenForward := portForward(t, ctx, kubeconfig, "service/oidc-issuer", 8443) + alice := fixtureToken(t, ctx, tokenForward.addr, filepath.Join(state, "pki"), "alice") + bob := fixtureToken(t, ctx, tokenForward.addr, filepath.Join(state, "pki"), "bob") + tokenForward.stop() + agent := portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + defer agent.stop() + + source := createSession(t, ctx, agent.addr, alice) + assertMockJourney(t, prompt(t, ctx, agent.addr, source, alice, "run the scripted remote qualification")) + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "alice"} + allocation := executionenv.EnsureEnvironmentResponse{Environment: environmentForBinding(t, ctx, kubeconfig, source)} + attached := waitReady(t, ctx, provider, owner, source, allocation.Environment) + before := readExecutionStatus(t, ctx, kubeconfig, allocation.Environment.ID) + + forkID := successorSession(t, ctx, agent.addr, source, alice, "fork") + clearID := successorSession(t, ctx, agent.addr, source, alice, "clear") + if forkID == source || clearID == source || forkID == clearID { + t.Fatal("Clear/Fork did not publish distinct session identities") + } + if got := getSession(t, ctx, agent.addr, forkID, bob); got != http.StatusNotFound { + t.Fatalf("unknown owner observed fork: status=%d", got) + } + if got := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); got < 1 { + t.Fatal("successor publication lost its source environment") + } + waitReferenceSet(t, ctx, kubeconfig, allocation.Environment.ID, source, forkID, clearID) + after := readExecutionStatus(t, ctx, kubeconfig, allocation.Environment.ID) + if after.PVCUID != before.PVCUID || after.PodUID != before.PodUID || after.Epoch != before.Epoch { + t.Fatal("Clear/Fork changed the exact remote execution identity") + } + for _, binding := range []string{forkID, clearID} { + if _, err := provider.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: allocation.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); err != nil { + t.Fatalf("successor %s did not attach to source environment: %v", binding, err) + } + } + + claim, err := provider.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: source, RunID: "source-blocker", OperationID: "source-blocker-acquire", TTL: time.Minute}) + if err != nil { + t.Fatal(err) + } + if _, err := provider.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: clearID, RunID: "clear-overlap", OperationID: "clear-overlap-acquire", TTL: time.Minute}); err == nil || !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("second session admitted an overlapping run: %v", err) + } + if err := provider.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: source, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, OperationID: "source-blocker-release"}); err != nil { + t.Fatal(err) + } + clearAttached := waitReady(t, ctx, provider, owner, clearID, allocation.Environment) + _, release := acquireRun(t, ctx, provider, owner, clearID, clearAttached, "clear-after-release") + release() + + for _, id := range []string{forkID, clearID, source} { + status, body := request(t, ctx, http.MethodPost, "http://"+agent.addr+"/v1/sessions/"+id+"/delete", alice, nil) + if status != http.StatusNoContent { + t.Fatalf("delete session %s status=%d body=%s", id, status, body) + } + } + waitReferenceSet(t, ctx, kubeconfig, allocation.Environment.ID) + final := readExecutionStatus(t, ctx, kubeconfig, allocation.Environment.ID) + if final.PVCUID != before.PVCUID || resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+allocation.Environment.ID) != 1 { + t.Fatal("session deletion removed or replaced the retained workspace") + } +} + +func environmentForBinding(t *testing.T, ctx context.Context, kubeconfig, binding string) executionenv.EnvironmentRef { + t.Helper() + var list struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Revision string `json:"revision"` + } `json:"spec"` + Status struct { + References []struct { + BindingID string `json:"bindingID"` + } `json:"references"` + } `json:"status"` + } `json:"items"` + } + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "executionenvironments", "-n", namespace, "-o", "json"), &list); err != nil { + t.Fatal(err) + } + for _, item := range list.Items { + for _, ref := range item.Status.References { + if ref.BindingID == binding && item.Metadata.Name != "" && item.Spec.Revision != "" { + return executionenv.EnvironmentRef{ID: item.Metadata.Name, Revision: item.Spec.Revision} + } + } + } + t.Fatalf("no exact execution environment owns binding %s", binding) + return executionenv.EnvironmentRef{} +} + +func successorSession(t *testing.T, ctx context.Context, addr, source, token, operation string) string { + t.Helper() + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+source+"/"+operation, token, []byte(`{}`)) + if status != http.StatusCreated { + t.Fatalf("%s session status=%d body=%s", operation, status, body) + } + var out struct { + SessionID string `json:"session_id"` + } + if json.Unmarshal(body, &out) != nil || out.SessionID == "" { + t.Fatalf("%s session returned no identity", operation) + } + return out.SessionID +} + +func waitReferenceSet(t *testing.T, ctx context.Context, kubeconfig, environmentID string, want ...string) { + t.Helper() + deadline := time.Now().Add(90 * time.Second) + for time.Now().Before(deadline) { + got := executionReferences(t, ctx, kubeconfig, environmentID) + if len(got) == len(want) { + matched := true + for _, binding := range want { + if got[binding] != string(executionenv.ReferencePublished) { + matched = false + } + } + if matched { + return + } + } + time.Sleep(500 * time.Millisecond) + } + t.Fatalf("environment references did not converge to published set %v", want) +} + +func executionReferences(t *testing.T, ctx context.Context, kubeconfig, environmentID string) map[string]string { + t.Helper() + var object struct { + Status struct { + References []struct { + BindingID string `json:"bindingID"` + State string `json:"state"` + } `json:"references"` + } `json:"status"` + } + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "executionenvironment", environmentID, "-n", namespace, "-o", "json"), &object); err != nil { + t.Fatal(err) + } + out := make(map[string]string, len(object.Status.References)) + for _, ref := range object.Status.References { + out[ref.BindingID] = ref.State + } + return out +} + +func TestKindExecutionProductionFailureArtifactBoundary(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + ctx, stop := context.WithTimeout(ctx, time.Minute) + defer stop() + output := filepath.Join(state, fmt.Sprintf("artifact-boundary-%d.jsonl", time.Now().UnixNano())) + cmd := exec.CommandContext(ctx, "sh", "../../deploy/mecatl-execution-kind/collect-failure.sh", kubeconfig, os.Getenv("MECATL_KUBE_CONTEXT"), output) + cmd.Env = cleanEnv() + if err := cmd.Run(); err != nil { + t.Fatal("bounded failure collector failed") + } + artifact, err := os.ReadFile(output) + if err != nil { + t.Fatal("sanitized failure artifact unavailable") + } + if len(artifact) > 1<<20 { + t.Fatal("sanitized failure artifact exceeds one MiB") + } + for _, forbidden := range []string{"kubeconfig", "BEGIN PRIVATE KEY", "Bearer ", "grant-key", "ownerSubject", "clientHash", "Secret/data"} { + if strings.Contains(string(artifact), forbidden) { + t.Fatalf("sanitized failure artifact contains forbidden class %q", forbidden) + } + } +} diff --git a/e2e/k8s_execution/qualification_test.go b/e2e/k8s_execution/qualification_test.go new file mode 100644 index 0000000000..6fc28b8f39 --- /dev/null +++ b/e2e/k8s_execution/qualification_test.go @@ -0,0 +1,610 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const namespace = "execution-qualification" + +func TestKindExecutionQualification(t *testing.T) { + state := os.Getenv("MECATL_EXECUTION_QUAL_STATE") + if state == "" || os.Getenv("MECATL_KUBE_CONTEXT") == "" { + t.Fatal("MECATL_EXECUTION_QUAL_STATE and MECATL_KUBE_CONTEXT are required; run task e2e:k8s:execution") + } + kubeconfig := filepath.Join(state, "kubeconfig") + pki := filepath.Join(state, "pki") + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + defer cancel() + applyMockScript(t, ctx, kubeconfig, "mock-script.json") + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecak8s", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecak8s", "-n", namespace, "--timeout=240s") + + baseline := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev") + providerForward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + providerTLS := loadTLS(t, pki, "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") + providerClient, err := executionclient.New(providerForward.addr, providerTLS) + if err != nil { + t.Fatal(err) + } + defer providerClient.Close() + if _, err = providerClient.ValidateProfile(ctx, "go"); err != nil { + t.Fatalf("validate profile: %v", err) + } + if got := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); got != baseline { + t.Fatalf("profile validation changed environment count from %d to %d", baseline, got) + } + + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "alice"} + binding := fmt.Sprintf("direct-binding-%d", time.Now().UnixNano()) + operationID := fmt.Sprintf("direct-ensure-%d", time.Now().UnixNano()) + first, err := providerClient.Ensure(ctx, binding, "go", owner, operationID) + if err != nil { + t.Fatalf("ensure: %v", err) + } + if err := providerClient.CommitReference(ctx, executionenv.ReferenceRequest{Environment: first.Environment, Owner: owner, BindingID: binding, OperationID: operationID}); err != nil { + t.Fatalf("commit reference: %v", err) + } + if first.Environment.ID == "" || first.Environment.Revision == "" { + t.Fatal("ensure returned an empty exact environment reference") + } + ready := waitReady(t, ctx, providerClient, owner, binding, first.Environment) + if os.Getenv("MECATL_EXECUTION_QUAL_PROFILE") == "production" { + assertProductionExecutorPod(t, ctx, kubeconfig, first.Environment.ID) + } + if ready.Environment != first.Environment { + t.Fatalf("ready reference drifted: got %+v want %+v", ready.Environment, first.Environment) + } + second, err := providerClient.Ensure(ctx, binding, "go", owner, operationID) + if err != nil { + t.Fatalf("repeat ensure: %v", err) + } + if second.Environment != first.Environment { + t.Fatalf("repeat ensure changed identity: got %+v want %+v", second.Environment, first.Environment) + } + if got := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); got != baseline+1 { + t.Fatalf("repeat ensure left %d environments, want baseline + 1 (%d)", got, baseline+1) + } + if got := resourceCount(t, ctx, kubeconfig, "pods -l execution.mecatl.dev/environment="+first.Environment.ID); got != 1 { + t.Fatalf("executor pods = %d, want 1", got) + } + if got := resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+first.Environment.ID); got != 1 { + t.Fatalf("workspace PVCs = %d, want 1", got) + } + + intruderTLS := loadTLS(t, pki, "intruder", "mecatl-execution.execution-qualification.svc.cluster.local") + intruder, err := executionclient.New(providerForward.addr, intruderTLS) + if err != nil { + t.Fatal(err) + } + defer intruder.Close() + _, err = intruder.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: first.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}) + if err == nil || (!isRemoteCode(err, executionenv.CodePermissionDenied) && !isRemoteCode(err, executionenv.CodeNotFound)) { + t.Fatalf("different client attach error = %v, want an existence-hiding denial", err) + } + rc := executionenv.RequestContext{Environment: first.Environment, Owner: owner, BindingID: binding, Epoch: ready.Epoch, Grant: ready.Grant} + for name, call := range map[string]func() error{ + "command status": func() error { + _, err := intruder.CommandStatus(ctx, executionenv.CommandQueryRequest{Context: rc, CommandID: "not-owned"}) + return err + }, + "command cancel": func() error { + _, err := intruder.CancelCommand(ctx, executionenv.CommandQueryRequest{Context: rc, CommandID: "not-owned"}) + return err + }, + "environment retire": func() error { + return intruder.RetireEnvironment(ctx, executionenv.RetireEnvironmentRequest{Environment: rc.Environment, Owner: rc.Owner, ExpectedEpoch: rc.Epoch, ExpectedPodUID: "wrong-pod", ExpectedPVCUID: "wrong-pvc", OperationID: "unauthorized-retire"}) + }, + } { + if err := call(); err == nil || (!isRemoteCode(err, executionenv.CodePermissionDenied) && !isRemoteCode(err, executionenv.CodeUnauthenticated)) { + t.Fatalf("different client %s error = %v, want denial", name, err) + } + } + providerForward.stop() + if _, err := providerClient.ValidateProfile(ctx, "go"); err == nil { + t.Fatal("provider endpoint loss unexpectedly fell back") + } + providerForward = portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + providerTLS = loadTLS(t, pki, "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") + providerClient, err = executionclient.New(providerForward.addr, providerTLS) + if err != nil { + t.Fatal(err) + } + waitProviderRPCReady(t, ctx, providerClient, providerForward.addr, providerTLS) + defer providerClient.Close() + + tokenForward := portForward(t, ctx, kubeconfig, "service/oidc-issuer", 8443) + alice := fixtureToken(t, ctx, tokenForward.addr, pki, "alice") + bob := fixtureToken(t, ctx, tokenForward.addr, pki, "bob") + tokenForward.stop() + agentForward := portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + beforeNoFS := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev") + status, noFSBody := request(t, ctx, http.MethodPost, "http://"+agentForward.addr+"/v1/sessions", alice, []byte(`{"profile":"no-fs","mode":"default"}`)) + if status != http.StatusCreated { + t.Fatalf("create no-fs session status=%d body=%s", status, noFSBody) + } + if afterNoFS := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); afterNoFS != beforeNoFS { + t.Fatalf("no-fs session allocated execution resources: before=%d after=%d", beforeNoFS, afterNoFS) + } + sessionID := createSession(t, ctx, agentForward.addr, alice) + body := prompt(t, ctx, agentForward.addr, sessionID, alice, "run the scripted remote qualification") + if err := os.WriteFile(filepath.Join(state, "mock-journey.sse"), body, 0o600); err != nil { + t.Fatal(err) + } + assertMockJourney(t, body) + lookup := environmentForBinding(t, ctx, kubeconfig, sessionID) + attached := waitReady(t, ctx, providerClient, owner, sessionID, lookup) + rc, releaseVerification := acquireRun(t, ctx, providerClient, owner, sessionID, attached, fmt.Sprintf("independent-verify-%d", time.Now().UnixNano())) + proof, err := providerClient.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: "proof.txt"}) + if err != nil || string(proof.Data) != "beta\n" { + t.Fatalf("independent file proof failed: err=%v", err) + } + verification, err := providerClient.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go test ./...", TimeoutMillis: 120000}) + if err != nil || verification.State != executionenv.CommandSucceeded || verification.Result.ExitCode != 0 { + t.Fatalf("independent go test proof failed: err=%v state=%s", err, verification.State) + } + releaseVerification() + qualifyRemoteFileTools(t, ctx, providerClient, owner, sessionID, lookup) + if status := getSession(t, ctx, agentForward.addr, sessionID, bob); status != http.StatusNotFound { + t.Fatalf("different OIDC owner read status = %d, want 404", status) + } + agentForward.stop() + + applyReattachScript(t, ctx, kubeconfig, state) + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=180s") + providerForward = portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + providerTLS = loadTLS(t, pki, "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") + providerClient, err = executionclient.New(providerForward.addr, providerTLS) + if err != nil { + t.Fatal(err) + } + waitProviderRPCReady(t, ctx, providerClient, providerForward.addr, providerTLS) + defer providerClient.Close() + if refreshed := waitReady(t, ctx, providerClient, owner, sessionID, lookup); refreshed.Environment != lookup { + t.Fatal("provider did not reattach the exact environment after restart") + } + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecak8s", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecak8s", "-n", namespace, "--timeout=240s") + agentForward = portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + defer agentForward.stop() + body = promptEventually(t, ctx, agentForward.addr, sessionID, alice, "verify exact reattachment after both deployments restarted") + for _, marker := range []string{"reattach-read", "reattach-shell", "beta", "REMOTE_EXECUTION_REATTACH_COMPLETE", "ok"} { + if !bytes.Contains(body, []byte(marker)) { + t.Fatalf("reattach SSE omitted %q", marker) + } + } + if bytes.Contains(body, []byte(`"is_error":true`)) { + t.Fatal("reattached remote operation contained an error result") + } + if got := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); got != baseline+2 { + t.Fatalf("post-restart environments = %d, want baseline + direct + session (%d)", got, baseline+2) + } +} + +func assertMockJourney(t *testing.T, body []byte) { + t.Helper() + expected := map[string]struct { + name string + key string + want string + }{ + "write-proof": {"Write", "path", "proof.txt"}, "read-before-edit": {"Read", "path", "proof.txt"}, + "edit-proof": {"Edit", "path", "proof.txt"}, "copy-proof": {"Copy", "destination", "copy.txt"}, + "move-proof": {"Move", "destination", "moved.txt"}, "glob-proof": {"Glob", "pattern", "*.txt"}, + "grep-proof": {"Grep", "pattern", "beta"}, "write-mod": {"Write", "path", "go.mod"}, + "write-test": {"Write", "path", "remote_test.go"}, "shell-go-test": {"Shell", "command", "go test ./..."}, + "remove-moved": {"Remove", "path", "moved.txt"}, "read-final": {"Read", "path", "proof.txt"}, + } + calls := map[string]string{} + results := map[string]*mecatlv1.ToolResult{} + stop := "" + for _, line := range bytes.Split(body, []byte("\n")) { + if !bytes.HasPrefix(line, []byte("data: ")) { + continue + } + var ev mecatlv1.Event + if err := json.Unmarshal(bytes.TrimPrefix(line, []byte("data: ")), &ev); err != nil { + t.Fatalf("decode harness SSE event: %v", err) + } + if ev.ToolCall != nil { + want, ok := expected[ev.ToolCall.Id] + if !ok { + t.Fatalf("unexpected tool call id %q", ev.ToolCall.Id) + } + if ev.ToolCall.Name != want.name { + t.Fatalf("tool call %s name=%s want=%s", ev.ToolCall.Id, ev.ToolCall.Name, want.name) + } + var args map[string]json.RawMessage + var got string + if json.Unmarshal([]byte(ev.ToolCall.Args), &args) != nil || json.Unmarshal(args[want.key], &got) != nil || got != want.want { + t.Fatalf("tool call %s did not carry expected structured argument", ev.ToolCall.Id) + } + calls[ev.ToolCall.Id] = ev.ToolCall.Name + } + if ev.ToolResult != nil { + results[ev.ToolResult.CallId] = ev.ToolResult + } + if ev.Result != nil { + stop = ev.Result.Stop + } + } + for id := range expected { + if calls[id] == "" { + t.Fatalf("missing structured tool call %s (%s)", id, mockSSEStatus(body)) + } + result := results[id] + if result == nil || result.IsError { + t.Fatalf("tool call %s has no correlated successful result", id) + } + } + if result := results["shell-go-test"]; !strings.Contains(result.Content, "[exit code: 0]") { + t.Fatal("model-issued exact go test did not return exit code 0") + } + if result := results["read-final"]; !strings.Contains(result.Content, "beta") { + t.Fatal("final Read result did not contain persisted proof") + } + if stop != "end_turn" { + t.Fatalf("mock harness stop=%q, want end_turn", stop) + } +} + +func mockSSEStatus(body []byte) string { + var names []string + errorClasses := map[string]int{} + stop := "missing" + for _, line := range bytes.Split(body, []byte("\n")) { + if !bytes.HasPrefix(line, []byte("data: ")) { + continue + } + var ev mecatlv1.Event + if json.Unmarshal(bytes.TrimPrefix(line, []byte("data: ")), &ev) != nil { + continue + } + if ev.ToolCall != nil { + names = append(names, ev.ToolCall.Name) + } + if ev.ToolResult != nil && ev.ToolResult.IsError { + errorClasses[classifyMockToolError(ev.ToolResult.Content)]++ + } + if ev.Result != nil { + stop = ev.Result.Stop + } + } + return fmt.Sprintf("tools=%s error_classes=%v stop=%s", strings.Join(names, ","), errorClasses, stop) +} + +func classifyMockToolError(content string) string { + content = strings.ToLower(content) + for _, class := range []string{"grant", "permission", "owner", "stale", "unavailable", "not found", "version", "placement", "environment", "connection", "timeout", "binding", "argument", "request", "profile", "reference", "invalid", "read before", "already exists"} { + if strings.Contains(content, class) { + return strings.ReplaceAll(class, " ", "_") + } + } + return "other_redacted" +} + +func acquireRun(t *testing.T, ctx context.Context, c *executionclient.Client, owner executionenv.Owner, binding string, attached executionenv.AttachEnvironmentResponse, runID string) (executionenv.RequestContext, func()) { + t.Helper() + operationID := "acquire-" + runID + claim, err := c.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: runID, OperationID: operationID, TTL: time.Minute}) + if err != nil { + t.Fatalf("acquire run: %v", err) + } + rc := executionenv.RequestContext{Environment: claim.Environment, Owner: owner, BindingID: binding, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant} + release := func() { + err := c.ReleaseRun(context.WithoutCancel(ctx), executionenv.RunClaimRequest{Environment: claim.Environment, Owner: owner, BindingID: binding, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, OperationID: "release-" + runID}) + if err != nil { + t.Errorf("release run: %v", err) + } + } + return rc, release +} + +func waitReady(t *testing.T, ctx context.Context, c *executionclient.Client, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) executionenv.AttachEnvironmentResponse { + t.Helper() + deadline := time.Now().Add(3 * time.Minute) + lastCode := "none" + lastReady := false + for time.Now().Before(deadline) { + out, err := c.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}) + lastCode, lastReady = remoteErrorCode(err), out.Ready + if err == nil && out.Ready { + return out + } + time.Sleep(500 * time.Millisecond) + } + t.Fatalf("execution environment %q did not become ready: last_code=%s last_ready=%t", ref.ID, lastCode, lastReady) + return executionenv.AttachEnvironmentResponse{} +} +func isRemoteCode(err error, code executionenv.ErrorCode) bool { + var remote *executionenv.Error + return errors.As(err, &remote) && remote.Code == code +} + +func remoteErrorCode(err error) string { + if err == nil { + return "none" + } + var remote *executionenv.Error + if errors.As(err, &remote) && remote.Code.Valid() { + return string(remote.Code) + } + return "transport" +} + +func waitProviderRPCReady(t *testing.T, ctx context.Context, c *executionclient.Client, addr string, cfg *tls.Config) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + var lastErr error + for time.Now().Before(deadline) { + attemptCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + _, lastErr = c.ValidateProfile(attemptCtx, "go") + cancel() + if lastErr == nil { + return + } + time.Sleep(250 * time.Millisecond) + } + t.Fatalf("provider RPC did not become ready: code=%s tls=%s", remoteErrorCode(lastErr), classifyTLSFailure(addr, cfg)) +} + +func classifyTLSFailure(addr string, cfg *tls.Config) string { + dialer := &net.Dialer{Timeout: 2 * time.Second} + conn, err := tls.DialWithDialer(dialer, "tcp", addr, cfg.Clone()) + if err == nil { + _ = conn.Close() + return "ok" + } + var unknown x509.UnknownAuthorityError + if errors.As(err, &unknown) { + return "unknown-ca" + } + var invalid x509.CertificateInvalidError + if errors.As(err, &invalid) { + if invalid.Reason == x509.Expired { + return "expired" + } + return "bad-certificate" + } + var hostname x509.HostnameError + if errors.As(err, &hostname) { + return "bad-certificate" + } + if errors.Is(err, syscall.ECONNREFUSED) { + return "dial-refused" + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return "dial-timeout" + } + return "handshake-failed" +} + +func loadTLS(t *testing.T, dir, name, serverName string) *tls.Config { + t.Helper() + ca := filepath.Join(dir, "provider-roots.pem") + if _, err := os.Stat(ca); errors.Is(err, os.ErrNotExist) { + ca = filepath.Join(dir, "ca.crt") + } else if err != nil { + t.Fatal("inspect synthetic provider trust bundle") + } + cfg, err := executionclient.LoadTLSConfig(executionclient.TLSFiles{CA: ca, Cert: filepath.Join(dir, name+".crt"), Key: filepath.Join(dir, name+".key")}) + if err != nil { + t.Fatal(err) + } + cfg.ServerName = serverName + return cfg +} + +type forward struct { + addr string + cmd *exec.Cmd +} + +func (f *forward) stop() { + if f == nil || f.cmd == nil || f.cmd.Process == nil { + return + } + _ = f.cmd.Process.Signal(os.Interrupt) + _, _ = f.cmd.Process.Wait() +} +func portForward(t *testing.T, ctx context.Context, kubeconfig, target string, remote int) *forward { + t.Helper() + local := freePort(t) + cmd := command(ctx, kubeconfig, "port-forward", "-n", namespace, target, fmt.Sprintf("%d:%d", local, remote)) + var stderr bytes.Buffer + cmd.Stdout = io.Discard + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + f := &forward{addr: fmt.Sprintf("127.0.0.1:%d", local), cmd: cmd} + t.Cleanup(f.stop) + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", f.addr, 200*time.Millisecond) + if err == nil { + _ = conn.Close() + return f + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("port-forward %s did not start: %s", target, stderr.String()) + return nil +} +func freePort(t *testing.T) int { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + return ln.Addr().(*net.TCPAddr).Port +} + +func cleanEnv() []string { + return []string{"HOME=" + os.Getenv("HOME"), "PATH=" + os.Getenv("PATH")} +} +func command(ctx context.Context, kubeconfig string, args ...string) *exec.Cmd { + contextName := os.Getenv("MECATL_KUBE_CONTEXT") + fullArgs := append([]string{"--kubeconfig", kubeconfig, "--context", contextName}, args...) + cmd := exec.CommandContext(ctx, "kubectl", fullArgs...) + cmd.Env = cleanEnv() + return cmd +} +func runKubectl(t *testing.T, ctx context.Context, kubeconfig string, args ...string) []byte { + t.Helper() + out, err := command(ctx, kubeconfig, args...).CombinedOutput() + if err != nil { + t.Fatalf("kubectl %s failed: %v: %s", args[0], err, out) + } + return out +} +func resourceCount(t *testing.T, ctx context.Context, kubeconfig, spec string) int { + t.Helper() + args := append([]string{"get"}, strings.Fields(spec)...) + args = append(args, "-n", namespace, "-o", "name") + out := runKubectl(t, ctx, kubeconfig, args...) + if len(bytes.TrimSpace(out)) == 0 { + return 0 + } + return len(bytes.Split(bytes.TrimSpace(out), []byte{'\n'})) +} + +func fixtureToken(t *testing.T, ctx context.Context, addr, pki, sub string) string { + t.Helper() + ca, err := os.ReadFile(filepath.Join(pki, "ca.crt")) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(ca) { + t.Fatal("fixture CA invalid") + } + tr := &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: pool, ServerName: "oidc-issuer.execution-qualification.svc.cluster.local"}} + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+addr+"/token?sub="+sub, nil) + resp, err := (&http.Client{Transport: tr}).Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out struct { + IDToken string `json:"id_token"` + } + if resp.StatusCode != 200 || json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&out) != nil || out.IDToken == "" { + t.Fatal("fixture token issuance failed") + } + return out.IDToken +} +func createSession(t *testing.T, ctx context.Context, addr, token string) string { + t.Helper() + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions", token, []byte(`{"mode":"default","limits":{"max_turns":16,"max_tool_calls":24,"max_consecutive_failures":3}}`)) + if status != http.StatusCreated { + t.Fatalf("create session status=%d body=%s", status, body) + } + var out struct { + SessionID string `json:"session_id"` + } + if json.Unmarshal(body, &out) != nil || out.SessionID == "" { + t.Fatal("create session returned no id") + } + return out.SessionID +} +func prompt(t *testing.T, ctx context.Context, addr, id, token, text string) []byte { + t.Helper() + raw, _ := json.Marshal(map[string]string{"text": text}) + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+id+"/prompt", token, raw) + if status/100 != 2 { + t.Fatalf("prompt status=%d body=%s", status, body) + } + return body +} +func promptEventually(t *testing.T, ctx context.Context, addr, id, token, text string) []byte { + t.Helper() + raw, _ := json.Marshal(map[string]string{"text": text}) + deadline := time.Now().Add(90 * time.Second) + for { + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+id+"/prompt", token, raw) + if status/100 == 2 { + return body + } + if status != http.StatusConflict || time.Now().After(deadline) { + t.Fatalf("reattach prompt status=%d body=%s", status, body) + } + time.Sleep(time.Second) + } +} +func getSession(t *testing.T, ctx context.Context, addr, id, token string) int { + t.Helper() + status, _ := request(t, ctx, http.MethodGet, "http://"+addr+"/v1/sessions/"+id, token, nil) + return status +} +func request(t *testing.T, ctx context.Context, method, url, token string, body []byte) (int, []byte) { + t.Helper() + req, _ := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("Accept", "text/event-stream") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + if err != nil { + t.Fatal(err) + } + return resp.StatusCode, raw +} +func applyReattachScript(t *testing.T, ctx context.Context, kubeconfig, state string) { + t.Helper() + root := filepath.Clean(filepath.Join(state, "..", "..", "..")) + _ = root + applyMockScript(t, ctx, kubeconfig, "mock-script-reattach.json") +} +func applyMockScript(t *testing.T, ctx context.Context, kubeconfig, filename string) { + t.Helper() + source := filepath.Join(repoRoot(t), "deploy", "mecatl-execution-kind", filename) + cmd := command(ctx, kubeconfig, "create", "configmap", "execution-mock", "-n", namespace, "--from-file=mock-script.json="+source, "--dry-run=client", "-o", "yaml") + rendered, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + apply := command(ctx, kubeconfig, "apply", "-f", "-") + apply.Stdin = bytes.NewReader(rendered) + if out, err := apply.CombinedOutput(); err != nil { + t.Fatalf("update mock script: %v: %s", err, out) + } +} +func repoRoot(t *testing.T) string { + t.Helper() + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + return filepath.Clean(filepath.Join(wd, "..", "..")) +} diff --git a/e2e/k8s_execution/zz_production_helm_lifetime_test.go b/e2e/k8s_execution/zz_production_helm_lifetime_test.go new file mode 100644 index 0000000000..425dd87cdc --- /dev/null +++ b/e2e/k8s_execution/zz_production_helm_lifetime_test.go @@ -0,0 +1,497 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +// Last in the serial production suite: rotation and migration share this +// throwaway release. Always restore its current authority, never the initial one. +func TestKindExecutionProductionHelmLifetime(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + requireOwnedHelmFixture(t, state, kubeconfig) + client, _ := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, client, "helm-lifetime") + rc, release := acquireRun(t, ctx, client, owner, binding, attached, "helm-lifetime-write") + probe := []byte("package main\nimport (\"net\";\"os\";\"time\")\nfunc main(){c,e:=net.DialTimeout(\"tcp\",os.Args[1],2*time.Second);if e!=nil{os.Exit(42)};c.Close()}\n") + for path, data := range map[string][]byte{"helm-sentinel": []byte("retained-helm-data\n"), "helm-probe.go": probe} { + if _, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: path, Data: data}); err != nil { + t.Fatal("create lifetime workspace data:", remoteErrorCode(err)) + } + } + built, err := client.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go build -o helm-probe helm-probe.go", TimeoutMillis: 30000}) + if err != nil || built.State != executionenv.CommandSucceeded || built.Result.ExitCode != 0 { + t.Fatal("build lifetime probe failed") + } + assertRetiredFixtureKeyDenied(ctx, t, client, state, rc, "helm-sentinel", "retained-helm-data\n") + release() + // Occupy the single-slot profile; its reservation must still constrain Ensure + // after adoption, rather than only matching a ConfigMap annotation. + quotaBinding := fmt.Sprintf("helm-quota-%d", time.Now().UnixNano()) + quota, err := client.Ensure(ctx, quotaBinding, "quota-cas", owner, "ensure-"+quotaBinding) + if err != nil { + t.Fatal("reserve lifetime capacity:", remoteErrorCode(err)) + } + if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: quota.Environment, Owner: owner, BindingID: quotaBinding, OperationID: "ensure-" + quotaBinding}); err != nil { + t.Fatal("commit lifetime capacity:", remoteErrorCode(err)) + } + waitReady(t, ctx, client, owner, quotaBinding, quota.Environment) + + before := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + envUID := kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") + pod := kubeValue(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[0].metadata.name}") + if before.PVCUID != kubeValue(t, ctx, kubeconfig, "get", "pvc", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[*].metadata.uid}") || before.PodUID != kubeValue(t, ctx, kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "jsonpath={.metadata.uid}") { + t.Fatal("runtime status does not name the actual PVC/Pod UIDs") + } + fixtureIP := kubeValue(t, ctx, kubeconfig, "get", "pod/network-fixture", "-n", namespace, "-o", "jsonpath={.status.podIP}") + peerIP := kubeValue(t, ctx, kubeconfig, "get", "pod/network-intruder", "-n", namespace, "-o", "jsonpath={.status.podIP}") + + // Only nonsecret chart values and manifests are captured. No Helm manifest or + // Secret dump is written to artifacts; command errors report status only. + raw, err := helmLifetime(ctx, kubeconfig, "get", "values", "mecatl-execution", "-o", "json") + if err != nil { + t.Fatal("read owned release values:", err) + } + var values map[string]any + if err := json.Unmarshal(raw, &values); err != nil { + t.Fatal("decode release values") + } + current := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-security-manifest") + oldManifest := current.Data["manifest.json"] + var manifest map[string]any + if err := json.Unmarshal([]byte(oldManifest), &manifest); err != nil { + t.Fatal("decode nonsecret authority manifest") + } + generation, ok := manifest["generation"].(float64) + if !ok || generation < 1 { + t.Fatal("missing authority generation") + } + manifest["generation"] = generation + 1 + newManifest, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + provider, ok := values["provider"].(map[string]any) + if !ok { + t.Fatal("missing provider values") + } + provider["securityManifest"] = string(newManifest) + dir, err := os.MkdirTemp(state, "helm-lifetime-") + if err != nil { + t.Fatal(err) + } + valuesPath := filepath.Join(dir, "values.json") + raw, err = json.Marshal(values) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(valuesPath, raw, 0o600); err != nil { + t.Fatal(err) + } + chart := filepath.Join(repoRoot(t), "deploy/helm/mecatl-execution") + restore := func(ctx context.Context) error { + if err := quiesceLifetimeProvider(ctx, kubeconfig); err != nil { + return err + } + _, err := helmLifetime(ctx, kubeconfig, "upgrade", "--install", "mecatl-execution", chart, "-f", valuesPath, "--wait", "--timeout=4m") + return err + } + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 7*time.Minute) + defer cleanupCancel() + requireOwnedHelmFixture(t, state, kubeconfig) + if err := restore(cleanupCtx); err != nil { + t.Error("restore owned release/current authority failed:", err) + } + }) + capacityBefore := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-profile-allocations") + authorityBefore := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-security-authority") + var priorHistory struct { + Fingerprints map[string]string `json:"fingerprints"` + } + if err := json.Unmarshal([]byte(authorityBefore.Data["state.json"]), &priorHistory); err != nil || len(priorHistory.Fingerprints) == 0 { + t.Fatal("missing pre-upgrade authority history") + } + policiesBefore := lifetimePolicies(ctx, t, kubeconfig) + liveLedgers := map[string]corev1.ConfigMap{capacityBefore.Name: capacityBefore, authorityBefore.Name: authorityBefore, current.Name: current} + liveLedgers["mecatl-execution-profiles"] = lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-profiles") + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + if _, err := helmLifetime(ctx, kubeconfig, "upgrade", "mecatl-execution", chart, "-f", valuesPath, "--wait", "--timeout=4m"); !errors.Is(err, errLifetimeNotQuiesced) { + t.Fatal("live same-release upgrade did not reject provider writers with the quiescence error") + } + assertLifetimeRetained(ctx, t, kubeconfig, liveLedgers, policiesBefore) + if status := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID); status != before || envUID != kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") || before.PodUID != kubeValue(t, ctx, kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "jsonpath={.metadata.uid}") || before.PVCUID != kubeValue(t, ctx, kubeconfig, "get", "pvc", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[*].metadata.uid}") { + t.Fatal("rejected live upgrade changed runtime identity or status") + } + // Quiesce all writers before lookup snapshots are rendered into the upgrade. + if err := restore(ctx); err != nil { + t.Fatal("compatible Helm upgrade failed:", err) + } + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + ledgers := map[string]corev1.ConfigMap{} + for _, name := range []string{"mecatl-execution-security-authority", "mecatl-execution-profile-allocations", "mecatl-execution-profiles", "mecatl-execution-security-manifest"} { + ledgers[name] = lifetimeConfigMap(ctx, t, kubeconfig, name) + } + var highWater struct { + Generation uint64 `json:"generation"` + Fingerprints map[string]string `json:"fingerprints"` + } + if err := json.Unmarshal([]byte(ledgers["mecatl-execution-security-authority"].Data["state.json"]), &highWater); err != nil || highWater.Generation != uint64(generation+1) || !reflect.DeepEqual(highWater.Fingerprints, priorHistory.Fingerprints) { + t.Fatal("upgrade did not preserve key history and publish higher authority") + } + if capacityAfter := ledgers[capacityBefore.Name]; capacityAfter.UID != capacityBefore.UID || !reflect.DeepEqual(capacityAfter.Data, capacityBefore.Data) { + t.Fatal("compatible upgrade changed capacity reservations") + } + policies := lifetimePolicies(ctx, t, kubeconfig) + if len(policies) < 2 { + t.Fatal("workload policies missing before uninstall") + } + if _, err := helmLifetime(ctx, kubeconfig, "uninstall", "mecatl-execution", "--wait", "--timeout=2m"); err != nil { + t.Fatal("uninstall owned release failed:", err) + } + waitResourceAbsent(t, ctx, kubeconfig, "deployment", "mecatl-execution") + assertLifetimeRetained(ctx, t, kubeconfig, ledgers, policies) + after := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + if after.PVCUID != before.PVCUID || after.PodUID != before.PodUID || before.PVCUID != kubeValue(t, ctx, kubeconfig, "get", "pvc", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[*].metadata.uid}") || before.PodUID != kubeValue(t, ctx, kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "jsonpath={.metadata.uid}") || envUID != kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") { + t.Fatal("uninstall replaced retained runtime identity") + } + // Bypass the absent provider only to observe the *same executor*. Positive + // reachability prevents an all-network-down result masquerading as isolation. + lifetimeProbe(ctx, t, kubeconfig, pod, fixtureIP+":8080", 0) + lifetimeProbe(ctx, t, kubeconfig, pod, peerIP+":8080", 42) + lifetimeProbe(ctx, t, kubeconfig, pod, "10.96.0.1:443", 42) + lifetimeProbe(ctx, t, kubeconfig, pod, "1.1.1.1:443", 42) + data := runKubectl(t, ctx, kubeconfig, "exec", "-n", namespace, pod, "--", "cat", "/workspace/helm-sentinel") + if string(data) != "retained-helm-data\n" { + t.Fatal("retained workspace data changed during provider absence") + } + // An incompatible profile is refused before a new release can mutate anything. + if _, err := helmLifetime(ctx, kubeconfig, "install", "mecatl-execution", chart, "-f", valuesPath, "--set", "profiles.go.maxEnvironments=30", "--dry-run=server"); err == nil { + t.Fatal("incompatible reinstall was accepted") + } + // Refusal must happen without adopting another release or bootstrapping a + // new authority name over this namespace's retained allocations. + for _, args := range [][]string{ + {"install", "foreign-execution", chart, "-f", valuesPath, "--dry-run=server"}, + {"install", "mecatl-execution", chart, "-f", valuesPath, "--set", "fullnameOverride=missing-history", "--dry-run=server"}, + } { + if _, err := helmLifetime(ctx, kubeconfig, args...); err == nil { + t.Fatal("foreign/missing-history reinstall was accepted") + } + } + assertLifetimeRetained(ctx, t, kubeconfig, ledgers, policies) + if _, err := helmLifetime(ctx, kubeconfig, "install", "mecatl-execution", chart, "-f", valuesPath, "--wait", "--timeout=4m"); err != nil { + t.Fatal("same-identity Helm reinstall/adoption failed:", err) + } + assertLifetimeRetained(ctx, t, kubeconfig, ledgers, policies) + reattachedClient, _ := productionClient(t, ctx, state, kubeconfig) + reattached := waitReady(t, ctx, reattachedClient, owner, binding, attached.Environment) + if reattached.Environment != attached.Environment { + t.Fatal("reattach changed exact environment revision") + } + after = readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + if after.PVCUID != before.PVCUID || after.PodUID != before.PodUID || before.PVCUID != kubeValue(t, ctx, kubeconfig, "get", "pvc", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[*].metadata.uid}") || before.PodUID != kubeValue(t, ctx, kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "jsonpath={.metadata.uid}") || envUID != kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") { + t.Fatal("adoption replaced retained runtime identity") + } + rc, release = acquireRun(t, ctx, reattachedClient, owner, binding, reattached, "helm-lifetime-read") + waitFileContent(t, ctx, reattachedClient, rc, "helm-sentinel", "retained-helm-data\n", "same-release Helm adoption") + assertRetiredFixtureKeyDenied(ctx, t, reattachedClient, state, rc, "helm-sentinel", "retained-helm-data\n") + release() + if _, err := reattachedClient.Ensure(ctx, quotaBinding+"-excess", "quota-cas", owner, "ensure-"+quotaBinding+"-excess"); !isRemoteCode(err, executionenv.CodeResourceExhausted) { + t.Fatal("retained capacity did not reject excess allocation:", remoteErrorCode(err)) + } + + // The old manifest still has valid key material, windows, and clients. Only + // its generation is older. An existing connection must fail after reload. + patch, err := json.Marshal(map[string]any{"data": map[string]string{"manifest.json": oldManifest}}) + if err != nil { + t.Fatal(err) + } + runKubectl(t, ctx, kubeconfig, "patch", "configmap/mecatl-execution-security-manifest", "-n", namespace, "--type=merge", "-p", string(patch)) + waitProviderReadyReplicas(t, ctx, kubeconfig, 0) + if _, err := reattachedClient.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); !isRemoteCode(err, executionenv.CodeNotReady) { + t.Fatal("older still-valid authority did not fail closed with not_ready after reinstall:", remoteErrorCode(err)) + } + authority := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-security-authority") + if !reflect.DeepEqual(authority.Data, ledgers[authority.Name].Data) { + t.Fatal("stale candidate reset authority history") + } + if err := restore(ctx); err != nil { + t.Fatal("restore current authority:", err) + } + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + restoredClient, _ := productionClient(t, ctx, state, kubeconfig) + waitReady(t, ctx, restoredClient, owner, binding, attached.Environment) + // Retain the test's data by default, as the chart does; no forced cleanup or + // finalizer removal. The explicit test-cluster owner controls final disposal. +} + +// Quiesced upgrades can outlive a grant/claim. Re-sign the CURRENT claim with +// the fixture's retired k1, before and after adoption: expiry, released claims, +// and stale epochs cannot masquerade as durable authority rejection. +func assertRetiredFixtureKeyDenied(ctx context.Context, t *testing.T, client *executionclient.Client, state string, rc executionenv.RequestContext, path, want string) { + t.Helper() + old := resignFixtureGrant(t, rc, filepath.Join(state, "pki", "grant-key.pem"), "k1", rc.GrantGeneration) + waitFileContent(t, ctx, client, rc, path, want, "current authority before retired-key probe") + if _, err := client.File(ctx, executionenv.FileRequest{Context: old, Operation: executionenv.OpFileRead, Path: path}); !isRemoteCode(err, executionenv.CodePermissionDenied) { + var remote *executionenv.Error + t.Fatalf("retired signing authority rejection: errorCode=%s retryable=%t", remoteErrorCode(err), errors.As(err, &remote) && remote.Retryable) + } + waitFileContent(t, ctx, client, rc, path, want, "current authority after retired-key probe") +} + +func resignFixtureGrant(t *testing.T, rc executionenv.RequestContext, keyPath, keyID string, generation uint64) executionenv.RequestContext { + t.Helper() + parts := strings.Split(rc.Grant, ".") + if len(parts) != 3 { + t.Fatal("invalid current fixture grant") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + t.Fatal("decode current fixture grant") + } + var claims executionenv.GrantClaims + if err := json.Unmarshal(payload, &claims); err != nil { + t.Fatal("decode current fixture claims") + } + material, err := os.ReadFile(keyPath) + if err != nil { + t.Fatal("read test-owned signing key") + } + block, _ := pem.Decode(material) + if block == nil { + t.Fatal("decode test-owned signing key") + } + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + t.Fatal("parse test-owned signing key") + } + private, ok := key.(ed25519.PrivateKey) + if !ok { + t.Fatal("fixture signing key is not Ed25519") + } + claims.KeyID = keyID + claims.GrantGeneration = generation + probe := rc + probe.GrantGeneration = generation + probe.Grant, err = executionenv.SignGrant(private, claims) + if err != nil { + t.Fatal("sign fixture grant") + } + verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{keyID: private.Public().(ed25519.PublicKey)}, Issuer: claims.Issuer, Audience: claims.Audience, MaxLifetime: time.Minute} + if _, err := verifier.Verify(probe.Grant, executionenv.GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: generation, Operation: executionenv.OpFileRead}); err != nil { + t.Fatal("fixture probe is not cryptographically valid and current") + } + return probe +} + +func TestResignFixtureGrantChangesOnlyRequestedAuthority(t *testing.T) { + key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, ed25519.SeedSize)) + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal("marshal synthetic signing key") + } + keyPath := filepath.Join(t.TempDir(), "synthetic.pem") + if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), 0o600); err != nil { + t.Fatal("write synthetic signing key") + } + now := time.Now().UTC().Truncate(time.Second) + claims := executionenv.GrantClaims{KeyID: "k2", Issuer: "fixture", Audience: "fixture", Client: "fixture-client", OwnerHash: "fixture-owner", BindingID: "binding", RunID: "run", ClaimID: "claim", Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, Epoch: 3, GrantGeneration: 2, Operations: []executionenv.Operation{executionenv.OpFileRead}, NotBefore: now.Add(-time.Second), ExpiresAt: now.Add(30 * time.Second), Nonce: "fixture-nonce"} + grant, err := executionenv.SignGrant(key, claims) + if err != nil { + t.Fatal("sign synthetic control") + } + rc := executionenv.RequestContext{Environment: claims.Environment, Owner: executionenv.Owner{Issuer: "issuer", Subject: "subject"}, BindingID: claims.BindingID, RunID: claims.RunID, ClaimID: claims.ClaimID, Epoch: claims.Epoch, GrantGeneration: claims.GrantGeneration, Grant: grant} + for _, probe := range []struct { + keyID string + generation uint64 + }{{"k2", 1}, {"k1", 2}} { + resigned := resignFixtureGrant(t, rc, keyPath, probe.keyID, probe.generation) + verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{probe.keyID: key.Public().(ed25519.PublicKey)}, Issuer: claims.Issuer, Audience: claims.Audience, MaxLifetime: time.Minute} + got, err := verifier.Verify(resigned.Grant, executionenv.GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: probe.generation, Operation: executionenv.OpFileRead}) + want := claims + want.KeyID, want.GrantGeneration = probe.keyID, probe.generation + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatal("probe changed claims beyond key ID and generation") + } + resigned.Grant, resigned.GrantGeneration = rc.Grant, rc.GrantGeneration + if resigned != rc { + t.Fatal("probe changed request identity") + } + } +} + +func requireOwnedHelmFixture(t *testing.T, state, kubeconfig string) { + t.Helper() + ownership, err := os.ReadFile(filepath.Join(state, "ownership")) + if err != nil { + t.Fatal("lifetime test requires fixture ownership record") + } + fields := map[string]string{} + for line := range strings.SplitSeq(string(ownership), "\n") { + key, value, ok := strings.Cut(line, "=") + if ok { + fields[key] = value + } + } + cluster := fields["cluster"] + if !strings.HasPrefix(cluster, "mecatl-execution-qual-") || fields["context"] != "kind-"+cluster || fields["context"] != os.Getenv("MECATL_KUBE_CONTEXT") || fields["kubeconfig"] != kubeconfig || fields["namespace"] != namespace || fields["profile"] != "production" { + t.Fatal("lifetime test refuses non-owned cluster/context/namespace") + } +} + +func lifetimeConfigMap(ctx context.Context, t *testing.T, kubeconfig, name string) corev1.ConfigMap { + t.Helper() + var cm corev1.ConfigMap + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "configmap", name, "-n", namespace, "-o", "json"), &cm); err != nil { + t.Fatal("decode nonsecret lifetime ConfigMap") + } + if cm.UID == "" || cm.Labels["app.kubernetes.io/managed-by"] != "Helm" || cm.Annotations["meta.helm.sh/release-name"] != "mecatl-execution" || cm.Annotations["meta.helm.sh/release-namespace"] != namespace { + t.Fatal("refuse foreign lifetime ConfigMap", name) + } + return cm +} + +func lifetimePolicies(ctx context.Context, t *testing.T, kubeconfig string) map[string]networkingv1.NetworkPolicy { + t.Helper() + var list networkingv1.NetworkPolicyList + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "networkpolicy", "-n", namespace, "-o", "json"), &list); err != nil { + t.Fatal("decode lifetime policies") + } + result := map[string]networkingv1.NetworkPolicy{} + for _, p := range list.Items { + if p.Name == "mecatl-execution-workload-default-deny" || strings.HasPrefix(p.Name, "mecatl-execution-profile-") { + result[p.Name] = p + } + } + return result +} + +func assertLifetimeRetained(ctx context.Context, t *testing.T, kubeconfig string, cms map[string]corev1.ConfigMap, policies map[string]networkingv1.NetworkPolicy) { + t.Helper() + for name, before := range cms { + after := lifetimeConfigMap(ctx, t, kubeconfig, name) + if before.UID != after.UID || !reflect.DeepEqual(before.Data, after.Data) { + t.Fatal("retained ConfigMap identity/data changed", name) + } + } + live := lifetimePolicies(ctx, t, kubeconfig) + if len(live) != len(policies) { + t.Fatal("retained policy set changed") + } + for name, before := range policies { + after := live[name] + if before.UID != after.UID || !reflect.DeepEqual(before.Spec, after.Spec) { + t.Fatal("retained policy identity/confinement changed", name) + } + } +} + +func lifetimeProbe(ctx context.Context, t *testing.T, kubeconfig, pod, endpoint string, want int) { + t.Helper() + cmd := command(ctx, kubeconfig, "exec", "-n", namespace, pod, "--", "/workspace/helm-probe", endpoint) + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + err := cmd.Run() + if want == 0 && err == nil { + return + } + var exit *exec.ExitError + if want != 0 && errors.As(err, &exit) && exit.ExitCode() == want { + return + } + t.Fatalf("retained executor network probe did not produce exit %d", want) +} + +func quiesceLifetimeProvider(ctx context.Context, kubeconfig string) error { + cmd := command(ctx, kubeconfig, "get", "deployment/mecatl-execution", "-n", namespace, "--ignore-not-found", "-o", "json") + var out lifetimeOutput + cmd.Stdout = &out + cmd.Stderr = io.Discard + if err := cmd.Run(); err != nil { + return fmt.Errorf("read owned provider deployment: %w", err) + } + if out.Len() == 0 { + return nil + } + var deployment struct { + Metadata struct{ Labels, Annotations map[string]string } + } + if err := json.Unmarshal(out.Bytes(), &deployment); err != nil { + return errors.New("decode owned provider deployment") + } + if deployment.Metadata.Labels["app.kubernetes.io/managed-by"] != "Helm" || deployment.Metadata.Annotations["meta.helm.sh/release-name"] != "mecatl-execution" || deployment.Metadata.Annotations["meta.helm.sh/release-namespace"] != namespace { + return errors.New("refuse to quiesce a foreign provider deployment") + } + if err := command(ctx, kubeconfig, "scale", "deployment/mecatl-execution", "-n", namespace, "--replicas=0").Run(); err != nil { + return fmt.Errorf("quiesce owned provider: %w", err) + } + if err := command(ctx, kubeconfig, "wait", "--for=delete", "pod", "-n", namespace, "-l", "app.kubernetes.io/name=mecatl-execution", "--timeout=2m").Run(); err != nil { + return fmt.Errorf("wait for provider quiescence: %w", err) + } + return nil +} + +var errLifetimeNotQuiesced = errors.New("execution provider must be quiesced") + +func helmLifetime(ctx context.Context, kubeconfig string, args ...string) ([]byte, error) { + full := append([]string{"--kubeconfig", kubeconfig, "--kube-context", os.Getenv("MECATL_KUBE_CONTEXT"), "--namespace", namespace}, args...) + cmd := exec.CommandContext(ctx, "helm", full...) + cmd.Env = cleanEnv() + var out, stderr lifetimeOutput + cmd.Stdout = &out + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if bytes.Contains(stderr.Bytes(), []byte("quiesce the execution provider (scale to zero and wait for its Pods to disappear) before upgrading")) { + return nil, errLifetimeNotQuiesced + } + return nil, fmt.Errorf("Helm %s failed: %w (output suppressed)", args[0], err) + } + return out.Bytes(), nil +} + +func TestLifetimeOutputRejectsOversizedArtifacts(t *testing.T) { + var out lifetimeOutput + if _, err := io.Copy(&out, strings.NewReader(strings.Repeat("x", 1<<20))); err != nil || out.Len() != 1<<20 { + t.Fatal("bounded output did not accept its exact limit") + } + if _, err := io.Copy(&out, strings.NewReader("overflow")); err == nil || out.Len() != 1<<20 { + t.Fatal("output exceeded its bound through io.Copy") + } +} + +type lifetimeOutput struct{ buffer bytes.Buffer } + +func (b *lifetimeOutput) Len() int { return b.buffer.Len() } +func (b *lifetimeOutput) Bytes() []byte { return b.buffer.Bytes() } + +func (b *lifetimeOutput) Write(p []byte) (int, error) { + if b.Len()+len(p) > 1<<20 { + return 0, errors.New("lifetime output exceeds 1 MiB") + } + return b.buffer.Write(p) +} diff --git a/engine/agent/authority_evaluator_test.go b/engine/agent/authority_evaluator_test.go index 70bec40d10..6cedc58805 100644 --- a/engine/agent/authority_evaluator_test.go +++ b/engine/agent/authority_evaluator_test.go @@ -3,6 +3,7 @@ package agent_test import ( "context" "encoding/json" + "errors" "strings" "sync" "sync/atomic" @@ -431,6 +432,54 @@ func TestADR_0233_AuthorityEvaluator_Scenario7_ResourceAttributeIsDerivedWithout }) } +type failingAuthorityWorkspace struct { + tool.Workspace + err error +} + +func (w failingAuthorityWorkspace) AuthorityResourcePath(string) (string, string, error) { + return "", "", w.err +} + +func TestAuthorityResourceResolutionFailureDeniesBeforeEvaluator(t *testing.T) { + for _, tc := range []struct { + name string + err error + }{ + {name: "remote resolver unavailable", err: context.DeadlineExceeded}, + {name: "physical symlink escape", err: errors.New("path is outside workspace")}, + } { + t.Run(tc.name, func(t *testing.T) { + read := &authorityTool{name: "Read"} + evaluator := &recordingAuthorityEvaluator{decision: port.AuthorityDecision{Allowed: true}} + eng := newEngine(agent.Deps{ + LLM: mockllm.New(mockllm.ToolCallTurn(toolCall("read", "Read", `{"path":"escape"}`))), + Catalog: catalogWith(t, read), + AuthorityEvaluator: evaluator, + }) + base := agent.MemEnv("/workspace") + env := tool.MustEnvironment(base.Ref(), failingAuthorityWorkspace{Workspace: base.Workspace(), err: tc.err}, base.ReadLedger(), nil) + sess := authoritySession(t, "Read") + if err := sess.Rehome(env.Ref()); err != nil { + t.Fatal(err) + } + events := drain(eng.Run(context.Background(), sess, env, agent.RunRequest{Text: "read"})) + if read.ran.Load() != 0 || evaluator.calls() != 0 { + t.Fatalf("tool executions=%d evaluator calls=%d, want 0/0", read.ran.Load(), evaluator.calls()) + } + for _, event := range events { + if event.ToolResult != nil && event.ToolResult.CallID == "read" { + if !event.ToolResult.IsError || !strings.Contains(event.ToolResult.Content, "authority resource") { + t.Fatalf("tool result=%+v, want fail-closed authority resolution error", event.ToolResult) + } + return + } + } + t.Fatal("missing fail-closed tool result") + }) + } +} + // sequencedAuthorityEvaluator returns one decision per call in order, so a test // can distinguish the source-resource request from the destination-resource // request in a dual-resource call (Copy/Move). diff --git a/go.work.sum b/go.work.sum index 17b4f1cdd6..75c96d67f0 100644 --- a/go.work.sum +++ b/go.work.sum @@ -500,6 +500,7 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/certificate-transparency-go v1.3.3 h1:hq/rSxztSkXN2tx/3jQqF6Xc0O565UQPdHrOWvZwybo= github.com/google/certificate-transparency-go v1.3.3/go.mod h1:iR17ZgSaXRzSa5qvjFl8TnVD5h8ky2JMVio+dzoKMgA= diff --git a/internal/adapter/executionclient/client.go b/internal/adapter/executionclient/client.go new file mode 100644 index 0000000000..2ab3b875ec --- /dev/null +++ b/internal/adapter/executionclient/client.go @@ -0,0 +1,1180 @@ +// Package executionclient implements the private mTLS client and placement adapter +// for the independently deployed Kubernetes execution provider. +// +//nolint:revive // Private protocol methods are intentionally explicit across adapter boundaries. +package executionclient + +import ( + "context" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "net" + "os" + "strings" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + grpccredentials "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/creatediag" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + remoteRoot = "/workspace" + authorityResolveRequestTimeout = 10 * time.Second +) + +// TLSFiles names the runtime-only mTLS material mounted into mecak8s. +type TLSFiles struct{ CA, Cert, Key string } + +// LoadTLSConfig loads provider trust and client identity at process startup. +func LoadTLSConfig(files TLSFiles) (*tls.Config, error) { + if files.CA == "" || files.Cert == "" || files.Key == "" { + return nil, errors.New("execution client: CA, certificate, and key are required") + } + ca, err := os.ReadFile(files.CA) + if err != nil { + return nil, fmt.Errorf("execution client: load CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(ca) { + return nil, errors.New("execution client: CA contains no certificates") + } + cert, err := tls.LoadX509KeyPair(files.Cert, files.Key) + if err != nil { + return nil, fmt.Errorf("execution client: load client identity: %w", err) + } + return &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: pool, Certificates: []tls.Certificate{cert}}, nil +} + +// Client is a bounded client for the private execution-provider gRPC API. +type Client struct { + conn *grpc.ClientConn + rpc executionv1.ExecutionProviderServiceClient +} + +// New constructs a production mTLS execution-provider client. Endpoint is a +// plain host:port authority; URI schemes and alternate resolvers are rejected. +func New(endpoint string, tlsConfig *tls.Config) (*Client, error) { + if strings.Contains(endpoint, "://") || strings.ContainsAny(endpoint, "/?#@") { + return nil, errors.New("execution client: endpoint must be host:port") + } + host, port, err := net.SplitHostPort(endpoint) + if err != nil || host == "" || port == "" { + return nil, errors.New("execution client: endpoint must be host:port") + } + if tlsConfig == nil || len(tlsConfig.Certificates) == 0 || tlsConfig.RootCAs == nil { + return nil, errors.New("execution client: production mTLS configuration is required") + } + cfg := tlsConfig.Clone() + if cfg.MinVersion < tls.VersionTLS13 { + cfg.MinVersion = tls.VersionTLS13 + } + conn, err := grpc.NewClient(endpoint, + grpc.WithTransportCredentials(grpccredentials.NewTLS(cfg)), + grpc.WithDisableRetry(), + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(executionenv.MaxMessageBytes), grpc.MaxCallSendMsgSize(executionenv.MaxMessageBytes)), + ) + if err != nil { + return nil, fmt.Errorf("execution client: connect: %w", err) + } + return &Client{conn: conn, rpc: executionv1.NewExecutionProviderServiceClient(conn)}, nil +} + +// Close releases the provider connection. +func (c *Client) Close() { + if c != nil && c.conn != nil { + _ = c.conn.Close() + } +} + +// ValidateProfile validates a provider profile without allocating an environment. +func (c *Client) ValidateProfile(ctx context.Context, profile string) (executionenv.ValidateProfileResponse, error) { + v, err := c.rpc.ValidateProfile(ctx, &executionv1.ValidateProfileRequest{Profile: profile}) + if err != nil { + return executionenv.ValidateProfileResponse{}, decodeError(ctx, err) + } + return executionenv.ValidateProfileResponse{Profile: v.Profile, Digest: v.Digest, Capabilities: v.Capabilities, MaxFileBytes: v.MaxFileBytes, MaxCommandBytes: v.MaxCommandBytes, MaxCommandDurationMillis: v.MaxCommandDurationMillis}, nil +} + +func newOperationID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +// Ensure idempotently allocates or resolves the environment for a binding. +func (c *Client) Ensure(ctx context.Context, binding, profile string, owner executionenv.Owner, operationID string) (executionenv.EnsureEnvironmentResponse, error) { + v, err := c.rpc.EnsureEnvironment(ctx, &executionv1.EnsureEnvironmentRequest{BindingId: binding, Profile: profile, Owner: ownerToProto(owner), OperationId: operationID}) + if err != nil { + return executionenv.EnsureEnvironmentResponse{}, decodeError(ctx, err) + } + return ensureFromProto(v) +} + +// Attach obtains a fresh grant for an exact environment reference. +func (c *Client) Attach(ctx context.Context, req executionenv.AttachEnvironmentRequest) (executionenv.AttachEnvironmentResponse, error) { + v, err := c.rpc.AttachEnvironment(ctx, &executionv1.AttachEnvironmentRequest{Context: contextToProto(req.Context), Purpose: req.Purpose}) + if err != nil { + return executionenv.AttachEnvironmentResponse{}, decodeError(ctx, err) + } + return attachFromProto(v) +} + +// AcquireRun acquires one environment-wide run claim. +func (c *Client) AcquireRun(ctx context.Context, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + v, err := c.rpc.AcquireRun(ctx, &executionv1.AcquireRunRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), BindingId: req.BindingID, RunId: req.RunID, OperationId: req.OperationID, TtlMillis: req.TTL.Milliseconds()}) + if err != nil { + return executionenv.RunClaim{}, decodeError(ctx, err) + } + return runClaimFromProto(v) +} +func (c *Client) RenewRun(ctx context.Context, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + v, err := c.rpc.RenewRun(ctx, &executionv1.RenewRunRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), BindingId: req.BindingID, RunId: req.RunID, ClaimId: req.ClaimID, Epoch: req.Epoch, GrantGeneration: req.GrantGeneration, OperationId: req.OperationID, TtlMillis: req.TTL.Milliseconds()}) + if err != nil { + return executionenv.RunClaim{}, decodeError(ctx, err) + } + return runClaimFromProto(v) +} +func (c *Client) ReleaseRun(ctx context.Context, req executionenv.RunClaimRequest) error { + _, err := c.rpc.ReleaseRun(ctx, &executionv1.ReleaseRunRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), BindingId: req.BindingID, RunId: req.RunID, ClaimId: req.ClaimID, Epoch: req.Epoch, GrantGeneration: req.GrantGeneration, OperationId: req.OperationID}) + return decodeError(ctx, err) +} +func (c *Client) referenceMutation(ctx context.Context, req executionenv.ReferenceRequest, call func(context.Context, *executionv1.ReferenceMutationRequest, ...grpc.CallOption) (*emptypb.Empty, error)) error { + _, err := call(ctx, &executionv1.ReferenceMutationRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), BindingId: req.BindingID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} +func (c *Client) CommitReference(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.CommitReference) +} +func (c *Client) AbortReference(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.AbortReference) +} +func (c *Client) PrepareReferenceDelete(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.PrepareReferenceDelete) +} +func (c *Client) ConfirmReferenceDelete(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.ConfirmReferenceDelete) +} +func (c *Client) CancelReferenceDelete(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.CancelReferenceDelete) +} +func (c *Client) ReserveSuccessor(ctx context.Context, req executionenv.ReferenceRequest) error { + _, err := c.rpc.ReserveSuccessor(ctx, &executionv1.ReserveSuccessorRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), SourceBindingId: req.SourceBindingID, DestinationBindingId: req.BindingID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} + +func (c *Client) ListReferenceIntents(ctx context.Context, owner executionenv.Owner) ([]executionenv.ReferenceIntent, error) { + return c.listReferenceIntents(ctx, &executionv1.ListReferenceIntentsRequest{Owner: ownerToProto(owner), Limit: 64}) +} + +func (c *Client) FindReferenceIntent(ctx context.Context, owner executionenv.Owner, environment executionenv.EnvironmentRef, binding string) (executionenv.ReferenceIntent, error) { + intents, err := c.listReferenceIntents(ctx, &executionv1.ListReferenceIntentsRequest{Owner: ownerToProto(owner), Limit: 1, Environment: refToProto(environment), BindingId: binding}) + if err != nil { + return executionenv.ReferenceIntent{}, err + } + if len(intents) != 1 { + return executionenv.ReferenceIntent{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference intent not found"} + } + return intents[0], nil +} + +func (c *Client) ListAllReferenceIntents(ctx context.Context) ([]executionenv.ReferenceIntent, error) { + return c.listReferenceIntents(ctx, &executionv1.ListReferenceIntentsRequest{Limit: 64}) +} + +func (c *Client) listReferenceIntents(ctx context.Context, req *executionv1.ListReferenceIntentsRequest) ([]executionenv.ReferenceIntent, error) { + v, err := c.rpc.ListReferenceIntents(ctx, req) + if err != nil { + return nil, decodeError(ctx, err) + } + out := make([]executionenv.ReferenceIntent, 0, len(v.GetIntents())) + for _, intent := range v.GetIntents() { + created, timeErr := checkedTime(intent.GetCreatedAt()) + if timeErr != nil { + return nil, sanitizedProviderError() + } + state := executionenv.ReferenceState(intent.GetState()) + if state != executionenv.ReferencePendingCreate && state != executionenv.ReferencePendingDelete { + return nil, sanitizedProviderError() + } + owner := executionenv.Owner{} + ownerOK := intent.GetOwner() == nil + if intent.GetOwner() != nil { + owner = executionenv.Owner{Issuer: intent.GetOwner().GetIssuer(), Subject: intent.GetOwner().GetSubject()} + ownerOK = owner.Issuer != "" && owner.Subject != "" && len(owner.Issuer) <= executionenv.MaxIdentityBytes && len(owner.Subject) <= executionenv.MaxIdentityBytes + } + if !ownerOK { + return nil, sanitizedProviderError() + } + out = append(out, executionenv.ReferenceIntent{Environment: refFromProto(intent.GetEnvironment()), Owner: owner, BindingID: intent.GetBindingId(), State: state, OperationID: intent.GetOperationId(), SourceBindingID: intent.GetSourceBindingId(), CreatedAt: created}) + } + return out, nil +} + +// File executes one authorized filesystem operation. +func (c *Client) File(ctx context.Context, req executionenv.FileRequest) (executionenv.FileResponse, error) { + if req.Limit < 0 || req.Limit > executionenv.MaxListEntries { + return executionenv.FileResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "invalid file request limit"} + } + v, err := c.rpc.Files(ctx, &executionv1.FileRequest{Context: contextToProto(req.Context), Operation: fileOperationToProto(req.Operation), Path: req.Path, Destination: req.Destination, Pattern: req.Pattern, Data: req.Data, Version: []byte(req.Version), Limit: int32(req.Limit)}) //nolint:gosec // bounded above + if err != nil { + return executionenv.FileResponse{}, decodeError(ctx, err) + } + return fileFromProto(v), nil +} + +// StartCommand executes one foreground command. +func (c *Client) StartCommand(ctx context.Context, req executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + v, err := c.rpc.StartCommand(ctx, &executionv1.CommandStartRequest{Context: contextToProto(req.Context), Command: req.Command, TimeoutMillis: req.TimeoutMillis}) + if err != nil { + return executionenv.CommandStartResponse{}, decodeError(ctx, err) + } + return commandStartFromProto(v) +} + +// CommandStatus queries a command after the request has been authorized. +func (c *Client) CommandStatus(ctx context.Context, req executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + v, err := c.rpc.CommandStatus(ctx, &executionv1.CommandQueryRequest{Context: contextToProto(req.Context), CommandId: req.CommandID, Offset: req.Offset}) + if err != nil { + return executionenv.CommandStatusResponse{}, decodeError(ctx, err) + } + return commandStatusFromProto(v) +} + +// CancelCommand cancels a command after the request has been authorized. +func (c *Client) CancelCommand(ctx context.Context, req executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + v, err := c.rpc.CancelCommand(ctx, &executionv1.CommandQueryRequest{Context: contextToProto(req.Context), CommandId: req.CommandID, Offset: req.Offset}) + if err != nil { + return executionenv.CommandStatusResponse{}, decodeError(ctx, err) + } + return commandStatusFromProto(v) +} + +// ReleaseReference releases one durable binding reference. +func (c *Client) ReleaseReference(ctx context.Context, req executionenv.ReferenceReleaseRequest) error { + _, err := c.rpc.ReleaseReference(ctx, &executionv1.ReleaseReferenceRequest{Context: contextToProto(req.Context)}) + return decodeError(ctx, err) +} + +// ReplaceExecutor requests UID-checked administrative executor replacement. +func (c *Client) ReplaceExecutor(ctx context.Context, req executionenv.RetireEnvironmentRequest) error { + _, err := c.rpc.ReplaceExecutor(ctx, &executionv1.ReplaceExecutorRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), ExpectedExecutionEpoch: req.ExpectedEpoch, ExpectedPodUid: req.ExpectedPodUID, ExpectedPvcUid: req.ExpectedPVCUID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} + +// RetireEnvironment requests administrative retirement. +func (c *Client) RetireEnvironment(ctx context.Context, req executionenv.RetireEnvironmentRequest) error { + _, err := c.rpc.RetireEnvironment(ctx, &executionv1.RetireEnvironmentRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), ExpectedExecutionEpoch: req.ExpectedEpoch, ExpectedPodUid: req.ExpectedPodUID, ExpectedPvcUid: req.ExpectedPVCUID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} + +// DeleteRetiredEnvironment deletes retained storage after retirement proof. +func (c *Client) DeleteRetiredEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, owner executionenv.Owner, expectedPVCUID, operationID string) error { + _, err := c.rpc.DeleteRetiredEnvironment(ctx, &executionv1.DeleteRetiredEnvironmentRequest{Environment: refToProto(ref), Owner: ownerToProto(owner), ExpectedPvcUid: expectedPVCUID, OperationId: operationID}) + return decodeError(ctx, err) +} + +// RevokeEnvironment atomically fences grants and replays a matching operation receipt. +func (c *Client) RevokeEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, owner executionenv.Owner, expectedGeneration uint64, operationID string) (uint64, error) { + out, err := c.rpc.RevokeEnvironment(ctx, &executionv1.RevokeEnvironmentRequest{Environment: refToProto(ref), Owner: ownerToProto(owner), ExpectedGrantGeneration: expectedGeneration, OperationId: operationID}) + if err != nil { + return 0, decodeError(ctx, err) + } + return out.GetGrantGeneration(), nil +} + +// MigrateEnvironment upgrades one explicitly identified legacy schema. +func (c *Client) MigrateEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, owner executionenv.Owner, expectedSchema uint32, podUID, pvcUID, operationID string) error { + _, err := c.rpc.MigrateEnvironment(ctx, &executionv1.MigrateEnvironmentRequest{Environment: refToProto(ref), Owner: ownerToProto(owner), ExpectedSchemaVersion: &expectedSchema, ExpectedPodUid: podUID, ExpectedPvcUid: pvcUID, OperationId: operationID}) + return decodeError(ctx, err) +} + +// RecoverEnvironment clears a fence only after the provider verifies exact terminal evidence. +func (c *Client) RecoverEnvironment(ctx context.Context, req executionenv.RetireEnvironmentRequest) error { + _, err := c.rpc.RecoverEnvironment(ctx, &executionv1.RecoverEnvironmentRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), ExpectedExecutionEpoch: req.ExpectedEpoch, ExpectedPodUid: req.ExpectedPodUID, ExpectedPvcUid: req.ExpectedPVCUID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} + +func decodeError(ctx context.Context, err error) error { + if err == nil { + return nil + } + if ctx != nil && ctx.Err() != nil { + return ctx.Err() + } + st, ok := status.FromError(err) + if !ok || len(st.Details()) != 1 { + return sanitizedProviderError() + } + detail, ok := st.Details()[0].(*executionv1.ErrorDetail) + if !ok { + return sanitizedProviderError() + } + code := executionenv.ErrorCode(detail.Code) + if !code.Valid() || st.Code() != grpcCodeForError(code) || (detail.Retryable && code != executionenv.CodeNotReady && code != executionenv.CodeUnauthenticated) { + return sanitizedProviderError() + } + return &executionenv.Error{Code: code, Message: "execution provider request failed", Retryable: detail.Retryable} +} + +func sanitizedProviderError() error { + return &executionenv.Error{Code: executionenv.CodeInternal, Message: "execution provider request failed"} +} + +func grpcCodeForError(code executionenv.ErrorCode) codes.Code { + switch code { + case executionenv.CodeInvalidArgument: + return codes.InvalidArgument + case executionenv.CodeUnauthenticated: + return codes.Unauthenticated + case executionenv.CodePermissionDenied: + return codes.PermissionDenied + case executionenv.CodeNotFound: + return codes.NotFound + case executionenv.CodeAlreadyExists: + return codes.AlreadyExists + case executionenv.CodeConflict, executionenv.CodeVersionMismatch, executionenv.CodeDirectoryNotEmpty: + return codes.Aborted + case executionenv.CodeNotReady: + return codes.Unavailable + case executionenv.CodeFenceUnknown: + return codes.FailedPrecondition + case executionenv.CodeResourceExhausted: + return codes.ResourceExhausted + default: + return codes.Internal + } +} +func ownerToProto(o executionenv.Owner) *executionv1.Owner { + return &executionv1.Owner{Issuer: o.Issuer, Subject: o.Subject} +} +func refToProto(r executionenv.EnvironmentRef) *executionv1.EnvironmentRef { + return &executionv1.EnvironmentRef{Id: r.ID, Revision: r.Revision} +} +func contextToProto(v executionenv.RequestContext) *executionv1.RequestContext { + return &executionv1.RequestContext{Environment: refToProto(v.Environment), Owner: ownerToProto(v.Owner), BindingId: v.BindingID, RunId: v.RunID, ClaimId: v.ClaimID, Epoch: v.Epoch, GrantGeneration: v.GrantGeneration, Grant: v.Grant} +} +func refFromProto(r *executionv1.EnvironmentRef) executionenv.EnvironmentRef { + return executionenv.EnvironmentRef{ID: r.GetId(), Revision: r.GetRevision()} +} +func runClaimFromProto(v *executionv1.RunClaimResponse) (executionenv.RunClaim, error) { + expiry, err := checkedTime(v.GetExpiresAt()) + if err != nil || v.GetClaimId() == "" || v.GetRunId() == "" || v.GetEpoch() == 0 || v.GetGrantGeneration() == 0 || v.GetGrant() == "" { + return executionenv.RunClaim{}, sanitizedProviderError() + } + return executionenv.RunClaim{Environment: refFromProto(v.GetEnvironment()), BindingID: v.GetBindingId(), RunID: v.GetRunId(), ClaimID: v.GetClaimId(), Epoch: v.GetEpoch(), GrantGeneration: v.GetGrantGeneration(), Grant: v.GetGrant(), ExpiresAt: expiry}, nil +} + +func ensureFromProto(v *executionv1.EnsureEnvironmentResponse) (executionenv.EnsureEnvironmentResponse, error) { + if v == nil { + return executionenv.EnsureEnvironmentResponse{}, sanitizedProviderError() + } + return executionenv.EnsureEnvironmentResponse{Environment: refFromProto(v.GetEnvironment()), Epoch: v.GetEpoch(), Ready: v.GetReady(), GrantGeneration: v.GetGrantGeneration()}, nil +} +func attachFromProto(v *executionv1.AttachEnvironmentResponse) (executionenv.AttachEnvironmentResponse, error) { + if v == nil { + return executionenv.AttachEnvironmentResponse{}, sanitizedProviderError() + } + return executionenv.AttachEnvironmentResponse{Environment: refFromProto(v.GetEnvironment()), Epoch: v.GetEpoch(), Ready: v.GetReady(), GrantGeneration: v.GetGrantGeneration()}, nil +} +func checkedTime(v interface { + CheckValid() error + AsTime() time.Time +}) (time.Time, error) { + if v == nil { + return time.Time{}, errors.New("execution provider omitted timestamp") + } + if err := v.CheckValid(); err != nil { + return time.Time{}, errors.New("execution provider returned invalid timestamp") + } + return v.AsTime(), nil +} +func fileOperationToProto(op executionenv.Operation) executionv1.FileOperation { + switch op { + case executionenv.OpFileRead: + return executionv1.FileOperation_FILE_OPERATION_READ + case executionenv.OpFileResolveAuthority: + return executionv1.FileOperation_FILE_OPERATION_RESOLVE_AUTHORITY + case executionenv.OpFileStat: + return executionv1.FileOperation_FILE_OPERATION_STAT + case executionenv.OpFileCreate: + return executionv1.FileOperation_FILE_OPERATION_CREATE + case executionenv.OpFileReplace: + return executionv1.FileOperation_FILE_OPERATION_REPLACE + case executionenv.OpFileList: + return executionv1.FileOperation_FILE_OPERATION_LIST + case executionenv.OpFileRemove: + return executionv1.FileOperation_FILE_OPERATION_REMOVE + case executionenv.OpFileRename: + return executionv1.FileOperation_FILE_OPERATION_RENAME + case executionenv.OpFileCopy: + return executionv1.FileOperation_FILE_OPERATION_COPY + case executionenv.OpFileGlob: + return executionv1.FileOperation_FILE_OPERATION_GLOB + case executionenv.OpFileGrep: + return executionv1.FileOperation_FILE_OPERATION_GREP + } + return executionv1.FileOperation_FILE_OPERATION_UNSPECIFIED +} +func fileFromProto(v *executionv1.FileResponse) executionenv.FileResponse { + r := executionenv.FileResponse{Data: v.GetData(), Version: string(v.GetVersion()), Paths: v.GetPaths(), AuthorityTarget: v.GetAuthorityTarget(), AuthorityWorkspace: v.GetAuthorityWorkspace()} + if v.Info != nil { + x := fileInfoFromProto(v.Info) + r.Info = &x + } + for _, x := range v.Entries { + r.Entries = append(r.Entries, fileInfoFromProto(x)) + } + for _, x := range v.Matches { + r.Matches = append(r.Matches, executionenv.GrepMatch{Path: x.Path, Line: int(x.Line), Text: x.Text}) + } + return r +} +func fileInfoFromProto(v *executionv1.FileInfo) executionenv.FileInfo { + var mt time.Time + if v.ModTime != nil && v.ModTime.CheckValid() == nil { + mt = v.ModTime.AsTime() + } + return executionenv.FileInfo{Name: v.Name, Size: v.Size, Mode: v.Mode, ModTime: mt, IsDir: v.IsDir} +} +func commandStartFromProto(v *executionv1.CommandStartResponse) (executionenv.CommandStartResponse, error) { + if v == nil { + return executionenv.CommandStartResponse{}, sanitizedProviderError() + } + state, ok := commandStateFromProto(v.GetState()) + if !ok { + return executionenv.CommandStartResponse{}, sanitizedProviderError() + } + result, err := commandStatusFromProto(v.GetResult()) + if err != nil { + return executionenv.CommandStartResponse{}, err + } + return executionenv.CommandStartResponse{CommandID: v.GetCommandId(), State: state, Result: result}, nil +} +func commandStatusFromProto(v *executionv1.CommandStatusResponse) (executionenv.CommandStatusResponse, error) { + if v == nil { + return executionenv.CommandStatusResponse{}, sanitizedProviderError() + } + state, ok := commandStateFromProto(v.State) + if !ok { + return executionenv.CommandStatusResponse{}, sanitizedProviderError() + } + return executionenv.CommandStatusResponse{CommandID: v.CommandId, State: state, ExitCode: int(v.ExitCode), Stdout: v.Stdout, Stderr: v.Stderr, NextOffset: v.NextOffset, Truncated: v.Truncated, TerminalReceipt: v.TerminalReceipt}, nil +} +func commandStateFromProto(v executionv1.CommandState) (executionenv.CommandState, bool) { + switch v { + case executionv1.CommandState_COMMAND_STATE_RUNNING: + return executionenv.CommandRunning, true + case executionv1.CommandState_COMMAND_STATE_SUCCEEDED: + return executionenv.CommandSucceeded, true + case executionv1.CommandState_COMMAND_STATE_FAILED: + return executionenv.CommandFailed, true + case executionv1.CommandState_COMMAND_STATE_CANCELLED: + return executionenv.CommandCancelled, true + case executionv1.CommandState_COMMAND_STATE_FENCE_UNKNOWN: + return executionenv.CommandFenceUnknown, true + default: + return "", false + } +} + +// Provider adapts the execution service to server placement binding. +type Provider struct { + client *Client + profile string +} + +// NewProvider binds a client to one operator-selected profile. +func NewProvider(client *Client, profile string) (*Provider, error) { + if client == nil || profile == "" { + return nil, errors.New("execution client: client and profile are required") + } + return &Provider{client: client, profile: profile}, nil +} + +type referenceDeleteHandle struct { + client *Client + req executionenv.ReferenceRequest +} + +func (h *referenceDeleteHandle) Confirm(ctx context.Context) error { + return h.client.ConfirmReferenceDelete(ctx, h.req) +} +func (h *referenceDeleteHandle) Cancel(ctx context.Context) error { + return h.client.CancelReferenceDelete(ctx, h.req) +} +func (p *Provider) PrepareReferenceDelete(ctx context.Context, req server.PlacementSuccessorRequest) (server.ReferenceDeleteHandle, error) { + if !p.Applies(req.Ref) || req.SourceBindingID == "" { + return nil, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return nil, err + } + op := "" + exact := executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision} + intent, err := p.client.FindReferenceIntent(ctx, owner, exact, string(req.SourceBindingID)) + if err == nil { + if intent.State != executionenv.ReferencePendingDelete || intent.Environment != exact || intent.BindingID != string(req.SourceBindingID) { + return nil, server.ErrPlacementChanged + } + op = intent.OperationID + } else { + var remote *executionenv.Error + if !errors.As(err, &remote) || remote.Code != executionenv.CodeNotFound { + return nil, mapPlacementError(err) + } + } + if op == "" { + op, err = newOperationID() + if err != nil { + return nil, server.ErrPlacementUnavailable + } + } + mutation := executionenv.ReferenceRequest{Environment: executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision}, Owner: owner, BindingID: string(req.SourceBindingID), OperationID: op} + if err := p.client.PrepareReferenceDelete(ctx, mutation); err != nil { + return nil, mapPlacementError(err) + } + return &referenceDeleteHandle{client: p.client, req: mutation}, nil +} + +func (p *Provider) ReserveSuccessor(ctx context.Context, req server.PlacementSuccessorRequest) (server.PlacementBinding, error) { + if !p.Applies(req.Ref) || req.SourceBindingID == "" || req.DestinationBindingID == "" { + return server.PlacementBinding{}, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return server.PlacementBinding{}, err + } + op, err := newOperationID() + if err != nil { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + ref := executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision} + if err := p.client.ReserveSuccessor(ctx, executionenv.ReferenceRequest{Environment: ref, Owner: owner, SourceBindingID: string(req.SourceBindingID), BindingID: string(req.DestinationBindingID), OperationID: op}); err != nil { + return server.PlacementBinding{}, mapPlacementError(err) + } + attached, err := p.client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: string(req.SourceBindingID)}, Purpose: executionenv.PurposeSession}) + if err != nil { + return server.PlacementBinding{}, mapPlacementError(err) + } + base, err := p.binding(owner, string(req.DestinationBindingID), attached) + if err != nil { + return server.PlacementBinding{}, err + } + return p.withReferenceTransaction(base, owner, string(req.DestinationBindingID), op), nil +} + +func (p *Provider) Applies(ref session.EnvironmentRef) bool { + return ref.Kind == session.EnvironmentKind("kubernetes") +} + +type runHandle struct { + provider *Provider + owner executionenv.Owner + claim executionenv.RunClaimRequest + credentials *grantContext + environment tool.Environment + mu sync.Mutex +} + +func (h *runHandle) Environment() tool.Environment { return h.environment } +func (h *runHandle) RenewalDeadline() time.Time { + h.mu.Lock() + defer h.mu.Unlock() + return h.credentials.expiry() +} +func (h *runHandle) Renew(ctx context.Context) error { + h.mu.Lock() + defer h.mu.Unlock() + op, err := newOperationID() + if err != nil { + return err + } + h.claim.OperationID = op + claim, err := h.provider.client.RenewRun(ctx, h.claim) + if err != nil { + return err + } + h.claim.Epoch, h.claim.ClaimID = claim.Epoch, claim.ClaimID + h.credentials.replace(claim) + return nil +} +func (h *runHandle) Release(ctx context.Context) error { + h.mu.Lock() + defer h.mu.Unlock() + op, err := newOperationID() + if err != nil { + return err + } + h.claim.OperationID = op + return h.provider.client.ReleaseRun(ctx, h.claim) +} + +func (p *Provider) ListReferenceIntents(ctx context.Context, limit int) ([]server.ReferenceIntent, error) { + if limit <= 0 || limit > 64 { + limit = 64 + } + intents, err := p.client.ListAllReferenceIntents(ctx) + if err != nil { + return nil, mapPlacementError(err) + } + if len(intents) > limit { + intents = intents[:limit] + } + out := make([]server.ReferenceIntent, 0, len(intents)) + for _, intent := range intents { + if intent.Owner.Issuer == "" || intent.Owner.Subject == "" || intent.BindingID == "" || intent.OperationID == "" { + return nil, server.ErrPlacementUnavailable + } + out = append(out, server.ReferenceIntent{ + Ref: toSessionRef(intent.Environment), + Principal: &session.Principal{Issuer: intent.Owner.Issuer, Subject: intent.Owner.Subject}, + BindingID: session.SessionID(intent.BindingID), + SourceBindingID: session.SessionID(intent.SourceBindingID), + OperationID: intent.OperationID, + PendingDelete: intent.State == executionenv.ReferencePendingDelete, + }) + } + return out, nil +} + +func (p *Provider) referenceIntentRequest(intent server.ReferenceIntent) (executionenv.ReferenceRequest, error) { + owner, err := ownerOf(intent.Principal) + if err != nil || !p.Applies(intent.Ref) || intent.BindingID == "" || intent.OperationID == "" { + return executionenv.ReferenceRequest{}, server.ErrInvalidPlacementSelection + } + return executionenv.ReferenceRequest{Environment: executionenv.EnvironmentRef{ID: intent.Ref.ID, Revision: intent.Ref.Revision}, Owner: owner, BindingID: string(intent.BindingID), SourceBindingID: string(intent.SourceBindingID), OperationID: intent.OperationID}, nil +} + +func (p *Provider) CommitReferenceIntent(ctx context.Context, intent server.ReferenceIntent) error { + req, err := p.referenceIntentRequest(intent) + if err != nil { + return err + } + return mapPlacementError(p.client.CommitReference(ctx, req)) +} +func (p *Provider) ConfirmReferenceIntentDelete(ctx context.Context, intent server.ReferenceIntent) error { + req, err := p.referenceIntentRequest(intent) + if err != nil { + return err + } + return mapPlacementError(p.client.ConfirmReferenceDelete(ctx, req)) +} +func (p *Provider) CancelReferenceIntentDelete(ctx context.Context, intent server.ReferenceIntent) error { + req, err := p.referenceIntentRequest(intent) + if err != nil { + return err + } + return mapPlacementError(p.client.CancelReferenceDelete(ctx, req)) +} + +func (p *Provider) AcquireRun(ctx context.Context, req server.ExecutionRunRequest) (server.ExecutionRunHandle, error) { + if !p.Applies(req.Ref) || req.BindingID == "" || req.RunID == "" { + return nil, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return nil, err + } + intents, err := p.client.ListReferenceIntents(ctx, owner) + if err != nil { + return nil, mapPlacementError(err) + } + for _, intent := range intents { + if intent.Environment == (executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision}) && intent.BindingID == string(req.BindingID) && intent.State == executionenv.ReferencePendingCreate { + if err := p.client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: intent.Environment, Owner: owner, BindingID: intent.BindingID, OperationID: intent.OperationID}); err != nil { + return nil, mapPlacementError(err) + } + } + } + op, err := newOperationID() + if err != nil { + return nil, server.ErrPlacementUnavailable + } + claimReq := executionenv.RunClaimRequest{Environment: executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision}, Owner: owner, BindingID: string(req.BindingID), RunID: req.RunID, OperationID: op, TTL: executionenv.DefaultRunTTL} + claim, err := p.client.AcquireRun(ctx, claimReq) + if err != nil { + return nil, mapPlacementError(err) + } + if toSessionRef(claim.Environment) != req.Ref || claim.BindingID != string(req.BindingID) || claim.RunID != req.RunID { + return nil, server.ErrPlacementChanged + } + credentials := &grantContext{client: p.client, context: executionenv.RequestContext{Environment: claim.Environment, Owner: owner, BindingID: claim.BindingID, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant}, expiresAt: claim.ExpiresAt} + ws := &workspace{credentials: credentials} + runner := &runner{credentials: credentials} + env, envErr := tool.NewEnvironment(req.Ref, ws, memledger.New(), runner) + if envErr != nil { + return nil, server.ErrPlacementUnavailable + } + claimReq.ClaimID, claimReq.Epoch, claimReq.GrantGeneration = claim.ClaimID, claim.Epoch, claim.GrantGeneration + handle := &runHandle{provider: p, owner: owner, claim: claimReq, credentials: credentials, environment: env} + credentials.renewClaim = handle.Renew + return handle, nil +} + +// ValidatePlacement performs side-effect-free profile preflight. +func (p *Provider) ValidatePlacement(ctx context.Context) error { + _, err := p.client.ValidateProfile(ctx, p.profile) + return mapPlacementError(err) +} + +func ownerOf(principal *session.Principal) (executionenv.Owner, error) { + if principal == nil || principal.Issuer == "" || principal.Subject == "" { + return executionenv.Owner{}, server.ErrPlacementNotFound + } + return executionenv.Owner{Issuer: principal.Issuer, Subject: principal.Subject}, nil +} + +// Bind allocates the environment for the final session binding. +func (p *Provider) Bind(ctx context.Context, req server.PlacementBindRequest) (server.PlacementBinding, error) { + if !req.Selector.IsDefault() || req.BindingID == "" { + return server.PlacementBinding{}, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return server.PlacementBinding{}, err + } + operationID, err := newOperationID() + if err != nil { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + ensureDone := creatediag.Begin(ctx, "bind_ensure") + ensured, err := p.client.Ensure(ctx, string(req.BindingID), p.profile, owner, operationID) + ensureDone(err) + if err != nil { + return server.PlacementBinding{}, mapPlacementError(err) + } + if ensured.Environment.ID == "" || ensured.Environment.Revision == "" { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + return p.waitForBinding(ctx, owner, string(req.BindingID), ensured.Environment, operationID) +} + +// Reattach exactly resolves a persisted remote environment for its binding. +func (p *Provider) Reattach(ctx context.Context, req server.PlacementReattachRequest) (server.PlacementBinding, error) { + if req.BindingID == "" { + return server.PlacementBinding{}, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return server.PlacementBinding{}, err + } + ref := executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision} + attached, err := p.client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: string(req.BindingID)}, Purpose: executionenv.PurposeSession}) + if err != nil { + return server.PlacementBinding{}, mapPlacementError(err) + } + if toSessionRef(attached.Environment) != req.Ref { + return server.PlacementBinding{}, server.ErrPlacementChanged + } + return p.binding(owner, string(req.BindingID), attached) +} + +func (p *Provider) waitForBinding(ctx context.Context, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef, operationID string) (server.PlacementBinding, error) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + calls, last := 0, "begin" + creatediag.Note(ctx, "attach_poll", last, calls) + defer func() { creatediag.Note(ctx, "attach_poll_end", last, calls) }() + for { + calls++ + attached, err := p.client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}) + state := bindingPollState(ctx, attached.Ready, err) + if state != last { + last = state + creatediag.Note(ctx, "attach_poll", state, calls) + } + if err == nil && attached.Ready { + if attached.Environment != ref { + return server.PlacementBinding{}, server.ErrPlacementChanged + } + base, bindErr := p.binding(owner, binding, attached) + if bindErr != nil { + return server.PlacementBinding{}, bindErr + } + return p.withReferenceTransaction(base, owner, binding, operationID), nil + } + if err != nil { + var remote *executionenv.Error + if !errors.As(err, &remote) || (remote.Code != executionenv.CodeNotReady && !remote.Retryable) { + return server.PlacementBinding{}, mapPlacementError(err) + } + } + select { + case <-ctx.Done(): + last = bindingPollState(ctx, false, ctx.Err()) + return server.PlacementBinding{}, server.ErrPlacementUnavailable + case <-ticker.C: + } + } +} + +func bindingPollState(ctx context.Context, ready bool, err error) string { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return "deadline" + } + if ctx.Err() != nil { + return "cancelled" + } + var remote *executionenv.Error + if errors.As(err, &remote) { + if remote.Code == executionenv.CodeNotReady { + if remote.Retryable { + return "not_ready_retryable" + } + return "not_ready_nonretryable" + } + if remote.Retryable { + return "remote_retryable" + } + return "remote_terminal" + } + if err != nil { + return "transport_error" + } + if ready { + return "ready" + } + return "pending" +} + +func (p *Provider) binding(owner executionenv.Owner, binding string, attached executionenv.AttachEnvironmentResponse) (server.PlacementBinding, error) { + if !attached.Ready || attached.Environment.ID == "" || attached.Environment.Revision == "" || attached.Epoch == 0 { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + credentials := &grantContext{client: p.client, context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding, Epoch: attached.Epoch, GrantGeneration: attached.GrantGeneration}, expiresAt: time.Time{}} + ws := &workspace{credentials: credentials} + runner := &runner{credentials: credentials} + sref := toSessionRef(attached.Environment) + env, err := tool.NewEnvironment(sref, ws, memledger.New(), runner) + if err != nil { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + return server.PlacementBinding{Environment: env, Ref: sref, Metadata: server.PlacementMetadata{Kind: "kubernetes", Label: "Remote Kubernetes workspace", Revision: attached.Environment.Revision}}, nil +} +func (p *Provider) withReferenceTransaction(binding server.PlacementBinding, owner executionenv.Owner, bindingID, operationID string) server.PlacementBinding { + ref := executionenv.EnvironmentRef{ID: binding.Ref.ID, Revision: binding.Ref.Revision} + var mu sync.Mutex + finalized := false + binding.Commit = func(ctx context.Context) error { + mu.Lock() + defer mu.Unlock() + if finalized { + return nil + } + // Once dispatched, a missing response is ambiguous: the provider may have + // committed. Close must never turn that uncertainty into an abort. + finalized = true + commitDone := creatediag.Begin(ctx, "reference_commit") + err := p.client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: ref, Owner: owner, BindingID: bindingID, OperationID: operationID}) + commitDone(err) + return err + } + binding.Close = func() error { + mu.Lock() + defer mu.Unlock() + if finalized { + return nil + } + finalized = true + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return p.client.AbortReference(ctx, executionenv.ReferenceRequest{Environment: ref, Owner: owner, BindingID: bindingID, OperationID: operationID}) + } + return binding +} + +func toSessionRef(ref executionenv.EnvironmentRef) session.EnvironmentRef { + return session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: ref.ID, Revision: ref.Revision} +} +func mapPlacementError(err error) error { + if err == nil { + return nil + } + var remote *executionenv.Error + if errors.As(err, &remote) { + switch remote.Code { + case executionenv.CodeNotFound, executionenv.CodePermissionDenied, executionenv.CodeUnauthenticated: + return server.ErrPlacementNotFound + case executionenv.CodeConflict, executionenv.CodeVersionMismatch, executionenv.CodeDirectoryNotEmpty: + return server.ErrPlacementChanged + case executionenv.CodeInvalidArgument: + return server.ErrInvalidPlacementSelection + } + } + return server.ErrPlacementUnavailable +} + +type grantContext struct { + mu sync.Mutex + client *Client + context executionenv.RequestContext + expiresAt time.Time + renewClaim func(context.Context) error +} + +func (g *grantContext) current(_ context.Context) (executionenv.RequestContext, error) { + g.mu.Lock() + defer g.mu.Unlock() + if g.context.Grant == "" || !time.Now().Before(g.expiresAt) { + return executionenv.RequestContext{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "run grant unavailable", Retryable: true} + } + return g.context, nil +} + +func (g *grantContext) refresh(ctx context.Context) (executionenv.RequestContext, error) { + g.mu.Lock() + before := g.context.Grant + if before == "" || !time.Now().Before(g.expiresAt) { + g.mu.Unlock() + return executionenv.RequestContext{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "active run grant expired; retry the run", Retryable: true} + } + renew := g.renewClaim + g.mu.Unlock() + g.mu.Lock() + if g.context.Grant != before && time.Now().Before(g.expiresAt) { + out := g.context + g.mu.Unlock() + return out, nil + } + g.mu.Unlock() + if renew == nil { + return executionenv.RequestContext{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "active run grant cannot be refreshed", Retryable: true} + } + if err := renew(ctx); err != nil { + return executionenv.RequestContext{}, err + } + return g.current(ctx) +} + +func (g *grantContext) expiry() time.Time { + g.mu.Lock() + defer g.mu.Unlock() + return g.expiresAt +} + +func (g *grantContext) replace(claim executionenv.RunClaim) { + g.mu.Lock() + defer g.mu.Unlock() + g.context.RunID, g.context.ClaimID, g.context.Epoch, g.context.GrantGeneration, g.context.Grant = claim.RunID, claim.ClaimID, claim.Epoch, claim.GrantGeneration, claim.Grant + g.expiresAt = claim.ExpiresAt +} + +// workspace is bound to one provider-issued environment grant. +type workspace struct { + credentials *grantContext + opMu sync.Mutex +} + +var _ tool.AuthorityResourceResolver = (*workspace)(nil) + +func (*workspace) Root() string { return remoteRoot } + +// AuthorityResourcePath asks the authenticated executor to derive the same +// physical, confined identity that the subsequent filesystem operation uses. +func (w *workspace) AuthorityResourcePath(path string) (target, root string, err error) { + ctx, cancel := context.WithTimeout(context.Background(), authorityResolveRequestTimeout) + defer cancel() + response, err := w.call(ctx, executionenv.OpFileResolveAuthority, path, "", "", nil, "") + if err != nil { + return "", "", err + } + if response.AuthorityWorkspace != remoteRoot || (response.AuthorityTarget != remoteRoot && !strings.HasPrefix(response.AuthorityTarget, remoteRoot+"/")) { + return "", "", errors.New("execution provider returned an invalid authority resource identity") + } + return response.AuthorityTarget, response.AuthorityWorkspace, nil +} + +func (w *workspace) call(ctx context.Context, op executionenv.Operation, path, dest, pattern string, data []byte, version string) (executionenv.FileResponse, error) { + // The provider's CRD deliberately carries one active operation. The engine may + // dispatch read-only sibling tools concurrently, so serialize this bound + // workspace before claiming that provider-side operation slot. + w.opMu.Lock() + defer w.opMu.Unlock() + credentials, err := w.credentials.current(ctx) + if err != nil { + return executionenv.FileResponse{}, mapFileError(path, err) + } + limit := 0 + if op == executionenv.OpFileList || op == executionenv.OpFileGlob || op == executionenv.OpFileGrep { + limit = executionenv.MaxListEntries + } + request := executionenv.FileRequest{Context: credentials, Operation: op, Path: path, Destination: dest, Pattern: pattern, Data: data, Version: version, Limit: limit} + out, err := w.credentials.client.File(ctx, request) + if isDefinitiveAuthDenial(err) && readOnlyFileOperation(op) { + credentials, refreshErr := w.credentials.refresh(ctx) + if refreshErr != nil { + return executionenv.FileResponse{}, mapFileError(path, refreshErr) + } + request.Context = credentials + out, err = w.credentials.client.File(ctx, request) + } + return out, mapFileError(path, err) +} + +func isDefinitiveAuthDenial(err error) bool { + var remote *executionenv.Error + return errors.As(err, &remote) && remote.Code == executionenv.CodeUnauthenticated && remote.Retryable +} + +func readOnlyFileOperation(op executionenv.Operation) bool { + switch op { + case executionenv.OpFileRead, executionenv.OpFileResolveAuthority, executionenv.OpFileStat, executionenv.OpFileList, executionenv.OpFileGlob, executionenv.OpFileGrep: + return true + default: + return false + } +} +func (w *workspace) Read(ctx context.Context, path string) ([]byte, error) { + r, e := w.call(ctx, executionenv.OpFileRead, path, "", "", nil, "") + return r.Data, e +} +func (w *workspace) ReadVersion(ctx context.Context, path string) ([]byte, tool.FileVersion, error) { + r, e := w.call(ctx, executionenv.OpFileRead, path, "", "", nil, "") + if e != nil { + return nil, tool.FileVersion{}, e + } + return r.Data, tool.DecodeFileVersion(r.Version), nil +} +func (w *workspace) Stat(ctx context.Context, path string) (tool.FileInfo, error) { + r, e := w.call(ctx, executionenv.OpFileStat, path, "", "", nil, "") + if e != nil { + return tool.FileInfo{}, e + } + if r.Info == nil { + return tool.FileInfo{}, errors.New("execution provider omitted file metadata") + } + return fileInfo(*r.Info), nil +} +func (w *workspace) CreateFile(ctx context.Context, path string, data []byte) (tool.FileVersion, error) { + r, e := w.call(ctx, executionenv.OpFileCreate, path, "", "", data, "") + if e != nil { + return tool.FileVersion{}, e + } + return tool.DecodeFileVersion(r.Version), nil +} +func (w *workspace) ReplaceFile(ctx context.Context, path string, old tool.FileVersion, data []byte) (tool.FileVersion, error) { + v, e := tool.EncodeFileVersion(old) + if e != nil { + if _, statErr := w.Stat(ctx, path); statErr != nil { + return tool.FileVersion{}, statErr + } + return tool.FileVersion{}, e + } + r, e := w.call(ctx, executionenv.OpFileReplace, path, "", "", data, v) + if e != nil { + return tool.FileVersion{}, e + } + return tool.DecodeFileVersion(r.Version), nil +} +func (w *workspace) ReadDir(ctx context.Context, path string) ([]tool.FileInfo, error) { + r, e := w.call(ctx, executionenv.OpFileList, path, "", "", nil, "") + if e != nil { + return nil, e + } + out := make([]tool.FileInfo, len(r.Entries)) + for i := range r.Entries { + out[i] = fileInfo(r.Entries[i]) + } + return out, nil +} +func (w *workspace) Remove(ctx context.Context, path string) error { + _, e := w.call(ctx, executionenv.OpFileRemove, path, "", "", nil, "") + return e +} +func (w *workspace) Rename(ctx context.Context, a, b string) error { + _, e := w.call(ctx, executionenv.OpFileRename, a, b, "", nil, "") + return e +} +func (w *workspace) CopyFile(ctx context.Context, a, b string) (tool.FileVersion, error) { + r, e := w.call(ctx, executionenv.OpFileCopy, a, b, "", nil, "") + if e != nil { + return tool.FileVersion{}, e + } + return tool.DecodeFileVersion(r.Version), nil +} +func (w *workspace) Glob(ctx context.Context, pattern string) ([]string, error) { + r, e := w.call(ctx, executionenv.OpFileGlob, "", "", pattern, nil, "") + return r.Paths, e +} +func (w *workspace) Grep(ctx context.Context, pattern, pathGlob string) ([]tool.GrepMatch, error) { + r, e := w.call(ctx, executionenv.OpFileGrep, pathGlob, "", pattern, nil, "") + if e != nil { + return nil, e + } + out := make([]tool.GrepMatch, len(r.Matches)) + for i, m := range r.Matches { + out[i] = tool.GrepMatch{Path: m.Path, Line: m.Line, Text: m.Text} + } + return out, nil +} +func fileInfo(i executionenv.FileInfo) tool.FileInfo { + return tool.FileInfo{Name: i.Name, Size: i.Size, Mode: fs.FileMode(i.Mode), ModTime: i.ModTime, IsDir: i.IsDir} +} +func mapFileError(path string, err error) error { + if err == nil { + return nil + } + var remote *executionenv.Error + if errors.As(err, &remote) { + switch remote.Code { + case executionenv.CodeNotFound: + return fmt.Errorf("%s: %w", path, fs.ErrNotExist) + case executionenv.CodeAlreadyExists: + return fmt.Errorf("%s: %w", path, fs.ErrExist) + case executionenv.CodeVersionMismatch: + return &tool.VersionMismatchError{Path: path} + case executionenv.CodeDirectoryNotEmpty: + return fmt.Errorf("%s: %w", path, tool.ErrDirectoryNotEmpty) + } + } + return err +} + +// runner deliberately implements only foreground CommandRunner. +type runner struct { + credentials *grantContext +} + +func (*runner) BoundWorkspaceRoot() string { return remoteRoot } +func (r *runner) Run(ctx context.Context, command string) (tool.CommandResult, error) { + credentials, err := r.credentials.current(ctx) + if err != nil { + return tool.CommandResult{}, err + } + resp, err := r.credentials.client.StartCommand(ctx, executionenv.CommandStartRequest{Context: credentials, Command: command}) + if err != nil { + return tool.CommandResult{}, err + } + result := resp.Result + out := tool.CommandResult{Stdout: session.ToValidUTF8(string(result.Stdout)), Stderr: session.ToValidUTF8(string(result.Stderr)), ExitCode: result.ExitCode} + switch result.State { + case executionenv.CommandSucceeded, executionenv.CommandFailed: + return out, nil + case executionenv.CommandCancelled: + return out, context.Canceled + case executionenv.CommandFenceUnknown: + return out, errors.New("execution provider lost terminal command receipt; environment fencing is unknown") + default: + return out, errors.New("execution provider returned a non-terminal foreground command") + } +} diff --git a/internal/adapter/executionclient/client_test.go b/internal/adapter/executionclient/client_test.go new file mode 100644 index 0000000000..59f86ddcf5 --- /dev/null +++ b/internal/adapter/executionclient/client_test.go @@ -0,0 +1,352 @@ +//nolint:revive // Test doubles mirror the private protocol's complete method set. +package executionclient + +import ( + "context" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "math/big" + "net" + "net/url" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type integrationBackend struct { + allocation executioncontroller.Allocation + ensureCalls, attachCalls, fileCalls int + expireNextRead bool +} + +func (*integrationBackend) ValidateProfile(context.Context, string) (executioncontroller.Profile, error) { + return executioncontroller.Profile{Name: "coding", Digest: "sha256:test"}, nil +} +func (b *integrationBackend) Ensure(_ context.Context, client, owner, binding, _, _ string) (executioncontroller.Allocation, error) { + b.ensureCalls++ + b.allocation = executioncontroller.Allocation{Environment: executionenv.EnvironmentRef{ID: "env-real", Revision: "rev-1"}, Epoch: 1, OwnerHash: owner, BindingID: binding, Client: client, Ready: true} + return b.allocation, nil +} +func (b *integrationBackend) Attach(_ context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) (executioncontroller.Allocation, error) { + b.attachCalls++ + if ref != b.allocation.Environment || client != b.allocation.Client || owner != b.allocation.OwnerHash || binding != b.allocation.BindingID { + return executioncontroller.Allocation{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "not found"} + } + return b.allocation, nil +} +func (*integrationBackend) ReleaseReference(context.Context, executionenv.EnvironmentRef, string, string, string) error { + return nil +} +func (*integrationBackend) Retire(context.Context, executionenv.EnvironmentRef, string) error { + return nil +} +func (b *integrationBackend) File(_ context.Context, _, _ string, q executionenv.FileRequest) (executionenv.FileResponse, error) { + b.fileCalls++ + if b.expireNextRead && q.Operation == executionenv.OpFileRead { + b.expireNextRead = false + return executionenv.FileResponse{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "expired grant", Retryable: true} + } + if q.Operation == executionenv.OpFileResolveAuthority { + return executionenv.FileResponse{AuthorityTarget: "/workspace/main.go", AuthorityWorkspace: "/workspace"}, nil + } + return executionenv.FileResponse{Data: []byte("from-grpc"), Version: string([]byte{0xff, 0, 1})}, nil +} +func (*integrationBackend) StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + return executionenv.CommandStartResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Result: executionenv.CommandStatusResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Stdout: []byte("ok\n\xff"), Stderr: []byte("err\xe2")}}, nil +} +func (*integrationBackend) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeInternal, Message: "unimplemented"} +} +func (*integrationBackend) CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeInternal, Message: "unimplemented"} +} + +func (b *integrationBackend) EnsurePending(ctx context.Context, client, owner, binding, profile, fp, _ string) (executioncontroller.Allocation, error) { + return b.Ensure(ctx, client, owner, binding, profile, fp) +} +func (b *integrationBackend) AcquireRun(_ context.Context, ref executionenv.EnvironmentRef, client, owner, binding, run, _ string, ttl time.Duration) (executionenv.RunClaim, error) { + return executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: run, ClaimID: "claim", Epoch: b.allocation.Epoch + 1, GrantGeneration: 1, ExpiresAt: time.Now().Add(ttl)}, nil +} +func (b *integrationBackend) RenewRun(_ context.Context, _ executionenv.EnvironmentRef, _ string, _ string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + return executionenv.RunClaim{Environment: req.Environment, BindingID: req.BindingID, RunID: req.RunID, ClaimID: req.ClaimID, Epoch: req.Epoch, GrantGeneration: 1, ExpiresAt: time.Now().Add(req.TTL)}, nil +} +func (*integrationBackend) ReleaseRun(context.Context, executionenv.EnvironmentRef, string, string, executionenv.RunClaimRequest) error { + return nil +} +func (*integrationBackend) CommitReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) AbortReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) ReserveSuccessor(context.Context, executionenv.EnvironmentRef, string, string, string, string, string) error { + return nil +} +func (*integrationBackend) PrepareReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) ConfirmReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) CancelReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) { + return nil, nil +} +func (*integrationBackend) FindReferenceIntent(context.Context, executionenv.EnvironmentRef, string, string, string) (executionenv.ReferenceIntent, error) { + return executionenv.ReferenceIntent{}, nil +} + +func certificate(t *testing.T, parent *x509.Certificate, parentKey *ecdsa.PrivateKey, serverName string, client bool) (tls.Certificate, *x509.Certificate, *ecdsa.PrivateKey) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: serverName}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, IsCA: parent == nil, BasicConstraintsValid: true} + if client { + tmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + u, _ := url.Parse("spiffe://example.test/mecak8s") + tmpl.URIs = []*url.URL{u} + } else if parent != nil { + tmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + tmpl.DNSNames = []string{"example.test"} + } + issuer, signer := tmpl, key + if parent != nil { + issuer, signer = parent, parentKey + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, issuer, &key.PublicKey, signer) + if err != nil { + t.Fatal(err) + } + parsed, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: parsed}, parsed, key +} + +type grpcFixture struct { + endpoint string + clientTLS *tls.Config + stop func() +} + +func startFixture(t *testing.T, backend executioncontroller.Backend, ready func() bool) grpcFixture { + t.Helper() + _, ca, caKey := certificate(t, nil, nil, "ca", false) + serverCert, _, _ := certificate(t, ca, caKey, "example.test", false) + clientCert, _, _ := certificate(t, ca, caKey, "client", true) + _, private, _ := ed25519.GenerateKey(rand.Reader) + h := executioncontroller.NewHandler(executioncontroller.HandlerConfig{Clients: map[string]executioncontroller.ClientPolicy{"spiffe://example.test/mecak8s": {MayAttestOwner: true}}, Signer: executioncontroller.GrantSigner{KeyID: "k1", PrivateKey: private, Issuer: "provider", Audience: "executor", Lifetime: time.Minute}, Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": private.Public().(ed25519.PublicKey)}, Issuer: "provider", Audience: "executor", MaxLifetime: 2 * time.Minute}, Ready: ready}, backend) + pool := x509.NewCertPool() + pool.AddCert(ca) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + s := grpc.NewServer(grpc.Creds(credentials.NewTLS(executioncontroller.TLSConfig(serverCert, pool))), grpc.MaxRecvMsgSize(executionenv.MaxMessageBytes), grpc.MaxSendMsgSize(executionenv.MaxMessageBytes)) + executionv1.RegisterExecutionProviderServiceServer(s, h) + go func() { _ = s.Serve(ln) }() + roots := x509.NewCertPool() + roots.AddCert(ca) + return grpcFixture{endpoint: ln.Addr().String(), clientTLS: &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots, Certificates: []tls.Certificate{clientCert}, ServerName: "example.test"}, stop: func() { s.Stop(); _ = ln.Close() }} +} + +func TestProviderThroughRealGRPCSignedHandlerRefreshesAndReattachesExactly(t *testing.T) { + backend := &integrationBackend{} + fx := startFixture(t, backend, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, _ := NewProvider(client, "coding") + principal := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + binding, err := provider.Bind(context.Background(), server.PlacementBindRequest{Selector: server.DefaultPlacement(), Principal: principal, BindingID: "session-real"}) + if err != nil { + t.Fatal(err) + } + if binding.Commit != nil { + if err := binding.Commit(context.Background()); err != nil { + t.Fatal(err) + } + } + reattached, err := provider.Reattach(context.Background(), server.PlacementReattachRequest{Ref: binding.Ref, Principal: principal, BindingID: "session-real"}) + if err != nil { + t.Fatal(err) + } + handle, err := provider.AcquireRun(context.Background(), server.ExecutionRunRequest{Ref: reattached.Ref, Principal: principal, BindingID: "session-real", RunID: "run-real"}) + if err != nil { + t.Fatal(err) + } + defer handle.Release(context.Background()) + runEnv := handle.Environment() + resolver := runEnv.Workspace().(tool.AuthorityResourceResolver) + target, root, err := resolver.AuthorityResourcePath("main.go") + if err != nil || target != "/workspace/main.go" || root != "/workspace" { + t.Fatalf("authority=(%q,%q) err=%v", target, root, err) + } + data, version, err := runEnv.Workspace().ReadVersion(context.Background(), "main.go") + if err != nil || string(data) != "from-grpc" { + t.Fatalf("read=%q err=%v", data, err) + } + encoded, err := tool.EncodeFileVersion(version) + if err != nil || encoded != string([]byte{0xff, 0, 1}) { + t.Fatalf("opaque version=%q err=%v", encoded, err) + } + result, err := runEnv.CommandRunner().Run(context.Background(), "go test ./...") + if err != nil || result.Stdout != "ok\n�" || result.Stderr != "err�" { + t.Fatalf("result=%+v err=%v", result, err) + } + if backend.ensureCalls != 1 || backend.attachCalls < 2 || backend.fileCalls != 2 { + t.Fatalf("ensure=%d attach=%d file=%d", backend.ensureCalls, backend.attachCalls, backend.fileCalls) + } +} + +func TestCommandStatusIsUnimplementedOnlyAfterAuthorization(t *testing.T) { + backend := &integrationBackend{} + fx := startFixture(t, backend, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + ensured, err := client.Ensure(context.Background(), "binding", "coding", owner, "create") + if err != nil { + t.Fatal(err) + } + claim, err := client.AcquireRun(context.Background(), executionenv.RunClaimRequest{Environment: ensured.Environment, Owner: owner, BindingID: "binding", RunID: "run", OperationID: "acquire", TTL: time.Minute}) + if err != nil { + t.Fatal(err) + } + rc := executionenv.RequestContext{Environment: claim.Environment, Owner: owner, BindingID: "binding", RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant} + _, err = client.rpc.CommandStatus(context.Background(), &executionv1.CommandQueryRequest{Context: contextToProto(rc), CommandId: "c1"}) + if status.Code(err) != codes.Unimplemented { + t.Fatalf("authorized status code=%v", status.Code(err)) + } + rc.Owner.Subject = "mallory" + _, err = client.rpc.CommandStatus(context.Background(), &executionv1.CommandQueryRequest{Context: contextToProto(rc), CommandId: "c1"}) + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("wrong-owner status code=%v", status.Code(err)) + } +} + +func TestGRPCReadinessAndMTLSAreMandatory(t *testing.T) { + fx := startFixture(t, &integrationBackend{}, func() bool { return false }) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + _, err = client.ValidateProfile(context.Background(), "coding") + var remote *executionenv.Error + if !errors.As(err, &remote) || remote.Code != executionenv.CodeNotReady { + t.Fatalf("error=%v", err) + } + noCert := fx.clientTLS.Clone() + noCert.Certificates = nil + conn, err := grpc.NewClient(fx.endpoint, grpc.WithTransportCredentials(credentials.NewTLS(noCert))) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := executionv1.NewExecutionProviderServiceClient(conn).ValidateProfile(ctx, &executionv1.ValidateProfileRequest{Profile: "coding"}); err == nil { + t.Fatal("server accepted client without certificate") + } +} + +func TestDecodeErrorRejectsUntrustedOrContradictoryMetadata(t *testing.T) { + const marker = "/var/run/secrets/provider-key" + typed := func(code executionenv.ErrorCode, retry bool, grpcCode codes.Code) error { + st, err := status.New(grpcCode, marker).WithDetails(&executionv1.ErrorDetail{Code: string(code), Retryable: retry}) + if err != nil { + t.Fatal(err) + } + return st.Err() + } + multiple, err := status.New(codes.NotFound, marker).WithDetails( + &executionv1.ErrorDetail{Code: string(executionenv.CodeNotFound)}, + &executionv1.ErrorDetail{Code: string(executionenv.CodePermissionDenied)}, + ) + if err != nil { + t.Fatal(err) + } + cases := []error{ + errors.New(marker), + status.Error(codes.Internal, marker), + typed("", false, codes.Internal), + typed(executionenv.ErrorCode("future"), false, codes.Internal), + typed(executionenv.CodeNotFound, false, codes.PermissionDenied), + typed(executionenv.CodeNotFound, true, codes.NotFound), + multiple.Err(), + } + for i, input := range cases { + got := decodeError(context.Background(), input) + var remote *executionenv.Error + if !errors.As(got, &remote) || remote.Code != executionenv.CodeInternal || remote.Retryable || strings.Contains(got.Error(), marker) { + t.Fatalf("case %d escaped or misclassified: %v", i, got) + } + } +} + +func TestDecodeErrorAcceptsClosedTypedErrorAndLocalCancellation(t *testing.T) { + st, err := status.New(codes.Unauthenticated, "peer text").WithDetails(&executionv1.ErrorDetail{Code: string(executionenv.CodeUnauthenticated), Retryable: true}) + if err != nil { + t.Fatal(err) + } + var remote *executionenv.Error + if got := decodeError(context.Background(), st.Err()); !errors.As(got, &remote) || remote.Code != executionenv.CodeUnauthenticated || !remote.Retryable { + t.Fatalf("typed error=%v", got) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if got := decodeError(ctx, status.Error(codes.Canceled, "peer text")); !errors.Is(got, context.Canceled) { + t.Fatalf("local cancellation=%v", got) + } +} + +func TestCommandResponsesRejectUnknownStates(t *testing.T) { + if _, err := commandStartFromProto(&executionv1.CommandStartResponse{State: executionv1.CommandState_COMMAND_STATE_UNSPECIFIED, Result: &executionv1.CommandStatusResponse{State: executionv1.CommandState_COMMAND_STATE_SUCCEEDED}}); err == nil { + t.Fatal("accepted unspecified command-start state") + } + if _, err := commandStatusFromProto(&executionv1.CommandStatusResponse{State: executionv1.CommandState(99), TerminalReceipt: "clean"}); err == nil { + t.Fatal("accepted unknown command-status state") + } +} + +func TestEndpointRejectsHTTPAndAlternateResolvers(t *testing.T) { + cfg := &tls.Config{RootCAs: x509.NewCertPool(), Certificates: []tls.Certificate{{Certificate: [][]byte{{1}}}}} + for _, endpoint := range []string{"https://provider:8443", "dns:///provider:8443", "provider"} { + if c, err := New(endpoint, cfg); err == nil { + c.Close() + t.Fatalf("accepted %q", endpoint) + } + } +} diff --git a/internal/adapter/executionclient/create_diagnostics_test.go b/internal/adapter/executionclient/create_diagnostics_test.go new file mode 100644 index 0000000000..f6bacc775a --- /dev/null +++ b/internal/adapter/executionclient/create_diagnostics_test.go @@ -0,0 +1,153 @@ +package executionclient + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/adapter/slogdiag" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type pendingCreateBackend struct { + integrationBackend + calls atomic.Int64 + waiting chan struct{} + release chan struct{} + retryable bool +} + +func (b *pendingCreateBackend) Attach(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) (executioncontroller.Allocation, error) { + if b.calls.Add(1) <= 3 { + return executioncontroller.Allocation{}, &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: b.retryable, Message: "PRIVATE-ERROR-SENTINEL"} + } + close(b.waiting) + select { + case <-b.release: + return b.integrationBackend.Attach(ctx, ref, client, owner, binding) + case <-ctx.Done(): + return executioncontroller.Allocation{}, ctx.Err() + } +} + +func TestRemoteCreateAttachPollingDiagnosticsAndCancellation(t *testing.T) { + for _, tc := range []struct { + name string + retryable, cancel bool + }{ + {"ready", true, false}, {"not-ready-nonretryable", false, false}, {"cancel", true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + backend := &pendingCreateBackend{waiting: make(chan struct{}), release: make(chan struct{}), retryable: tc.retryable} + fx := startFixture(t, backend, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + var logs bytes.Buffer + store := memstore.New() + engine := agent.NewEngine(agent.Deps{LLM: mockllm.New(), Catalog: tool.NewCatalog(), Model: "offline"}) + svc, err := server.NewService(server.Config{ + Engine: engine, Store: store, PlacementProvider: provider, ExecutionAccess: provider, + PlacementScope: "fixture", SharedEngineRoot: "/workspace", NewID: func() session.SessionID { return "fixed" }, + Diagnostics: slogdiag.New(&logs, true, port.LevelDebug), + }) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + ctx = session.WithPrincipal(ctx, &session.Principal{Issuer: "PRIVATE-ISSUER", Subject: "PRIVATE-OWNER", GrantType: session.GrantTypeUser}) + req := httptest.NewRequest(http.MethodPost, "/v1/sessions", strings.NewReader(`{}`)).WithContext(ctx) + response := httptest.NewRecorder() + done := make(chan struct{}) + go func() { server.NewHTTPHandler(svc).ServeHTTP(response, req); close(done) }() + select { + case <-backend.waiting: + case <-ctx.Done(): + t.Fatal("Attach polling did not reach gated call") + } + if tc.cancel { + cancel() + } else { + close(backend.release) + } + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("create did not return after release/cancel") + } + _, loadErr := store.Load(t.Context(), "fixed") + if tc.cancel { + if !errors.Is(loadErr, port.ErrSessionNotFound) || response.Code == http.StatusCreated { + t.Fatal("cancelled create persisted") + } + } else if loadErr != nil || response.Code != http.StatusCreated { + t.Fatalf("ready create status=%d error=%v", response.Code, loadErr) + } + if strings.Contains(logs.String(), "PRIVATE") || strings.Contains(logs.String(), "env-real") || strings.Contains(logs.String(), "/workspace") { + t.Fatal("private data leaked") + } + reasons := map[string]int{} + stages := map[string]bool{} + var finalCalls float64 + for _, line := range strings.Split(strings.TrimSpace(logs.String()), "\n") { + var row map[string]any + if err := json.Unmarshal([]byte(line), &row); err != nil { + t.Fatal(err) + } + if row["msg"] != "remote create stage" { + continue + } + stage, _ := row["stage"].(string) + stages[stage] = true + if stage == "attach_poll" { + reasons[row["reason"].(string)]++ + } + if stage == "attach_poll_end" { + finalCalls = row["calls"].(float64) + } + } + reason := "not_ready_retryable" + if !tc.retryable { + reason = "not_ready_nonretryable" + } + if reasons[reason] != 1 || finalCalls != 4 { + t.Fatalf("polling not bounded/discriminating: reasons=%v calls=%v", reasons, finalCalls) + } + for _, stage := range []string{"http_handler", "session_id_probe", "bind_ensure", "attach_poll", "attach_poll_end"} { + if !stages[stage] { + t.Errorf("missing stage %s", stage) + } + } + if !tc.cancel && (!stages["session_persist"] || !stages["reference_commit"] || !stages["http_response"]) { + t.Fatal("missing persistence/commit/response stages") + } + if tc.cancel && (stages["session_persist"] || stages["reference_commit"]) { + t.Fatal("cancel reached publication") + } + }) + } +} diff --git a/internal/adapter/executionclient/files_conformance_test.go b/internal/adapter/executionclient/files_conformance_test.go new file mode 100644 index 0000000000..ebf9985ef8 --- /dev/null +++ b/internal/adapter/executionclient/files_conformance_test.go @@ -0,0 +1,124 @@ +package executionclient + +import ( + "context" + "encoding/json" + "errors" + "io/fs" + "strings" + "testing" + + "github.com/stacklok/mecatl/engine/adapter/fsconformance" + "github.com/stacklok/mecatl/engine/adapter/fstools" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" + "github.com/stacklok/mecatl/internal/executionexecutor" +) + +type executorBackend struct { + integrationBackend + executor *executionexecutor.Executor +} + +func (b *executorBackend) File(ctx context.Context, _, _ string, q executionenv.FileRequest) (executionenv.FileResponse, error) { + r, err := b.executor.Execute(ctx, executionenv.ExecutorRequest{Operation: q.Operation, Path: q.Path, Destination: q.Destination, Pattern: q.Pattern, Data: q.Data, Version: q.Version, Limit: q.Limit}) + return r.FileResponse, err +} + +func remoteExecutorEnvironment(t *testing.T) tool.Environment { + t.Helper() + x, err := executionexecutor.New(t.TempDir(), executionexecutor.Limits{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = x.Close() }) + fx := startFixture(t, &executorBackend{executor: x}, nil) + t.Cleanup(fx.stop) + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + t.Cleanup(client.Close) + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + principal := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + binding, err := provider.Bind(t.Context(), server.PlacementBindRequest{Selector: server.DefaultPlacement(), Principal: principal, BindingID: "files"}) + if err != nil { + t.Fatal(err) + } + if err := binding.Commit(t.Context()); err != nil { + t.Fatal(err) + } + handle, err := provider.AcquireRun(t.Context(), server.ExecutionRunRequest{Ref: binding.Ref, Principal: principal, BindingID: "files", RunID: "run-files"}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = handle.Release(context.Background()) }) + return handle.Environment() +} + +func TestRemoteWorkspaceConformanceThroughSignedGRPC(t *testing.T) { + factory := func(t *testing.T) tool.Workspace { return remoteExecutorEnvironment(t).Workspace() } + fsconformance.Run(t, factory) + fsconformance.RunNamespace(t, factory) +} + +func TestRemoteFileToolsPreserveLedgerAndNamespaceContracts(t *testing.T) { + env := remoteExecutorEnvironment(t) + ctx := t.Context() + call := func(body tool.Tool, args string, wantError bool, contains string) { + t.Helper() + result, err := body.Execute(ctx, session.ToolCall{ID: "file", Name: body.Spec().Name, Args: json.RawMessage(args)}, env) + if err != nil || result.IsError != wantError || !strings.Contains(result.Content, contains) { + t.Fatalf("%s: result=%+v error=%v", body.Spec().Name, result, err) + } + } + if _, err := env.Workspace().CreateFile(ctx, "main.go", []byte("package main\n// before\n")); err != nil { + t.Fatal(err) + } + call(fstools.EditTool{}, `{"path":"main.go","old_string":"before","new_string":"after"}`, true, "") + call(fstools.WriteTool{}, `{"path":"main.go","content":"clobber"}`, true, "") + call(fstools.ReadTool{}, `{"path":"main.go"}`, false, "before") + call(fstools.EditTool{}, `{"path":"main.go","old_string":"before","new_string":"after"}`, false, "") + call(fstools.ReadTool{}, `{"path":"main.go"}`, false, "after") + // A mutation outside the read ledger invalidates both Edit and existing Write. + _, version, err := env.Workspace().ReadVersion(ctx, "main.go") + if err != nil { + t.Fatal(err) + } + if _, err := env.Workspace().ReplaceFile(ctx, "main.go", version, []byte("package main\n// concurrent\n")); err != nil { + t.Fatal(err) + } + call(fstools.EditTool{}, `{"path":"main.go","old_string":"concurrent","new_string":"lost"}`, true, "") + call(fstools.WriteTool{}, `{"path":"main.go","content":"lost"}`, true, "") + call(fstools.ReadTool{}, `{"path":"main.go"}`, false, "concurrent") + call(fstools.WriteTool{}, `{"path":"main.go","content":"package main\n// final\n"}`, false, "") + call(fstools.ListDirTool{}, `{"path":"."}`, false, "main.go") + call(fstools.GrepTool{}, `{"path":"*.go","pattern":"final"}`, false, "final") + call(fstools.GlobTool{}, `{"pattern":"**/*.go"}`, false, "main.go") + call(fstools.WriteTool{}, `{"path":"fresh.go","content":"package main\n"}`, false, "") + // Seed without Write's automatic ledger update: namespace operations need no ledger. + if _, err := env.Workspace().CreateFile(ctx, "unread.go", []byte("package unread\n")); err != nil { + t.Fatal(err) + } + call(fstools.CopyTool{}, `{"source":"unread.go","destination":"copied.go"}`, false, "") + call(fstools.MoveTool{}, `{"source":"copied.go","destination":"moved.go"}`, false, "") + data, err := env.Workspace().Read(ctx, "moved.go") + if err != nil || string(data) != "package unread\n" { + t.Fatalf("moved=%q error=%v", data, err) + } + call(fstools.CopyTool{}, `{"source":"unread.go","destination":"main.go"}`, true, "") + call(fstools.MoveTool{}, `{"source":"moved.go","destination":"main.go"}`, true, "") + call(fstools.RemoveTool{}, `{"path":"moved.go"}`, false, "") + if _, err := env.Workspace().Stat(ctx, "moved.go"); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("removed file: %v", err) + } + data, err = env.Workspace().Read(ctx, "main.go") + if err != nil || string(data) != "package main\n// final\n" { + t.Fatalf("final=%q error=%v", data, err) + } +} diff --git a/internal/adapter/executionclient/reference_ambiguity_test.go b/internal/adapter/executionclient/reference_ambiguity_test.go new file mode 100644 index 0000000000..0e8929e769 --- /dev/null +++ b/internal/adapter/executionclient/reference_ambiguity_test.go @@ -0,0 +1,60 @@ +package executionclient + +import ( + "context" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type droppedCommitRPC struct { + executionv1.ExecutionProviderServiceClient + commits int + aborts int +} + +func (r *droppedCommitRPC) CommitReference(context.Context, *executionv1.ReferenceMutationRequest, ...grpc.CallOption) (*emptypb.Empty, error) { + r.commits++ + return nil, status.Error(codes.Unavailable, "response lost") +} +func (r *droppedCommitRPC) AbortReference(context.Context, *executionv1.ReferenceMutationRequest, ...grpc.CallOption) (*emptypb.Empty, error) { + r.aborts++ + return &emptypb.Empty{}, nil +} + +func TestDefinitePreCommitFailureAbortsReservation(t *testing.T) { + rpc := &droppedCommitRPC{} + provider := &Provider{client: &Client{rpc: rpc}, profile: "coding"} + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + binding := provider.withReferenceTransaction(server.PlacementBinding{Ref: ref}, executionenv.Owner{Issuer: "issuer", Subject: "alice"}, "session", "operation") + if err := binding.Close(); err != nil { + t.Fatal(err) + } + if rpc.commits != 0 || rpc.aborts != 1 { + t.Fatalf("commits=%d aborts=%d", rpc.commits, rpc.aborts) + } +} + +func TestAmbiguousCommitIsRetainedAndNeverAbortedByCleanup(t *testing.T) { + rpc := &droppedCommitRPC{} + provider := &Provider{client: &Client{rpc: rpc}, profile: "coding"} + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + binding := provider.withReferenceTransaction(server.PlacementBinding{Ref: ref}, executionenv.Owner{Issuer: "issuer", Subject: "alice"}, "session", "operation") + if err := binding.Commit(t.Context()); err == nil { + t.Fatal("commit transport ambiguity was hidden") + } + if err := binding.Close(); err != nil { + t.Fatal(err) + } + if rpc.commits != 1 || rpc.aborts != 0 { + t.Fatalf("commits=%d aborts=%d", rpc.commits, rpc.aborts) + } +} diff --git a/internal/adapter/executionclient/service_integration_test.go b/internal/adapter/executionclient/service_integration_test.go new file mode 100644 index 0000000000..3e2ba1b0ce --- /dev/null +++ b/internal/adapter/executionclient/service_integration_test.go @@ -0,0 +1,209 @@ +package executionclient + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" + "unicode/utf8" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/adapter/permpolicy" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/adapter/server" + adaptertools "github.com/stacklok/mecatl/internal/adapter/tools" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type recordingExecutor struct{ calls atomic.Int64 } + +func (e *recordingExecutor) Execute(_ context.Context, _ string, req executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + e.calls.Add(1) + switch req.Operation { + case executionenv.OpFileResolveAuthority: + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{AuthorityTarget: "/workspace/main.go", AuthorityWorkspace: "/workspace"}}, nil + case executionenv.OpFileRead: + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Data: []byte("package main\n"), Version: "v1"}}, nil + case executionenv.OpCommandStart: + return executionenv.ExecutorResponse{Command: &executionenv.CommandStatusResponse{CommandID: req.CommandID, State: executionenv.CommandSucceeded, Stdout: []byte("out\xff"), Stderr: []byte("err\xe2"), TerminalReceipt: "complete"}}, nil + default: + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "unsupported test operation"} + } +} + +func TestServiceUsesRealMTLSProviderStoreAndReleasesOnlyAfterDrain(t *testing.T) { + profilePath := filepath.Join(t.TempDir(), "profiles.yaml") + profileYAML := []byte("profiles:\n coding:\n image: example.test/executor@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n storageClass: standard\n storageSize: 1Gi\n cpuRequest: 100m\n memoryRequest: 128Mi\n cpuLimit: 1\n memoryLimit: 1Gi\n ephemeralStorageRequest: 256Mi\n ephemeralStorageLimit: 1Gi\n tmpSizeLimit: 128Mi\n runtimeClassName: gvisor\n maxFileBytes: 5242880\n maxCommandBytes: 1048576\n maxCommandDuration: 1m\n maxEnvironments: 10\n") + if err := os.WriteFile(profilePath, profileYAML, 0o600); err != nil { + t.Fatal(err) + } + profiles, err := executioncontroller.LoadProfiles(profilePath) + if err != nil { + t.Fatal(err) + } + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{executioncontroller.ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + executor := &recordingExecutor{} + controllerStore := executioncontroller.NewStore(dyn, "ns", profiles, executor) + profile, err := controllerStore.ValidateProfile(t.Context(), "coding") + if err != nil { + t.Fatal(err) + } + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + ownerSum := sha256.Sum256([]byte(owner.Issuer + "\x00" + owner.Subject)) + clientSum := sha256.Sum256([]byte("spiffe://example.test/mecak8s")) + now := time.Now().UTC() + envObj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env-real", "namespace": "ns"}, + "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev-1", "ownerHash": hex.EncodeToString(ownerSum[:]), "ownerIssuer": owner.Issuer, "ownerSubject": owner.Subject, "clientHash": hex.EncodeToString(clientSum[:]), "bindingID": "service-session", "profile": "coding", "profileDigest": profile.Digest, "desired": "Active"}, + "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "pod": map[string]any{"name": "executor-pod"}, "references": []any{map[string]any{"bindingID": "service-session", "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}, + }} + if _, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Create(t.Context(), envObj, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + fx := startFixture(t, controllerStore, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + catalog := tool.NewCatalog() + catalog.MustRegister(adaptertools.ReadTool{}) + catalog.MustRegister(agent.NewShellTool()) + var modelShellResult string + llm := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { + for _, message := range req.Messages { + if result := message.ToolResult; result != nil && result.CallID == "shell" { + modelShellResult = result.Content + } + } + })}, + mockllm.ToolCallTurn(session.NewToolCall("read", "Read", json.RawMessage(`{"path":"main.go"}`))), + mockllm.ToolCallTurn(session.NewToolCall("background", "Shell", json.RawMessage(`{"command":"touch background-leak","background":true}`))), + mockllm.ToolCallTurn(session.NewToolCall("shell", "Shell", json.RawMessage(`{"command":"true"}`))), + mockllm.TextTurn("done"), + mockllm.TextTurn("continued"), + ) + engine := agent.NewEngine(agent.Deps{LLM: llm, Catalog: catalog, Policy: permpolicy.NewPolicy(permpolicy.AllowAllFloorRules(), nil), Model: "test"}) + sessions := memstore.New() + svc, err := server.NewService(server.Config{Engine: engine, Store: sessions, PlacementProvider: provider, PlacementScope: "test", ExecutionAccess: provider, SharedEngineRoot: "/workspace", DefaultLimits: session.Limits{MaxTurns: 5}}) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + principal := &session.Principal{Issuer: owner.Issuer, Subject: owner.Subject, GrantType: session.GrantTypeUser} + sess := session.New("service-session", session.ModeDefault, session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env-real", Revision: "rev-1"}, session.Limits{MaxTurns: 5}, now) + if err := sess.RestoreLabels(principal, session.Authority{}); err != nil { + t.Fatal(err) + } + if err := sessions.Save(t.Context(), sess); err != nil { + t.Fatal(err) + } + ctx := session.WithPrincipal(t.Context(), principal) + run, err := svc.StartRunContent(ctx, sess.ID, "inspect", nil) + if err != nil { + t.Fatal(err) + } + backgroundRejected := false + var streamedShellResult string + for event := range run.Events() { + if result := event.ToolResult; result != nil && result.CallID == "shell" { + streamedShellResult = result.Content + } + if result := event.ToolResult; result != nil && result.CallID == "background" { + backgroundRejected = result.IsError && strings.Contains(result.Content, "not supported by this command runner") + } + } + if !backgroundRejected { + t.Fatal("remote background Shell did not return the named capability error") + } + if !utf8.ValidString(streamedShellResult) || !strings.Contains(streamedShellResult, "out�") || !strings.Contains(streamedShellResult, "err�") || modelShellResult != streamedShellResult { + t.Fatalf("private protobuf bytes were not repaired identically for client/model: stream=%q model=%q", streamedShellResult, modelShellResult) + } + if executor.calls.Load() != 3 { + t.Fatalf("executor calls=%d, want authority+read+shell", executor.calls.Load()) + } + if _, err := svc.StartRunContent(ctx, sess.ID, "early", nil); err == nil { + t.Fatal("replacement run acquired before relay release") + } + svc.FinishRun(sess.ID, run) + stored, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env-real", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if _, found, err := unstructured.NestedMap(stored.Object, "status", "activeRun"); err != nil || found { + t.Fatalf("native claim remained after actual drain release: found=%t err=%v", found, err) + } + continued, err := svc.StartRunContent(ctx, sess.ID, "continue", nil) + if err != nil { + t.Fatal(err) + } + for range continued.Events() { + } + svc.Persist(ctx, sess.ID) + svc.FinishRun(sess.ID, continued) + + stored, err = dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, "env-real", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + before, err := sessions.Load(ctx, sess.ID) + if err != nil || len(before.Conversation.Messages) == 0 { + t.Fatalf("source history missing: %v", err) + } + placement := server.SuccessorPlacement{Selector: "unsupported-worktree", SelectorPresent: true} + // A well-formed selector and binding ID reach the native provider's unsupported + // selection gate, not the Service's malformed-request validation. + if _, err := provider.Bind(ctx, server.PlacementBindRequest{Selector: server.SelectWorktree(sess.ID, sess.EnvironmentRef, placement.Selector), BindingID: "unused-destination", Principal: principal}); !errors.Is(err, server.ErrInvalidPlacementSelection) { + t.Fatalf("native selector gate: %v", err) + } + for _, operation := range []string{"clear", "fork"} { + t.Run(operation+"-unsupported-selector", func(t *testing.T) { + var destination session.SessionID + var err error + if operation == "clear" { + destination, err = svc.ClearSessionSuccessor(ctx, sess.ID, placement) + } else { + destination, err = svc.ForkSessionSuccessor(ctx, server.ForkSuccessorRequest{Source: sess.ID, Placement: placement}) + } + if !errors.Is(err, server.ErrInvalidPlacementSelection) || destination != "" { + t.Fatalf("destination=%q error=%v", destination, err) + } + after, err := sessions.Load(ctx, sess.ID) + if err != nil || after.EnvironmentRef != before.EnvironmentRef || !reflect.DeepEqual(after.Conversation, before.Conversation) { + t.Fatalf("unsupported successor changed source ref/history: %v", err) + } + all, err := svc.ListSessions(ctx) + if err != nil || len(all) != 1 || all[0].SessionID != string(sess.ID) { + t.Fatalf("unsupported successor published a destination: %+v, %v", all, err) + } + current, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, "env-real", metav1.GetOptions{}) + if err != nil || !reflect.DeepEqual(current.Object, stored.Object) { + t.Fatalf("unsupported successor mutated provider references: %v", err) + } + }) + } +} diff --git a/internal/adapter/executioncontroller/admin_lifecycle.go b/internal/adapter/executioncontroller/admin_lifecycle.go new file mode 100644 index 0000000000..346b3035da --- /dev/null +++ b/internal/adapter/executioncontroller/admin_lifecycle.go @@ -0,0 +1,457 @@ +package executioncontroller + +import ( + "context" + "math" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +type adminLifecycleRequest struct { + Environment executionenv.EnvironmentRef + OwnerHash string + Client string + AdministratorFor []string + ExpectedEpoch uint64 + ExpectedPodUID string + ExpectedPVCUID string + OperationID string + ExpectedSchema int64 +} + +// ReplaceExecutor starts an exact, crash-recoverable executor replacement. +func (s *Store) ReplaceExecutor(ctx context.Context, q adminLifecycleRequest) error { + return s.startLifecycle(ctx, q, "ReplaceExecutor", false) +} + +// RetireExact starts exact executor retirement while retaining its workspace. +func (s *Store) RetireExact(ctx context.Context, q adminLifecycleRequest) error { + return s.startLifecycle(ctx, q, "RetireEnvironment", true) +} + +func (s *Store) startLifecycle(ctx context.Context, q adminLifecycleRequest, kind string, requireNoRefs bool) error { //nolint:gocyclo // Exact admission keeps all identity and receipt checks together. + if q.ExpectedEpoch == 0 || q.ExpectedEpoch > math.MaxInt64 || q.ExpectedPodUID == "" || q.ExpectedPVCUID == "" || q.OperationID == "" { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "exact lifecycle identity is required"} + } + if err := s.persistObservedTerminalProof(ctx, q); err != nil { + return err + } + return s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { + if kind == "ReplaceExecutor" && replacementReceiptMatches(o, q) { + return nil + } + if err := exactAdminSubject(o, q); err != nil { + return err + } + if kind == "RetireEnvironment" && conditionTrue(o, "Retired") { + if exactTerminationProofOperationMatches(o, q) { + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is already retired by another operation"} + } + if existing, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation"); found { + if text(existing, "id") == q.OperationID && text(existing, "type") == kind && text(existing, "expectedPodUID") == q.ExpectedPodUID && text(existing, "expectedPVCUID") == q.ExpectedPVCUID && intNested(existing, "expectedEpoch") == int64(q.ExpectedEpoch) { //nolint:gosec // validated above. + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "another lifecycle operation is active"} + } + if lifecycleAdmissionBlocked(o, q) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not healthy and idle"} + } + refs, err := referenceRecords(o) + if err != nil { + return err + } + if requireNoRefs && len(refs) != 0 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment still has references"} + } + operation := map[string]any{"id": q.OperationID, "type": kind, "phase": "Quiescing", "expectedEpoch": int64(q.ExpectedEpoch), "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID, "createdAt": s.now().Format(time.RFC3339Nano)} //nolint:gosec // validated above. + setConditionObject(o, "Ready", false, "Quiescing", "administrator lifecycle operation is quiescing the executor") + return unstructured.SetNestedMap(o.Object, operation, "status", "lifecycleOperation") + }) +} + +func lifecycleAdmissionBlocked(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + return textNested(o.Object, "status", "activeRun", "claimID") != "" || + textNested(o.Object, "status", "activeOperation", "id") != "" || + textNested(o.Object, "status", "fenceState") != fenceHealthy || + !conditionTrue(o, "Ready") && !exactTerminationProofMatches(o, q) +} + +func replacementReceiptMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + return textNested(o.Object, "status", "lastReplacement", operationIDField) == q.OperationID && + textNested(o.Object, "status", "lastReplacement", "previousPodUID") == q.ExpectedPodUID && + textNested(o.Object, "status", "lastReplacement", "pvcUID") == q.ExpectedPVCUID && + intNested(o.Object, "status", "lastReplacement", "previousEpoch") == int64(q.ExpectedEpoch) //nolint:gosec // q.ExpectedEpoch is bounded by startLifecycle. +} + +func exactTerminationProofMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + return textNested(o.Object, "status", "terminationProof", "podUID") == q.ExpectedPodUID && + textNested(o.Object, "status", "terminationProof", "pvcUID") == q.ExpectedPVCUID && + intNested(o.Object, "status", "terminationProof", "epoch") == int64(q.ExpectedEpoch) //nolint:gosec // q.ExpectedEpoch is bounded above. +} + +func exactTerminationProofOperationMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + return exactTerminationProofMatches(o, q) && textNested(o.Object, "status", "terminationProof", operationIDField) == q.OperationID +} + +func (s *Store) persistObservedTerminalProof(ctx context.Context, q adminLifecycleRequest) error { //nolint:gocyclo // Exact terminal proof deliberately fails closed at every observable mismatch. + o, err := s.resources.Get(ctx, q.Environment.ID, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if err != nil { + return err + } + if err := adminSubject(o, q); err != nil { + return err + } + if err := requireCurrentSchema(o); err != nil { + return err + } + if replacementReceiptMatches(o, q) { + return nil + } + if err := exactAdminSubject(o, q); err != nil { + return err + } + if conditionTrue(o, "Ready") || conditionTrue(o, "Retired") || exactTerminationProofOperationMatches(o, q) { + return nil + } + if textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "activeOperation", "id") != "" || textNested(o.Object, "status", "fenceState") != fenceHealthy || s.kube == nil { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not healthy and idle"} + } + podName, pvcName := textNested(o.Object, "status", "pod", "name"), textNested(o.Object, "status", "pvc", "name") + pod, err := s.kube.CoreV1().Pods(s.namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil || string(pod.UID) != q.ExpectedPodUID || !podTerminal(pod) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "exact terminal executor proof is unavailable"} + } + pvc, err := s.kube.CoreV1().PersistentVolumeClaims(s.namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil || string(pvc.UID) != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "exact workspace identity is unavailable"} + } + return s.retryAdminStatus(ctx, q, func(current *unstructured.Unstructured) error { + if err := exactAdminSubject(current, q); err != nil { + return err + } + if conditionTrue(current, "Ready") || exactTerminationProofOperationMatches(current, q) { + return nil + } + if textNested(current.Object, "status", "activeRun", "claimID") != "" || textNested(current.Object, "status", "activeOperation", "id") != "" || textNested(current.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not healthy and idle"} + } + return unstructured.SetNestedMap(current.Object, terminationProof(q, pod), "status", "terminationProof") + }) +} + +// adminSubject uses only the policy captured by authentication for this request. +// The actor remains distinct from the immutable creator, including on receipt replay. +func adminSubject(o *unstructured.Unstructured, q adminLifecycleRequest) error { + creator := textNested(o.Object, "spec", "clientHash") + allowed := creator == hashText(q.Client) + for _, uri := range q.AdministratorFor { + allowed = allowed || creator == hashText(uri) + } + if !allowed || textNested(o.Object, "spec", "ownerHash") != q.OwnerHash || textNested(o.Object, "spec", "revision") != q.Environment.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + return nil +} + +func (s *Store) retryAdminStatus(ctx context.Context, q adminLifecycleRequest, mutate func(*unstructured.Unstructured) error) error { + return s.retryUpdateStatusRaw(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if err := adminSubject(o, q); err != nil { + return err + } + if err := requireCurrentSchema(o); err != nil { + return err + } + return mutate(o) + }) +} + +func exactAdminSubject(o *unstructured.Unstructured, q adminLifecycleRequest) error { + if err := adminSubject(o, q); err != nil { + return err + } + if !epochMatches(o, q.ExpectedEpoch) || textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment execution identity changed"} + } + return nil +} + +// RecoverEnvironment clears uncertainty only after built-in exact terminal verification. +func (s *Store) RecoverEnvironment(ctx context.Context, q adminLifecycleRequest) error { + if s.kube == nil || q.ExpectedPodUID == "" || q.ExpectedPVCUID == "" || q.OperationID == "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "built-in recovery verifier is unavailable"} + } + podName, pvcName := "", "" + if err := s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { + if err := exactAdminSubject(o, q); err != nil { + return err + } + if exactTerminationProofOperationMatches(o, q) { + return nil + } + if textNested(o.Object, "status", "fenceState") != "FenceUnknown" { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not fenced"} + } + podName, pvcName = textNested(o.Object, "status", "pod", "name"), textNested(o.Object, "status", "pvc", "name") + return nil + }); err != nil { + return err + } + if podName == "" { + return nil + } + pod, err := s.kube.CoreV1().Pods(s.namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil || string(pod.UID) != q.ExpectedPodUID || !podTerminal(pod) { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "terminal executor proof unavailable; use the external-fencing operator runbook"} + } + pvc, err := s.kube.CoreV1().PersistentVolumeClaims(s.namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil || string(pvc.UID) != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "workspace identity cannot be verified"} + } + return s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { + if err := exactAdminSubject(o, q); err != nil { + return err + } + if exactTerminationProofOperationMatches(o, q) { + return nil + } + proof := terminationProof(q, pod) + if err := unstructured.SetNestedMap(o.Object, proof, "status", "terminationProof"); err != nil { + return err + } + unstructured.RemoveNestedField(o.Object, "status", "activeOperation") + unstructured.RemoveNestedField(o.Object, "status", "activeRun") + return unstructured.SetNestedField(o.Object, fenceHealthy, "status", "fenceState") + }) +} + +func terminationProof(q adminLifecycleRequest, pod *corev1.Pod) map[string]any { + return map[string]any{operationIDField: q.OperationID, "podUID": q.ExpectedPodUID, "pvcUID": q.ExpectedPVCUID, "epoch": int64(q.ExpectedEpoch), "podPhase": string(pod.Status.Phase), "observedAt": time.Now().UTC().Format(time.RFC3339Nano)} //nolint:gosec // epoch is validated by caller. +} + +func podTerminal(pod *corev1.Pod) bool { + if pod == nil || pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed || len(pod.Spec.Containers) == 0 { + return false + } + if !allDeclaredContainersTerminated(pod.Spec.InitContainers, pod.Status.InitContainerStatuses) || !allDeclaredContainersTerminated(pod.Spec.Containers, pod.Status.ContainerStatuses) { + return false + } + return allDeclaredContainersTerminated(pod.Spec.EphemeralContainers, pod.Status.EphemeralContainerStatuses) +} + +func allDeclaredContainersTerminated[T interface { + corev1.Container | corev1.EphemeralContainer +}](declared []T, statuses []corev1.ContainerStatus) bool { + if len(statuses) != len(declared) { + return false + } + names := make(map[string]struct{}, len(declared)) + for _, container := range declared { + var name string + switch c := any(container).(type) { + case corev1.Container: + name = c.Name + case corev1.EphemeralContainer: + name = c.Name + } + if name == "" { + return false + } + names[name] = struct{}{} + } + if len(names) != len(declared) { + return false + } + seen := make(map[string]struct{}, len(statuses)) + for _, status := range statuses { + if _, declared := names[status.Name]; !declared || status.State.Terminated == nil { + return false + } + if _, duplicate := seen[status.Name]; duplicate { + return false + } + seen[status.Name] = struct{}{} + } + return len(seen) == len(names) +} + +// DeleteRetiredEnvironment starts exact deletion of an explicitly retained workspace. +func (s *Store) DeleteRetiredEnvironment(ctx context.Context, q adminLifecycleRequest) error { + if q.ExpectedPVCUID == "" || q.OperationID == "" { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "exact retained workspace identity is required"} + } + return s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "retained workspace identity changed"} + } + if existing, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation"); found { + if text(existing, "id") == q.OperationID && text(existing, "type") == deleteRetiredEnvironment { + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "another lifecycle operation is active"} + } + refs, err := referenceRecords(o) + if err != nil || len(refs) != 0 || textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "activeOperation", "id") != "" || !conditionTrue(o, "Retired") || !conditionTrue(o, "ExecutorTerminated") { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not safely retained and unreferenced"} + } + return unstructured.SetNestedMap(o.Object, map[string]any{"id": q.OperationID, "type": deleteRetiredEnvironment, "phase": "DeletingPVC", "expectedPVCUID": q.ExpectedPVCUID, "createdAt": s.now().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + }) +} + +// MigrateEnvironment explicitly upgrades a recognized, verified prototype schema. +func (s *Store) MigrateEnvironment(ctx context.Context, q adminLifecycleRequest) error { //nolint:gocyclo // Two Kubernetes subresources require a durable phased migration. + if s.kube == nil || q.ExpectedSchema < 0 || q.ExpectedSchema > 1 || q.OperationID == "" || q.ExpectedPodUID == "" || q.ExpectedPVCUID == "" { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "recognized prototype schema and exact runtime identities are required"} + } + if err := s.retryUpdateStatusRaw(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if err := adminSubject(o, q); err != nil { + return err + } + if textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype runtime identity mismatch"} + } + if completedMigrationMatches(o, q) { + return nil + } + if migrationID := textNested(o.Object, "status", "migrationOperation", "id"); migrationID != "" { + fromSchema, found, err := unstructured.NestedInt64(o.Object, "status", "migrationOperation", "fromSchema") + if migrationID != q.OperationID || err != nil || !found || fromSchema != q.ExpectedSchema { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "another schema migration is active"} + } + return nil + } + if intNested(o.Object, "spec", "schemaVersion") != q.ExpectedSchema || intNested(o.Object, "status", "schemaVersion") != q.ExpectedSchema || textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "activeOperation", "id") != "" || textNested(o.Object, "status", "lifecycleOperation", "id") != "" || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype environment is not eligible for migration"} + } + if err := s.verifyRuntimeUIDs(ctx, o, q); err != nil { + return err + } + refs, err := migrateReferenceRecords(o, q.OperationID, s.now()) + if err != nil { + return err + } + if err := setReferenceRecords(o, refs); err != nil { + return err + } + return unstructured.SetNestedMap(o.Object, map[string]any{"id": q.OperationID, "fromSchema": q.ExpectedSchema, "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID}, "status", "migrationOperation") + }); err != nil { + return err + } + for attempt := 0; attempt < 5; attempt++ { + o, err := s.resources.Get(ctx, q.Environment.ID, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if err != nil { + return err + } + if err := adminSubject(o, q); err != nil { + return err + } + if intNested(o.Object, "spec", "schemaVersion") == currentSchemaVersion { + break + } + if textNested(o.Object, "status", "migrationOperation", "id") != q.OperationID || intNested(o.Object, "spec", "schemaVersion") != q.ExpectedSchema { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "schema migration changed concurrently"} + } + _ = unstructured.SetNestedField(o.Object, currentSchemaVersion, "spec", "schemaVersion") + if _, err = s.resources.Update(ctx, o, metav1.UpdateOptions{}); err == nil { + break + } else if !apierrors.IsConflict(err) { + return err + } else if attempt == 4 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "schema migration conflict", Retryable: true} + } + } + return s.retryUpdateStatusRaw(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if err := adminSubject(o, q); err != nil { + return err + } + if completedMigrationMatches(o, q) { + return nil + } + fromSchema, found, schemaErr := unstructured.NestedInt64(o.Object, "status", "migrationOperation", "fromSchema") + if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion || textNested(o.Object, "status", "migrationOperation", "id") != q.OperationID || intNested(o.Object, "status", "schemaVersion") != q.ExpectedSchema || schemaErr != nil || !found || fromSchema != q.ExpectedSchema || textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "schema migration phase mismatch"} + } + epoch, ok := epochValue(o) + if !ok || epoch >= math.MaxInt64 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype epoch is invalid"} + } + _ = unstructured.SetNestedField(o.Object, int64(epoch+1), "status", "epoch") //nolint:gosec // checked above. + _ = unstructured.SetNestedField(o.Object, currentSchemaVersion, "status", "schemaVersion") + _ = unstructured.SetNestedField(o.Object, q.OperationID, "status", "lastMigrationOperationID") + _ = unstructured.SetNestedField(o.Object, q.ExpectedSchema, "status", "lastMigrationFromSchema") + unstructured.RemoveNestedField(o.Object, "status", "migrationOperation") + return nil + }) +} + +func completedMigrationMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + fromSchema, found, err := unstructured.NestedInt64(o.Object, "status", "lastMigrationFromSchema") + return err == nil && found && fromSchema == q.ExpectedSchema && + intNested(o.Object, "spec", "schemaVersion") == currentSchemaVersion && + intNested(o.Object, "status", "schemaVersion") == currentSchemaVersion && + textNested(o.Object, "status", "lastMigrationOperationID") == q.OperationID && + textNested(o.Object, "status", "pod", "uid") == q.ExpectedPodUID && + textNested(o.Object, "status", "pvc", "uid") == q.ExpectedPVCUID +} + +func (s *Store) verifyRuntimeUIDs(ctx context.Context, o *unstructured.Unstructured, q adminLifecycleRequest) error { + profile, ok := s.profiles.get(textNested(o.Object, "spec", "profile")) + if !ok || profile.Digest != textNested(o.Object, "spec", "profileDigest") { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype profile is unavailable or changed"} + } + pod, err := s.kube.CoreV1().Pods(s.namespace).Get(ctx, textNested(o.Object, "status", "pod", "name"), metav1.GetOptions{}) + if err != nil || string(pod.UID) != q.ExpectedPodUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype executor identity is not observable"} + } + pvc, err := s.kube.CoreV1().PersistentVolumeClaims(s.namespace).Get(ctx, textNested(o.Object, "status", "pvc", "name"), metav1.GetOptions{}) + if err != nil || string(pvc.UID) != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype workspace identity is not observable"} + } + if err := validatePVC(o, profile, pvc); err != nil { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype workspace shape is incompatible"} + } + if err := validatePod(o, profile, pvc.Name, pod); err != nil { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype executor shape is insecure or incompatible"} + } + return nil +} + +func migrateReferenceRecords(o *unstructured.Unstructured, operationID string, now time.Time) ([]referenceRecord, error) { + raw, found, err := unstructured.NestedSlice(o.Object, "status", "references") + if err != nil || !found { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} + } + out := make([]referenceRecord, 0, len(raw)) + for _, value := range raw { + switch item := value.(type) { + case string: + if item == "" { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} + } + out = append(out, referenceRecord{BindingID: item, State: executionenv.ReferencePublished, OperationID: "migration-" + operationID, CreatedAt: now}) + case map[string]any: + created, parseErr := time.Parse(time.RFC3339Nano, text(item, "createdAt")) + r := referenceRecord{BindingID: text(item, "bindingID"), State: executionenv.ReferenceState(text(item, "state")), OperationID: text(item, operationIDField), SourceBindingID: text(item, "sourceBindingID"), CreatedAt: created} + if parseErr != nil || r.BindingID == "" || r.OperationID == "" || r.State != executionenv.ReferencePendingCreate && r.State != executionenv.ReferencePublished && r.State != executionenv.ReferencePendingDelete { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} + } + out = append(out, r) + default: + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} + } + } + return out, nil +} diff --git a/internal/adapter/executioncontroller/admin_scope_security_test.go b/internal/adapter/executioncontroller/admin_scope_security_test.go new file mode 100644 index 0000000000..8e7992dce2 --- /dev/null +++ b/internal/adapter/executioncontroller/admin_scope_security_test.go @@ -0,0 +1,91 @@ +package executioncontroller + +import ( + "encoding/json" + "fmt" + "reflect" + "testing" + "time" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func scopedManifestClient(t *testing.T, uri string, admin bool, scope []string) securityClientManifest { + t.Helper() + return securityClientManifest{URI: uri, Administrator: admin, AdministratorFor: scope} +} + +func TestAdministratorScopeManifestValidation(t *testing.T) { + for _, tc := range []struct { + name, raw string + valid bool + }{ + {"absent", `{"uri":"spiffe://example/admin","administrator":true}`, true}, + {"empty", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":[]}`, true}, + {"scoped", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/creator"]}`, true}, + {"not admin", `{"uri":"spiffe://example/admin","administratorFor":["spiffe://example/creator"]}`, false}, + {"duplicate", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/creator","spiffe://example/creator"]}`, false}, + {"wildcard", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/*"]}`, false}, + {"escaped wildcard", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/%2A"]}`, false}, + {"case", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://EXAMPLE/creator"]}`, false}, + {"query", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/creator?q=1"]}`, false}, + {"fragment", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/creator#x"]}`, false}, + {"userinfo", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://user@example/creator"]}`, false}, + {"malformed", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["creator"]}`, false}, + {"unknown", `{"uri":"spiffe://example/admin","administrator":true,"adminFor":[]}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var entry securityClientManifest + err := executionenv.DecodeStrict([]byte(tc.raw), &entry) + if err == nil { + _, err = clientPolicies([]securityClientManifest{entry}) + } + if (err == nil) != tc.valid { + t.Fatalf("valid=%v err=%v", tc.valid, err) + } + }) + } + for _, count := range []int{256, 257} { + scope := make([]string, count) + for i := range scope { + scope[i] = fmt.Sprintf("spiffe://example/creator-%d", i) + } + entry := scopedManifestClient(t, "spiffe://example/admin", true, scope) + if _, err := clientPolicies([]securityClientManifest{entry}); (err == nil) != (count == 256) { + t.Fatalf("scope count=%d err=%v", count, err) + } + } +} + +func TestAdministratorScopeDigestIsNormalizedAndAuthorityBound(t *testing.T) { + a, b := "spiffe://example/a", "spiffe://example/b" + mf := securityManifest{Clients: []securityClientManifest{scopedManifestClient(t, "spiffe://example/admin", true, []string{b, a})}} + before, _ := json.Marshal(mf) + digest := func(m securityManifest) string { + t.Helper() + d, err := authorityDigest(m, time.Minute, time.Second, nil, nil, "server") + if err != nil { + t.Fatal(err) + } + return d + } + original := digest(mf) + after, _ := json.Marshal(mf) + if !reflect.DeepEqual(before, after) { + t.Fatal("digest mutated source manifest") + } + mf.Clients[0] = scopedManifestClient(t, "spiffe://example/admin", true, []string{a, b}) + if digest(mf) != original { + t.Fatal("scope ordering changed authority digest") + } + mf.Clients[0] = scopedManifestClient(t, "spiffe://example/admin", true, []string{a}) + if digest(mf) == original { + t.Fatal("scope removal did not change authority digest") + } + mf.Clients[0] = scopedManifestClient(t, "spiffe://example/admin", true, nil) + empty := digest(mf) + mf.Clients[0] = scopedManifestClient(t, "spiffe://example/admin", true, []string{}) + if digest(mf) != empty { + t.Fatal("empty and absent scope differ") + } +} diff --git a/internal/adapter/executioncontroller/admin_scope_test.go b/internal/adapter/executioncontroller/admin_scope_test.go new file mode 100644 index 0000000000..b12fe48d09 --- /dev/null +++ b/internal/adapter/executioncontroller/admin_scope_test.go @@ -0,0 +1,564 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "net" + "os" + "path/filepath" + "reflect" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const scopeCreator = "spiffe://example/creator" +const scopeAdmin = "spiffe://example/operations" + +var scopeOwner = executionenv.Owner{Issuer: "https://issuer.example", Subject: "owner"} + +type adminScopeFixture struct { + client executionv1.ExecutionProviderServiceClient + dynamic *dynamicfake.FakeDynamicClient + manager *SecurityManager + manifest securityManifest + path string +} + +func newAdminScopeFixture(t *testing.T, route, actor string) *adminScopeFixture { + t.Helper() + env := lifecycleAdminEnvironment(2, []any{}) + if route == "data" { + env = runFixtureEnvironment(4, 4) + } + if route == "migrate" { + _ = unstructured.SetNestedField(env.Object, int64(1), "spec", "schemaVersion") + _ = unstructured.SetNestedField(env.Object, int64(1), "status", "schemaVersion") + } + if route == "recover" { + _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") + } + if route == "delete" { + setConditionObject(env, "Retired", true, "Retired", "retired") + setConditionObject(env, "ExecutorTerminated", true, "Terminated", "terminated") + } + _ = unstructured.SetNestedField(env.Object, hashText(scopeCreator), "spec", "clientHash") + _ = unstructured.SetNestedField(env.Object, ownerHash(scopeOwner), "spec", "ownerHash") + _ = unstructured.SetNestedField(env.Object, int64(4), "status", "grantGeneration") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC()) + store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) + dir, now := t.TempDir(), time.Now().UTC() + ca, cert, key, clientCert := rpcSecurityPKI(t, now, actor) + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + writePKCS8(t, filepath.Join(dir, "grant.pem"), priv) + for name, data := range map[string][]byte{"server.crt": cert, "server.key": key, "clients.pem": ca} { + if err := os.WriteFile(filepath.Join(dir, name), data, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(pub) + mf := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: actor}}} + path := filepath.Join(dir, "manifest.json") + writeManifest(t, path, mf) + manager := NewSecurityManager(path, dir, "ns", "authority", k) + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig()))) + executionv1.RegisterExecutionProviderServiceServer(server, NewHandler(HandlerConfig{Security: manager}, store)) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(ca) + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, ServerName: "provider.test", RootCAs: roots, Certificates: []tls.Certificate{clientCert}}))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + return &adminScopeFixture{client: executionv1.NewExecutionProviderServiceClient(conn), dynamic: d, manager: manager, manifest: mf, path: path} +} + +func (f *adminScopeFixture) policy(t *testing.T, admin bool, scope []string) { + t.Helper() + f.manifest.Generation++ + f.manifest.Clients[0] = scopedManifestClient(t, f.manifest.Clients[0].URI, admin, scope) + writeManifest(t, f.path, f.manifest) + if err := f.manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } +} + +func callScopedAdmin(ctx context.Context, c executionv1.ExecutionProviderServiceClient, route string, q adminLifecycleRequest, owner executionenv.Owner) error { + ref := &executionv1.EnvironmentRef{Id: q.Environment.ID, Revision: q.Environment.Revision} + o := &executionv1.Owner{Issuer: owner.Issuer, Subject: owner.Subject} + var err error + switch route { + case "retire": + _, err = c.RetireEnvironment(ctx, &executionv1.RetireEnvironmentRequest{Environment: ref, Owner: o, ExpectedExecutionEpoch: q.ExpectedEpoch, ExpectedPodUid: q.ExpectedPodUID, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "replace": + _, err = c.ReplaceExecutor(ctx, &executionv1.ReplaceExecutorRequest{Environment: ref, Owner: o, ExpectedExecutionEpoch: q.ExpectedEpoch, ExpectedPodUid: q.ExpectedPodUID, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "recover": + _, err = c.RecoverEnvironment(ctx, &executionv1.RecoverEnvironmentRequest{Environment: ref, Owner: o, ExpectedExecutionEpoch: q.ExpectedEpoch, ExpectedPodUid: q.ExpectedPodUID, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "delete": + _, err = c.DeleteRetiredEnvironment(ctx, &executionv1.DeleteRetiredEnvironmentRequest{Environment: ref, Owner: o, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "migrate": + schema := uint32(q.ExpectedSchema) + _, err = c.MigrateEnvironment(ctx, &executionv1.MigrateEnvironmentRequest{Environment: ref, Owner: o, ExpectedSchemaVersion: &schema, ExpectedPodUid: q.ExpectedPodUID, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "revoke": + _, err = c.RevokeEnvironment(ctx, &executionv1.RevokeEnvironmentRequest{Environment: ref, Owner: o, ExpectedGrantGeneration: q.ExpectedEpoch, OperationId: q.OperationID}) + default: + panic("unknown test route") + } + return err +} + +func TestScopedAdminAllRoutesOverMTLS(t *testing.T) { + for _, route := range []string{"retire", "replace", "recover", "delete", "migrate", "revoke"} { + t.Run(route, func(t *testing.T) { + f := newAdminScopeFixture(t, route, scopeAdmin) + q := adminRequestFixture() + q.ExpectedSchema = 1 + call := func(q adminLifecycleRequest, owner executionenv.Owner) error { + return callScopedAdmin(t.Context(), f.client, route, q, owner) + } + if err := call(q, scopeOwner); status.Code(err) != codes.PermissionDenied { + t.Fatalf("nonadmin: %v", err) + } + for _, scope := range [][]string{nil, {"spiffe://example/unrelated"}} { + f.policy(t, true, scope) + denied := call(q, scopeOwner) + missing := q + missing.Environment.ID = "absent" + absent := call(missing, scopeOwner) + if status.Code(denied) != codes.NotFound || status.Convert(denied).Message() != status.Convert(absent).Message() || status.Code(absent) != codes.NotFound { + t.Fatalf("scope/missing disclosure: denied=%v absent=%v", denied, absent) + } + } + // Creator login is deliberately absent from the allowlist. + f.policy(t, true, []string{scopeCreator}) + wrongOwner := scopeOwner + wrongOwner.Subject = "another-owner" + if err := call(q, wrongOwner); status.Code(err) != codes.NotFound { + t.Fatalf("owner mismatch: %v", err) + } + stale := q + stale.Environment.Revision = "stale" + if err := call(stale, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("revision mismatch: %v", err) + } + stale = q + if route == "revoke" { + stale.ExpectedEpoch++ + } else { + stale.ExpectedPVCUID = "stale" + } + if err := call(stale, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("stale runtime identity: %v", err) + } + if route == "retire" || route == "replace" || route == "recover" { + stale = q + stale.ExpectedEpoch++ + if err := call(stale, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("stale epoch: %v", err) + } + } + if route != "delete" && route != "revoke" { + stale = q + stale.ExpectedPodUID = "stale" + if err := call(stale, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("stale Pod UID: %v", err) + } + } + if route == "migrate" { + stale = q + stale.ExpectedSchema = 0 + if err := call(stale, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("wrong schema: %v", err) + } + } + before, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := call(q, scopeOwner); err != nil { + t.Fatalf("scoped admin rejected: %v", err) + } + if err := call(q, scopeOwner); err != nil { + t.Fatalf("exact replay: %v", err) + } + after, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if reflect.DeepEqual(before.Object["status"], after.Object["status"]) { + t.Fatal("successful administration did not persist its transition") + } + if route == "revoke" { + receipts, _, err := unstructured.NestedSlice(after.Object, "status", "revocationReceipts") + if err != nil || len(receipts) != 1 || intNested(after.Object, "status", "grantGeneration") != 5 { + t.Fatalf("revocation replay was not idempotent: %v", after.Object["status"]) + } + if text(receipts[0].(map[string]any), "fingerprint") != fingerprint("env", "rev", scopeAdmin, ownerHash(scopeOwner), "4") { + t.Fatal("revocation receipt replaced the authenticated actor with the creator") + } + } + for _, field := range []string{"clientHash", "ownerHash", "revision"} { + if textNested(before.Object, "spec", field) != textNested(after.Object, "spec", field) { + t.Fatalf("admin rewrote immutable %s", field) + } + } + // Even receipt replay is a new admission on the established connection. + f.policy(t, true, nil) + if err := call(q, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("scope removal did not revoke replay: %v", err) + } + }) + t.Run(route+"/creator", func(t *testing.T) { + f := newAdminScopeFixture(t, route, scopeCreator) + q := adminRequestFixture() + q.ExpectedSchema = 1 + if err := callScopedAdmin(t.Context(), f.client, route, q, scopeOwner); status.Code(err) != codes.PermissionDenied { + t.Fatalf("normal creator is admin: %v", err) + } + f.policy(t, true, nil) + if err := callScopedAdmin(t.Context(), f.client, route, q, scopeOwner); err != nil { + t.Fatalf("legacy self-admin: %v", err) + } + }) + } +} + +func TestScopedAdminCASRetryRechecksSubject(t *testing.T) { + for _, route := range []string{"retire", "replace", "recover", "delete", "migrate", "revoke"} { + t.Run(route, func(t *testing.T) { + f := newAdminScopeFixture(t, route, scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + var conflicted atomic.Bool + f.dynamic.PrependReactor("update", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + if conflicted.Swap(true) { + return false, nil, nil + } + current, err := f.dynamic.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + if err != nil { + return true, nil, err + } + o := current.(*unstructured.Unstructured) + _ = unstructured.SetNestedField(o.Object, hashText("spiffe://example/other"), "spec", "clientHash") + if err := f.dynamic.Tracker().Update(ExecutionEnvironmentGVR, o, "ns"); err != nil { + return true, nil, err + } + return true, nil, apierrors.NewConflict(ExecutionEnvironmentGVR.GroupResource(), "env", nil) + }) + q := adminRequestFixture() + q.ExpectedSchema = 1 + if err := callScopedAdmin(t.Context(), f.client, route, q, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("CAS retry escaped scope: %v", err) + } + if !conflicted.Load() { + t.Fatal("did not exercise CAS retry") + } + }) + } +} + +func TestScopedAdminUsesAdmittedPolicySnapshot(t *testing.T) { + f := newAdminScopeFixture(t, "revoke", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + var changed atomic.Bool + f.manifest.Generation++ + f.manifest.Clients[0] = scopedManifestClient(t, scopeAdmin, true, nil) + writeManifest(t, f.path, f.manifest) + f.dynamic.PrependReactor("get", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + if !changed.Swap(true) { + // Authentication has already admitted this request; change the live + // snapshot before the store makes its subject decision. + if err := f.manager.Reload(t.Context()); err != nil { + return true, nil, err + } + } + return false, nil, nil + }) + q := adminRequestFixture() + if err := callScopedAdmin(t.Context(), f.client, "revoke", q, scopeOwner); err != nil { + t.Fatalf("admitted snapshot was replaced mid-request: %v", err) + } + if !changed.Load() { + t.Fatal("policy did not change during admission") + } + if err := callScopedAdmin(t.Context(), f.client, "revoke", q, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("next request reused old scope: %v", err) + } +} + +func TestScopedAdminCompletedReplacementReceiptReauthorizes(t *testing.T) { + f := newAdminScopeFixture(t, "replace", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + q := adminRequestFixture() + env, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + _ = unstructured.SetNestedMap(env.Object, map[string]any{"operationID": q.OperationID, "previousPodUID": q.ExpectedPodUID, "replacementPodUID": "new-pod", "pvcUID": q.ExpectedPVCUID, "previousEpoch": int64(4), "replacementEpoch": int64(5)}, "status", "lastReplacement") + _ = unstructured.SetNestedField(env.Object, int64(5), "status", "epoch") + _ = unstructured.SetNestedField(env.Object, "new-pod", "status", "pod", "uid") + if _, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(t.Context(), env, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := callScopedAdmin(t.Context(), f.client, "replace", q, scopeOwner); err != nil { + t.Fatal(err) + } + wrongOwner := scopeOwner + wrongOwner.Subject = "other" + if err := callScopedAdmin(t.Context(), f.client, "replace", q, wrongOwner); status.Code(err) != codes.NotFound { + t.Fatalf("receipt bypassed owner check: %v", err) + } + f.policy(t, true, nil) + if err := callScopedAdmin(t.Context(), f.client, "replace", q, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("receipt bypassed removed scope: %v", err) + } +} + +func TestScopedAdminMigrationReceiptRetainsUIDPreconditions(t *testing.T) { + f := newAdminScopeFixture(t, "migrate", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + q := adminRequestFixture() + q.ExpectedSchema = 1 + if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); err != nil { + t.Fatal(err) + } + if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); err != nil { + t.Fatalf("exact migration replay failed: %v", err) + } + q.ExpectedSchema = 0 + if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("migration receipt bypassed source schema: %v", err) + } + q.ExpectedSchema = 1 + q.ExpectedPVCUID = "stale" + if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("migration receipt bypassed UID: %v", err) + } +} + +func TestMigrationCompletedCASReplayRequiresExactSourceSchema(t *testing.T) { + for _, receipt := range []string{"exact", "changed", "absent", "expired"} { + t.Run(receipt, func(t *testing.T) { + f := newAdminScopeFixture(t, "migrate", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + var raced bool + f.dynamic.PrependReactor("update", "executionenvironments", func(action k8stesting.Action) (bool, runtime.Object, error) { + u := action.(k8stesting.UpdateAction).GetObject().(*unstructured.Unstructured) + if raced || textNested(u.Object, "status", "lastMigrationOperationID") == "" { + return false, nil, nil + } + raced = true + completed := u.DeepCopy() + switch receipt { + case "changed": + _ = unstructured.SetNestedField(completed.Object, int64(0), "status", "lastMigrationFromSchema") + case "absent": + unstructured.RemoveNestedField(completed.Object, "status", "lastMigrationFromSchema") + case "expired": + unstructured.RemoveNestedField(completed.Object, "status", "lastMigrationOperationID") + unstructured.RemoveNestedField(completed.Object, "status", "lastMigrationFromSchema") + } + if err := f.dynamic.Tracker().Update(ExecutionEnvironmentGVR, completed, "ns"); err != nil { + t.Fatal(err) + } + return true, nil, apierrors.NewConflict(ExecutionEnvironmentGVR.GroupResource(), "env", nil) + }) + q := adminRequestFixture() + q.ExpectedSchema = 1 + err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner) + want := codes.Aborted + if receipt == "exact" { + want = codes.OK + } + if !raced || status.Code(err) != want { + t.Fatalf("CAS replay raced=%t code=%v, want %v", raced, status.Code(err), want) + } + }) + } +} + +func TestScopedAdminEqualGenerationDriftFailsClosed(t *testing.T) { + f := newAdminScopeFixture(t, "revoke", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + q := adminRequestFixture() + if err := callScopedAdmin(t.Context(), f.client, "revoke", q, scopeOwner); err != nil { + t.Fatal(err) + } + f.manifest.Clients[0] = scopedManifestClient(t, scopeAdmin, true, nil) + writeManifest(t, f.path, f.manifest) + if err := f.manager.Reload(t.Context()); err == nil { + t.Fatal("equal-generation scope drift admitted") + } + if err := callScopedAdmin(t.Context(), f.client, "revoke", q, scopeOwner); status.Code(err) != codes.Unavailable { + t.Fatalf("stale snapshot admitted RPC: %v", err) + } +} + +func TestScopedAdminWithOwnerAttestationCannotUseAnotherCreatorsDataPlane(t *testing.T) { + f := newAdminScopeFixture(t, "data", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + f.manifest.Generation++ + f.manifest.Clients[0].MayAttestOwner = true + writeManifest(t, f.path, f.manifest) + if err := f.manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + ref := &executionv1.EnvironmentRef{Id: "env", Revision: "rev"} + owner := &executionv1.Owner{Issuer: scopeOwner.Issuer, Subject: scopeOwner.Subject} + before, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + refs, err := referenceRecords(before) + if err != nil { + t.Fatal(err) + } + refs = append(refs, referenceRecord{BindingID: "pending", State: executionenv.ReferencePendingCreate, OperationID: "pending-op", CreatedAt: time.Now().UTC()}) + if err := setReferenceRecords(before, refs); err != nil { + t.Fatal(err) + } + if _, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(t.Context(), before, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + for name, call := range map[string]func() error{ + "attach": func() error { + _, err := f.client.AttachEnvironment(t.Context(), &executionv1.AttachEnvironmentRequest{Context: &executionv1.RequestContext{Environment: ref, Owner: owner, BindingId: "binding"}, Purpose: "session"}) + return err + }, + "acquire": func() error { + _, err := f.client.AcquireRun(t.Context(), &executionv1.AcquireRunRequest{Environment: ref, Owner: owner, BindingId: "binding", RunId: "run", OperationId: "acquire", TtlMillis: 60000}) + return err + }, + "reference": func() error { + _, err := f.client.CommitReference(t.Context(), &executionv1.ReferenceMutationRequest{Environment: ref, Owner: owner, BindingId: "pending", OperationId: "pending-op"}) + return err + }, + "exact discovery": func() error { + _, err := f.client.ListReferenceIntents(t.Context(), &executionv1.ListReferenceIntentsRequest{Environment: ref, Owner: owner, BindingId: "pending"}) + return err + }, + } { + if err := call(); status.Code(err) != codes.NotFound { + t.Errorf("%s: expected creator-bound not found, got %v", name, err) + } + } + for _, query := range []*executionv1.ListReferenceIntentsRequest{{Owner: owner}, {}} { + list, err := f.client.ListReferenceIntents(t.Context(), query) + if err != nil || len(list.GetIntents()) != 0 { + t.Fatal("discovery exposed another creator's pending reference") + } + } + after, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil || !reflect.DeepEqual(before.Object, after.Object) { + t.Fatal("creator-bound denials changed foreign environment") + } + own, err := f.client.EnsureEnvironment(t.Context(), &executionv1.EnsureEnvironmentRequest{BindingId: "own", Profile: "go", Owner: owner, OperationId: "own-create"}) + if err != nil || own.GetEnvironment().GetId() == "env" { + t.Fatalf("explicit owner attestation did not permit own allocation: %v", err) + } + list, err := f.client.ListReferenceIntents(t.Context(), &executionv1.ListReferenceIntentsRequest{Owner: owner}) + if err != nil || len(list.GetIntents()) != 1 || list.Intents[0].GetEnvironment().GetId() != own.GetEnvironment().GetId() { + t.Fatal("discovery did not isolate the actor's own pending allocation") + } +} + +func TestScopedAdminDoesNotGrantAttestationOrDataPlane(t *testing.T) { + f := newAdminScopeFixture(t, "data", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + ref := &executionv1.EnvironmentRef{Id: "env", Revision: "rev"} + owner := &executionv1.Owner{Issuer: scopeOwner.Issuer, Subject: scopeOwner.Subject} + request := &executionv1.RequestContext{Environment: ref, Owner: owner, BindingId: "binding", Epoch: 4, RunId: "run", ClaimId: "claim", GrantGeneration: 4} + // A correctly signed actor-bound grant still cannot cross creator ownership. + h := NewHandler(HandlerConfig{Security: f.manager}, nil) + claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 4, GrantGeneration: 4} + grant, _, err := h.signClaim(t.Context(), claim, scopeAdmin, ownerHash(scopeOwner)) + if err != nil { + t.Fatal(err) + } + request.Grant = grant + calls := map[string]func() error{ + "ensure attestation": func() error { + _, err := f.client.EnsureEnvironment(t.Context(), &executionv1.EnsureEnvironmentRequest{BindingId: "new", Profile: "go", Owner: owner, OperationId: "ensure"}) + return err + }, + "attach": func() error { + _, err := f.client.AttachEnvironment(t.Context(), &executionv1.AttachEnvironmentRequest{Context: request, Purpose: "session"}) + return err + }, + "run": func() error { + _, err := f.client.AcquireRun(t.Context(), &executionv1.AcquireRunRequest{Environment: ref, Owner: owner, BindingId: "binding", RunId: "run", OperationId: "acquire", TtlMillis: 60000}) + return err + }, + "reference": func() error { + _, err := f.client.CommitReference(t.Context(), &executionv1.ReferenceMutationRequest{Environment: ref, Owner: owner, BindingId: "binding", OperationId: "commit"}) + return err + }, + "reference discovery": func() error { + _, err := f.client.ListReferenceIntents(t.Context(), &executionv1.ListReferenceIntentsRequest{Owner: owner}) + return err + }, + "files": func() error { + _, err := f.client.Files(t.Context(), &executionv1.FileRequest{Context: request, Operation: executionv1.FileOperation_FILE_OPERATION_READ, Path: "file"}) + return err + }, + "shell": func() error { + _, err := f.client.StartCommand(t.Context(), &executionv1.CommandStartRequest{Context: request, Command: "true"}) + return err + }, + } + before, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + for name, call := range calls { + want := codes.PermissionDenied + switch name { + case "run", "reference", "reference discovery": + want = codes.InvalidArgument + case "files", "shell": + want = codes.NotFound + } + if err := call(); status.Code(err) != want { + t.Errorf("%s: want %v, got %v", name, want, err) + } + } + after, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before.Object, after.Object) { + t.Fatal("denied data-plane calls changed environment") + } +} diff --git a/internal/adapter/executioncontroller/controller.go b/internal/adapter/executioncontroller/controller.go new file mode 100644 index 0000000000..f1c3f7deb3 --- /dev/null +++ b/internal/adapter/executioncontroller/controller.go @@ -0,0 +1,470 @@ +package executioncontroller + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + environmentFinalizer = "execution.mecatl.dev/retain-workspace" + executorFinalizer = "execution.mecatl.dev/verify-termination" + fenceHealthy = "Healthy" + statusField = "status" +) + +// Reconciler maintains executor Pods and retained workspace PVCs. +type Reconciler struct { + dynamic dynamic.Interface + kube kubernetes.Interface + namespace string + profiles *Profiles + queue workqueue.TypedRateLimitingInterface[string] + initOnce sync.Once + initErr error + ready atomic.Bool + now func() time.Time +} + +// NewReconciler constructs a reconciler for one namespace. +func NewReconciler(d dynamic.Interface, k kubernetes.Interface, namespace string, p *Profiles) *Reconciler { + return &Reconciler{dynamic: d, kube: k, namespace: namespace, profiles: p, queue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[string]()), now: func() time.Time { return time.Now().UTC() }} +} + +// Initialize synchronizes informer caches before the provider may expose mutating APIs. +// Startup never mutates operation ownership: another replica may still be its live holder. +func (r *Reconciler) Initialize(ctx context.Context) error { + r.initOnce.Do(func() { + if err := r.preflightProfiles(ctx); err != nil { + r.initErr = err + return + } + factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(r.dynamic, 0, r.namespace, nil) + environments := factory.ForResource(ExecutionEnvironmentGVR).Informer() + _, err := environments.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: r.enqueue, UpdateFunc: r.enqueueUpdate}) + if err != nil { + r.initErr = err + return + } + runtimeFactory := informers.NewSharedInformerFactoryWithOptions(r.kube, 0, informers.WithNamespace(r.namespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) { + options.LabelSelector = "execution.mecatl.dev/environment" + })) + pods := runtimeFactory.Core().V1().Pods().Informer() + pvcs := runtimeFactory.Core().V1().PersistentVolumeClaims().Informer() + for _, informer := range []cache.SharedIndexInformer{pods, pvcs} { + if _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: r.enqueueRuntime, UpdateFunc: func(_, next any) { r.enqueueRuntime(next) }, DeleteFunc: r.enqueueRuntime}); err != nil { + r.initErr = err + return + } + } + factory.Start(ctx.Done()) + runtimeFactory.Start(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), environments.HasSynced, pods.HasSynced, pvcs.HasSynced) { + r.initErr = errors.New("execution environment informer failed to sync") + return + } + go wait.UntilWithContext(ctx, r.worker, time.Second) + r.ready.Store(true) + }) + return r.initErr +} + +func (r *Reconciler) preflightProfiles(ctx context.Context) error { + runtimeClasses, storageClasses := r.profiles.clusterResources() + for _, name := range runtimeClasses { + if _, err := r.kube.NodeV1().RuntimeClasses().Get(ctx, name, metav1.GetOptions{}); err != nil { + return fmt.Errorf("profile preflight: RuntimeClass %q unavailable: %w", name, err) + } + } + for _, name := range storageClasses { + if _, err := r.kube.StorageV1().StorageClasses().Get(ctx, name, metav1.GetOptions{}); err != nil { + return fmt.Errorf("profile preflight: StorageClass %q unavailable: %w", name, err) + } + } + return nil +} + +// Ready reports whether startup fencing, profile preflight, and informer synchronization completed. +func (r *Reconciler) Ready() bool { return r.ready.Load() } + +// ReadinessReason returns a bounded operator-facing reason class. +func (r *Reconciler) ReadinessReason() string { + if r.ready.Load() { + return "ready" + } + if r.initErr != nil && strings.HasPrefix(r.initErr.Error(), "profile preflight:") { + return "profile-resource-unavailable" + } + if r.initErr != nil { + return "controller-cache-unavailable" + } + return "controller-starting" +} + +// Run initializes the reconciler and blocks until shutdown. +func (r *Reconciler) Run(ctx context.Context) error { + if err := r.Initialize(ctx); err != nil { + return err + } + <-ctx.Done() + r.queue.ShutDown() + return nil +} +func (r *Reconciler) enqueue(obj any) { + u, ok := obj.(*unstructured.Unstructured) + if ok { + r.queue.Add(u.GetName()) + } +} + +func (r *Reconciler) enqueueUpdate(oldObj, newObj any) { + oldEnv, oldOK := oldObj.(*unstructured.Unstructured) + newEnv, newOK := newObj.(*unstructured.Unstructured) + if !oldOK || !newOK || !reconcileInputsEqual(oldEnv, newEnv) { + r.enqueue(newObj) + } +} + +func reconcileInputsEqual(oldEnv, newEnv *unstructured.Unstructured) bool { + if oldEnv.GetGeneration() != newEnv.GetGeneration() || + !reflect.DeepEqual(oldEnv.GetDeletionTimestamp(), newEnv.GetDeletionTimestamp()) || + !reflect.DeepEqual(oldEnv.GetFinalizers(), newEnv.GetFinalizers()) { + return false + } + for _, path := range [][]string{{"spec"}, {statusField, "activeOperation"}, {statusField, "lifecycleOperation"}, {statusField, "migrationOperation"}, {statusField, "references"}, {statusField, "fenceState"}} { + oldValue, _, _ := unstructured.NestedFieldNoCopy(oldEnv.Object, path...) + newValue, _, _ := unstructured.NestedFieldNoCopy(newEnv.Object, path...) + if !reflect.DeepEqual(oldValue, newValue) { + return false + } + } + return true +} + +func (r *Reconciler) enqueueRuntime(obj any) { + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + var labels map[string]string + switch value := obj.(type) { + case *corev1.Pod: + labels = value.Labels + case *corev1.PersistentVolumeClaim: + labels = value.Labels + } + if name := labels["execution.mecatl.dev/environment"]; name != "" { + r.queue.Add(name) + } +} +func (r *Reconciler) worker(ctx context.Context) { + for r.process(ctx) { + } +} +func (r *Reconciler) process(ctx context.Context) bool { + name, shutdown := r.queue.Get() + if shutdown { + return false + } + defer r.queue.Done(name) + if err := r.Reconcile(ctx, name); err != nil { + r.queue.AddRateLimited(name) + } else { + r.queue.Forget(name) + } + return true +} + +// Reconcile converges one ExecutionEnvironment and its runtime resources. +func (r *Reconciler) Reconcile(ctx context.Context, name string) error { + res := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace) + env, err := res.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + if requireCurrentSchema(env) != nil { + return r.setCondition(ctx, env, "Ready", false, "IncompatibleSchema", "explicit administrator migration to schema version 2 is required") + } + if expires := textNested(env.Object, "status", "activeOperation", "expiresAt"); expires != "" { + deadline, parseErr := time.Parse(time.RFC3339Nano, expires) + now := r.now() + if parseErr != nil || !now.Before(deadline) { + return r.setFenceUnknown(ctx, env, "operation holder lease expired; operation identity retained for recovery") + } + r.queue.AddAfter(name, deadline.Sub(now)) + } + profileName := textNested(env.Object, "spec", "profile") + p, ok := r.profiles.get(profileName) + if !ok || p.Digest != textNested(env.Object, "spec", "profileDigest") { + return r.setCondition(ctx, env, "Ready", false, "InvalidProfile", "configured profile is unavailable or changed") + } + // The admitted durable delete owns finalization, including after DELETE has + // set deletionTimestamp. A peer must not re-arm the generic finalizer. + if textNested(env.Object, "status", "lifecycleOperation", "type") == deleteRetiredEnvironment { + return r.reconcileLifecycle(ctx, env) + } + if env.GetDeletionTimestamp() != nil { + return r.reconcileDeletion(ctx, res, env) + } + if !contains(env.GetFinalizers(), environmentFinalizer) { + updated := env.DeepCopy() + updated.SetFinalizers(append(updated.GetFinalizers(), environmentFinalizer)) + if _, err := res.Update(ctx, updated, metav1.UpdateOptions{}); err != nil { + return err + } + return nil + } + if textNested(env.Object, "status", "lifecycleOperation", "id") != "" { + return r.reconcileLifecycle(ctx, env) + } + if textNested(env.Object, "spec", "desired") == "Retiring" { + if conditionTrue(env, "Retired") { + return nil + } + return r.setCondition(ctx, env, "Ready", false, "UnsupportedRetirementRequest", "retirement requires the exact administrator lifecycle operation") + } + pvcName := resourceName("workspace", name) + podName := resourceName("executor", name) + pvc, err := r.ensurePVC(ctx, env, p, pvcName) + if err != nil { + return errors.Join(err, r.setCondition(ctx, env, "Ready", false, "PVCUnavailable", "workspace PVC unavailable")) + } + pod, err := r.ensurePod(ctx, env, p, podName, pvcName) + if err != nil { + return errors.Join(err, r.setCondition(ctx, env, "Ready", false, "ExecutorUnavailable", "executor Pod unavailable")) + } + ready := podReady(pod) + if err := r.updateRuntimeStatus(ctx, env, pvc, pod, ready); err != nil { + return err + } + if !ready { + r.queue.AddAfter(name, time.Second) + } + return nil +} +func (r *Reconciler) ensurePVC(ctx context.Context, env *unstructured.Unstructured, p resolvedProfile, name string) (*corev1.PersistentVolumeClaim, error) { + pvcs := r.kube.CoreV1().PersistentVolumeClaims(r.namespace) + cur, err := pvcs.Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return cur, validatePVC(env, p, cur) + } + if !apierrors.IsNotFound(err) { + return nil, err + } + if textNested(env.Object, "status", "pvc", "uid") != "" { + return nil, errors.New("authoritative PVC disappeared; replacement is forbidden") + } + qty := p.StorageSize + mode := corev1.PersistentVolumeFilesystem + created, err := pvcs.Create(ctx, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/revision": textNested(env.Object, "spec", "revision"), "execution.mecatl.dev/allocation-uid": string(env.GetUID())}}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, StorageClassName: &p.Spec.StorageClass, VolumeMode: &mode, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: qty}}}}, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + created, err = pvcs.Get(ctx, name, metav1.GetOptions{}) + } + if err != nil { + return nil, err + } + return created, validatePVC(env, p, created) +} +func (r *Reconciler) ensurePod(ctx context.Context, env *unstructured.Unstructured, p resolvedProfile, name, pvc string) (*corev1.Pod, error) { + pods := r.kube.CoreV1().Pods(r.namespace) + cur, err := pods.Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return cur, validatePod(env, p, pvc, cur) + } + if !apierrors.IsNotFound(err) { + return nil, err + } + if textNested(env.Object, "status", "pod", "uid") != "" { + return nil, errors.Join( + &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "previous executor termination unconfirmed"}, + r.setFenceUnknown(ctx, env, "previous executor disappearance is not externally fenced"), + ) + } + cpuReq := p.CPURequest + memReq := p.MemoryRequest + cpuLim := p.CPULimit + memLim := p.MemoryLimit + ephemeralReq := p.EphemeralStorageRequest + ephemeralLim := p.EphemeralStorageLimit + tmpLim := p.TmpSizeLimit + nonroot := true + uid := int64(65532) + noPriv := false + ro := true + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/profile": hashText(textNested(env.Object, "spec", "profile"))[:16]}, Finalizers: []string{executorFinalizer}, OwnerReferences: []metav1.OwnerReference{{APIVersion: env.GetAPIVersion(), Kind: env.GetKind(), Name: env.GetName(), UID: env.GetUID(), Controller: &nonroot}}}, Spec: corev1.PodSpec{AutomountServiceAccountToken: &noPriv, RuntimeClassName: &p.Spec.RuntimeClassName, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &nonroot, RunAsUser: &uid, RunAsGroup: &uid, FSGroup: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "executor", Image: p.Spec.Image, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &noPriv, ReadOnlyRootFilesystem: &ro, RunAsNonRoot: &nonroot, RunAsUser: &uid, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: cpuReq, corev1.ResourceMemory: memReq, corev1.ResourceEphemeralStorage: ephemeralReq}, Limits: corev1.ResourceList{corev1.ResourceCPU: cpuLim, corev1.ResourceMemory: memLim, corev1.ResourceEphemeralStorage: ephemeralLim}}, VolumeMounts: []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}, {Name: "tmp", MountPath: "/tmp"}}}}, Volumes: []corev1.Volume{{Name: "workspace", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc}}}, {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &tmpLim}}}}}} + created, err := pods.Create(ctx, pod, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + created, err = pods.Get(ctx, name, metav1.GetOptions{}) + } + if err != nil { + return nil, err + } + return created, validatePod(env, p, pvc, created) +} + +func validatePVC(env *unstructured.Unstructured, p resolvedProfile, pvc *corev1.PersistentVolumeClaim) error { + storedUID := textNested(env.Object, "status", "pvc", "uid") + qty, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + if pvc.Labels["execution.mecatl.dev/environment"] != env.GetName() || pvc.Labels["execution.mecatl.dev/revision"] != textNested(env.Object, "spec", "revision") || pvc.Labels["execution.mecatl.dev/allocation-uid"] != string(env.GetUID()) || len(pvc.OwnerReferences) != 0 || (storedUID != "" && storedUID != string(pvc.UID)) || pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName != p.Spec.StorageClass || len(pvc.Spec.AccessModes) != 1 || pvc.Spec.AccessModes[0] != corev1.ReadWriteOnce || pvc.Spec.VolumeMode == nil || *pvc.Spec.VolumeMode != corev1.PersistentVolumeFilesystem || !ok || qty.Cmp(p.StorageSize) != 0 { + return errors.New("PVC ownership or immutable specification mismatch") + } + return nil +} + +func validatePod(env *unstructured.Unstructured, p resolvedProfile, pvcName string, pod *corev1.Pod) error { //nolint:gocyclo // Every security-sensitive immutable field is checked explicitly. + if pod.DeletionTimestamp != nil { + return errors.New("pod is terminating") + } + storedUID := textNested(env.Object, "status", "pod", "uid") + if pod.Labels["execution.mecatl.dev/environment"] != env.GetName() || pod.Labels["execution.mecatl.dev/profile"] != hashText(textNested(env.Object, "spec", "profile"))[:16] || (storedUID != "" && storedUID != string(pod.UID)) || len(pod.OwnerReferences) != 1 { + return errors.New("pod ownership identity mismatch") + } + owner := pod.OwnerReferences[0] + if owner.APIVersion != env.GetAPIVersion() || owner.Kind != env.GetKind() || owner.Name != env.GetName() || owner.UID != env.GetUID() || owner.Controller == nil || !*owner.Controller || !reflect.DeepEqual(pod.Finalizers, []string{executorFinalizer}) { + return errors.New("pod controller ownership mismatch") + } + if pod.Spec.RestartPolicy != corev1.RestartPolicyNever || pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken || pod.Spec.RuntimeClassName == nil || *pod.Spec.RuntimeClassName != p.Spec.RuntimeClassName || len(pod.Spec.Containers) != 1 || len(pod.Spec.InitContainers) != 0 || len(pod.Spec.EphemeralContainers) != 0 || len(pod.Spec.Volumes) != 2 { + return errors.New("pod immutable specification mismatch") + } + psc := pod.Spec.SecurityContext + if psc == nil || psc.RunAsNonRoot == nil || !*psc.RunAsNonRoot || psc.RunAsUser == nil || *psc.RunAsUser != 65532 || psc.RunAsGroup == nil || *psc.RunAsGroup != 65532 || psc.FSGroup == nil || *psc.FSGroup != 65532 || psc.SeccompProfile == nil || psc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + return errors.New("pod security context mismatch") + } + c := pod.Spec.Containers[0] + if c.Name != "executor" || c.Image != p.Spec.Image || !reflect.DeepEqual(c.Command, []string{"/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"}) || !reflect.DeepEqual(c.VolumeMounts, []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}, {Name: "tmp", MountPath: "/tmp"}}) { + return errors.New("executor container specification mismatch") + } + cs := c.SecurityContext + if cs == nil || cs.AllowPrivilegeEscalation == nil || *cs.AllowPrivilegeEscalation || cs.ReadOnlyRootFilesystem == nil || !*cs.ReadOnlyRootFilesystem || cs.RunAsNonRoot == nil || !*cs.RunAsNonRoot || cs.RunAsUser == nil || *cs.RunAsUser != 65532 || cs.Capabilities == nil || !reflect.DeepEqual(cs.Capabilities.Drop, []corev1.Capability{"ALL"}) { + return errors.New("executor container security mismatch") + } + wantRequests := corev1.ResourceList{corev1.ResourceCPU: p.CPURequest, corev1.ResourceMemory: p.MemoryRequest, corev1.ResourceEphemeralStorage: p.EphemeralStorageRequest} + wantLimits := corev1.ResourceList{corev1.ResourceCPU: p.CPULimit, corev1.ResourceMemory: p.MemoryLimit, corev1.ResourceEphemeralStorage: p.EphemeralStorageLimit} + if !reflect.DeepEqual(c.Resources.Requests, wantRequests) || !reflect.DeepEqual(c.Resources.Limits, wantLimits) || pod.Spec.Volumes[0].Name != "workspace" || pod.Spec.Volumes[0].PersistentVolumeClaim == nil || pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName != pvcName || pod.Spec.Volumes[1].Name != "tmp" || pod.Spec.Volumes[1].EmptyDir == nil || pod.Spec.Volumes[1].EmptyDir.SizeLimit == nil || pod.Spec.Volumes[1].EmptyDir.SizeLimit.Cmp(p.TmpSizeLimit) != 0 { + return errors.New("executor resources or volumes mismatch") + } + return nil +} + +func (r *Reconciler) reconcileDeletion(ctx context.Context, _ dynamic.ResourceInterface, env *unstructured.Unstructured) error { + return r.setCondition(ctx, env, "DeletionBlocked", true, "ExactLifecycleRequired", "deletion requires the exact retained-environment lifecycle operation") +} +func (r *Reconciler) updateRuntimeStatus(ctx context.Context, env *unstructured.Unstructured, pvc *corev1.PersistentVolumeClaim, pod *corev1.Pod, ready bool) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + if !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pvc.Name, "uid": string(pvc.UID)}, "status", "pvc") + _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pod.Name, "uid": string(pod.UID)}, "status", "pod") + _ = unstructured.SetNestedField(o.Object, o.GetGeneration(), "status", "observedGeneration") + setConditionObject(o, "Ready", ready, "Reconciled", map[bool]string{true: "PVC and executor Pod are ready", false: "waiting for executor Pod readiness"}[ready]) + return nil + }) +} +func (r *Reconciler) setFenceUnknown(ctx context.Context, env *unstructured.Unstructured, msg string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + if !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + _ = unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") + setConditionObject(o, "Ready", false, "FenceUnknown", msg) + return nil + }) +} +func (r *Reconciler) setCondition(ctx context.Context, env *unstructured.Unstructured, name string, status bool, reason, msg string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { setConditionObject(o, name, status, reason, msg); return nil }) +} +func (r *Reconciler) updateStatus(ctx context.Context, env *unstructured.Unstructured, fn func(*unstructured.Unstructured) error) error { + res := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace) + for i := 0; i < 5; i++ { + cur, err := res.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + return err + } + before, _, _ := unstructured.NestedMap(cur.Object, "status") + if err := fn(cur); err != nil { + return err + } + after, _, _ := unstructured.NestedMap(cur.Object, "status") + if reflect.DeepEqual(before, after) { + return nil + } + if _, err = res.UpdateStatus(ctx, cur, metav1.UpdateOptions{}); err == nil { + return nil + } else if !apierrors.IsConflict(err) { + return err + } + } + return errors.New("status update conflict limit exceeded") +} +func setConditionObject(o *unstructured.Unstructured, name string, status bool, reason, msg string) { + conds, _, _ := unstructured.NestedSlice(o.Object, "status", "conditions") + st := "False" + if status { + st = "True" + } + replacement := map[string]any{"type": name, "status": st, "reason": reason, "message": msg, "observedGeneration": o.GetGeneration()} + next := make([]any, 0, len(conds)+1) + replaced := false + for _, value := range conds { + condition, ok := value.(map[string]any) + if !ok || text(condition, "type") != name { + next = append(next, value) + continue + } + if replaced { + continue + } + transitionTime := text(condition, "lastTransitionTime") + if text(condition, "status") != st || transitionTime == "" { + transitionTime = time.Now().UTC().Format(time.RFC3339) + } + replacement["lastTransitionTime"] = transitionTime + next = append(next, replacement) + replaced = true + } + if !replaced { + replacement["lastTransitionTime"] = time.Now().UTC().Format(time.RFC3339) + next = append(next, replacement) + } + _ = unstructured.SetNestedSlice(o.Object, next, "status", "conditions") +} +func podReady(p *corev1.Pod) bool { + if p == nil || p.DeletionTimestamp != nil { + return false + } + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + return true + } + } + return false +} +func resourceName(prefix, env string) string { + name := prefix + "-" + strings.TrimPrefix(env, "exec-") + if len(name) > 63 { + name = name[:63] + } + return strings.TrimRight(name, "-") +} diff --git a/internal/adapter/executioncontroller/controller_lifecycle.go b/internal/adapter/executioncontroller/controller_lifecycle.go new file mode 100644 index 0000000000..644c49cadf --- /dev/null +++ b/internal/adapter/executioncontroller/controller_lifecycle.go @@ -0,0 +1,302 @@ +package executioncontroller + +import ( + "context" + "errors" + "reflect" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +const deleteRetiredEnvironment = "DeleteRetiredEnvironment" + +func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.Unstructured) error { //nolint:gocyclo // Durable phases are deliberately explicit. + op, found, err := unstructured.NestedMap(env.Object, "status", "lifecycleOperation") + if err != nil || !found { + return nil + } + kind, phase := text(op, "type"), text(op, "phase") + operationID := text(op, "id") + podUID, pvcUID := text(op, "expectedPodUID"), text(op, "expectedPVCUID") + podName, pvcName := textNested(env.Object, "status", "pod", "name"), textNested(env.Object, "status", "pvc", "name") + if operationID == "" || pvcUID == "" { + return r.setLifecycleFenceUnknown(ctx, env, "lifecycle operation identity is malformed") + } + proofMatches := textNested(env.Object, "status", "terminationProof", operationIDField) == operationID && textNested(env.Object, "status", "terminationProof", "podUID") == podUID && textNested(env.Object, "status", "terminationProof", "pvcUID") == pvcUID && intNested(env.Object, "status", "terminationProof", "epoch") == intNested(op, "expectedEpoch") + podIdentityMatches := textNested(env.Object, "status", "pod", "uid") == podUID + if phase == "CreatingReplacement" { + podIdentityMatches = textNested(env.Object, "status", "pod", "uid") == "" && proofMatches + } + if kind == deleteRetiredEnvironment { + return r.reconcileRetainedDelete(ctx, env, op, pvcName, pvcUID) + } + epoch := intNested(op, "expectedEpoch") + if podUID == "" || epoch <= 0 || !podIdentityMatches || textNested(env.Object, "status", "pvc", "uid") != pvcUID || intNested(env.Object, "status", "epoch") != epoch { + return r.setLifecycleFenceUnknown(ctx, env, "lifecycle operation no longer matches the exact executor and workspace") + } + pvc, err := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil || string(pvc.UID) != pvcUID { + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return r.setLifecycleFenceUnknown(ctx, env, "authoritative workspace identity is unavailable") + } + switch phase { + case "Quiescing": + pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if getErr != nil || string(pod.UID) != podUID { + if getErr != nil && !apierrors.IsNotFound(getErr) { + return getErr + } + return r.setLifecycleFenceUnknown(ctx, env, "executor disappeared before durable terminal proof") + } + uid := types.UID(podUID) + grace := int64(30) + if deleteErr := r.kube.CoreV1().Pods(r.namespace).Delete(ctx, podName, metav1.DeleteOptions{GracePeriodSeconds: &grace, Preconditions: &metav1.Preconditions{UID: &uid}}); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return deleteErr + } + return r.setLifecyclePhase(ctx, env, operationID, "Quiescing", "WaitingForTermination") + case "WaitingForTermination": + pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if apierrors.IsNotFound(getErr) { + return r.setLifecycleFenceUnknown(ctx, env, "executor disappeared before durable terminal proof") + } + if getErr != nil { + return getErr + } + if string(pod.UID) != podUID { + return r.setLifecycleFenceUnknown(ctx, env, "executor UID changed during quiescing") + } + if !podTerminal(pod) { + return r.setLifecycleCondition(ctx, env, "Ready", false, "AwaitingTerminalExecutor", "waiting for kubelet terminal phase and terminated container states") + } + q := adminLifecycleRequest{ExpectedEpoch: uint64(epoch), ExpectedPodUID: podUID, ExpectedPVCUID: pvcUID, OperationID: operationID} //nolint:gosec // positivity checked above. + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + current, ok, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if !ok || text(current, "id") != operationID || text(current, "phase") != phase || !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + if err := unstructured.SetNestedMap(o.Object, terminationProof(q, pod), "status", "terminationProof"); err != nil { + return err + } + current["phase"] = "RemovingPodFinalizer" + return unstructured.SetNestedMap(o.Object, current, "status", "lifecycleOperation") + }) + case "RemovingPodFinalizer": + if !proofMatches { + return r.setLifecycleFenceUnknown(ctx, env, "durable terminal proof is missing or mismatched") + } + pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if getErr == nil { + if string(pod.UID) != podUID || !podTerminal(pod) { + return r.setLifecycleFenceUnknown(ctx, env, "terminal executor identity changed before finalizer removal") + } + pod = pod.DeepCopy() + pod.Finalizers = withoutString(pod.Finalizers, executorFinalizer) + if _, updateErr := r.kube.CoreV1().Pods(r.namespace).Update(ctx, pod, metav1.UpdateOptions{}); updateErr != nil { + return updateErr + } + } else if !apierrors.IsNotFound(getErr) { + return getErr + } + return r.setLifecyclePhase(ctx, env, operationID, "RemovingPodFinalizer", "WaitingForPodDeletion") + case "WaitingForPodDeletion": + if !proofMatches { + return r.setLifecycleFenceUnknown(ctx, env, "durable terminal proof is missing or mismatched") + } + pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if getErr == nil { + if string(pod.UID) != podUID { + return r.setLifecycleFenceUnknown(ctx, env, "unexpected executor occupies the retained name") + } + uid := types.UID(podUID) + if deleteErr := r.kube.CoreV1().Pods(r.namespace).Delete(ctx, podName, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return deleteErr + } + return nil + } + if !apierrors.IsNotFound(getErr) { + return getErr + } + if kind == "RetireEnvironment" { + return r.finishRetirement(ctx, env, operationID) + } + return r.prepareReplacement(ctx, env, operationID) + case "CreatingReplacement": + return r.finishReplacement(ctx, env, operationID) + default: + return r.setLifecycleFenceUnknown(ctx, env, "unknown lifecycle phase") + } +} + +func (r *Reconciler) setLifecyclePhase(ctx context.Context, env *unstructured.Unstructured, operationID, expectedPhase, nextPhase string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + op, found, err := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if err != nil || !found || text(op, "id") != operationID || text(op, "phase") != expectedPhase || !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + op["phase"] = nextPhase + return unstructured.SetNestedMap(o.Object, op, "status", "lifecycleOperation") + }) +} + +func (r *Reconciler) finishRetirement(ctx context.Context, env *unstructured.Unstructured, operationID string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + op, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if !found || text(op, "id") != operationID || text(op, "type") != "RetireEnvironment" || text(op, "phase") != "WaitingForPodDeletion" || !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + unstructured.RemoveNestedField(o.Object, "status", "lifecycleOperation") + setConditionObject(o, "ExecutorTerminated", true, "TerminalPodProof", "terminal executor proof is durable") + setConditionObject(o, "Retired", true, "WorkspaceRetained", "executor retired; workspace PVC retained") + setConditionObject(o, "Ready", false, "Retained", "environment is retired with retained storage") + return nil + }) +} + +func (r *Reconciler) prepareReplacement(ctx context.Context, env *unstructured.Unstructured, operationID string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + op, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if !found || text(op, "id") != operationID || text(op, "type") != "ReplaceExecutor" || text(op, "phase") != "WaitingForPodDeletion" || !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + unstructured.RemoveNestedField(o.Object, "status", "pod") + op["phase"] = "CreatingReplacement" + return unstructured.SetNestedMap(o.Object, op, "status", "lifecycleOperation") + }) +} + +func (r *Reconciler) finishReplacement(ctx context.Context, env *unstructured.Unstructured, operationID string) error { + profile, ok := r.profiles.get(textNested(env.Object, "spec", "profile")) + if !ok || profile.Digest != textNested(env.Object, "spec", "profileDigest") { + return r.setLifecycleFenceUnknown(ctx, env, "replacement profile is unavailable") + } + pvcName := textNested(env.Object, "status", "pvc", "name") + pod, err := r.ensurePod(ctx, env, profile, resourceName("executor", env.GetName()), pvcName) + if err != nil { + return err + } + if !podReady(pod) { + return r.setLifecycleCondition(ctx, env, "Ready", false, "ReplacementStarting", "waiting for replacement executor readiness") + } + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + op, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if !found || text(op, "id") != operationID || text(op, "type") != "ReplaceExecutor" || text(op, "phase") != "CreatingReplacement" || !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + epoch := intNested(o.Object, "status", "epoch") + if epoch <= 0 { + return errors.New("execution epoch cannot advance") + } + _ = unstructured.SetNestedField(o.Object, epoch+1, "status", "epoch") + _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pod.Name, "uid": string(pod.UID)}, "status", "pod") + // Completed migration receipts are bound to the runtime being replaced. + unstructured.RemoveNestedField(o.Object, "status", "lastMigrationOperationID") + unstructured.RemoveNestedField(o.Object, "status", "lastMigrationFromSchema") + _ = unstructured.SetNestedMap(o.Object, map[string]any{operationIDField: operationID, "previousPodUID": text(op, "expectedPodUID"), "replacementPodUID": string(pod.UID), "pvcUID": text(op, "expectedPVCUID"), "previousEpoch": intNested(op, "expectedEpoch"), "replacementEpoch": epoch + 1}, "status", "lastReplacement") + unstructured.RemoveNestedField(o.Object, "status", "lifecycleOperation") + setConditionObject(o, "Ready", true, "ReplacementReady", "replacement executor is ready on the retained workspace") + return nil + }) +} + +func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructured.Unstructured, op map[string]any, pvcName, pvcUID string) error { //nolint:gocyclo // Exact retained-deletion phases deliberately fail closed at each identity check. + phase := text(op, "phase") + if (phase != "DeletingPVC" && phase != "ReleasingSlot") || !conditionTrue(env, "Retired") || !conditionTrue(env, "ExecutorTerminated") { + return r.setLifecycleFenceUnknown(ctx, env, "retained deletion state is invalid") + } + pvc, err := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err == nil { + if phase != "DeletingPVC" { + return r.setLifecycleFenceUnknown(ctx, env, "retained PVC reappeared after deallocation began") + } + if string(pvc.UID) != pvcUID { + return r.setLifecycleFenceUnknown(ctx, env, "retained PVC UID changed") + } + uid := types.UID(pvcUID) + if deleteErr := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Delete(ctx, pvcName, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return deleteErr + } + return nil + } + if !apierrors.IsNotFound(err) { + return err + } + if phase == "DeletingPVC" { + return r.setLifecyclePhase(ctx, env, text(op, "id"), "DeletingPVC", "ReleasingSlot") + } + res := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace) + current, err := res.Get(ctx, env.GetName(), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + currentOp, found, _ := unstructured.NestedMap(current.Object, "status", "lifecycleOperation") + if current.GetUID() != env.GetUID() || !found || text(currentOp, "id") != text(op, "id") || text(currentOp, "type") != text(op, "type") || text(currentOp, "phase") != "ReleasingSlot" || !runtimeObservationMatches(current, env) { + return lifecycleConflict() + } + if err := releaseProfileSlot(ctx, r.kube, r.namespace, textNested(current.Object, "spec", "profile"), current.GetName()); err != nil { + return err + } + current.SetFinalizers(withoutString(current.GetFinalizers(), environmentFinalizer)) + updated, err := res.Update(ctx, current, metav1.UpdateOptions{}) + if err != nil { + return err + } + uid, version := updated.GetUID(), updated.GetResourceVersion() + if err := res.Delete(ctx, updated.GetName(), metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &version}}); err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +func lifecycleConflict() error { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "lifecycle operation changed concurrently"} +} + +func runtimeObservationMatches(current, observed *unstructured.Unstructured) bool { + currentOp, currentHasOp, _ := unstructured.NestedMap(current.Object, "status", "lifecycleOperation") + observedOp, observedHasOp, _ := unstructured.NestedMap(observed.Object, "status", "lifecycleOperation") + return currentHasOp == observedHasOp && reflect.DeepEqual(currentOp, observedOp) && + intNested(current.Object, "status", "epoch") == intNested(observed.Object, "status", "epoch") && + textNested(current.Object, "status", "pod", "uid") == textNested(observed.Object, "status", "pod", "uid") && + textNested(current.Object, "status", "pvc", "uid") == textNested(observed.Object, "status", "pvc", "uid") +} + +func (r *Reconciler) setLifecycleCondition(ctx context.Context, env *unstructured.Unstructured, name string, status bool, reason, msg string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + if !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + setConditionObject(o, name, status, reason, msg) + return nil + }) +} + +func (r *Reconciler) setLifecycleFenceUnknown(ctx context.Context, env *unstructured.Unstructured, msg string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + if !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + _ = unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") + setConditionObject(o, "Ready", false, "FenceUnknown", msg) + return nil + }) +} + +func withoutString(values []string, remove string) []string { + out := make([]string, 0, len(values)) + for _, value := range values { + if value != remove { + out = append(out, value) + } + } + return out +} diff --git a/internal/adapter/executioncontroller/controller_test.go b/internal/adapter/executioncontroller/controller_test.go new file mode 100644 index 0000000000..fe80cb70dd --- /dev/null +++ b/internal/adapter/executioncontroller/controller_test.go @@ -0,0 +1,412 @@ +package executioncontroller + +import ( + "context" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + nodev1 "k8s.io/api/node/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/util/workqueue" + clocktesting "k8s.io/utils/clock/testing" +) + +func TestReconcileRefusesForeignExistingPVCWithoutPersistingUID(t *testing.T) { + ctx := context.Background() + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + foreign := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "workspace-test", Namespace: "ns", UID: types.UID("foreign"), Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/revision": "rev"}}} + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(foreign) + r := NewReconciler(d, k, "ns", testProfiles()) + if err := r.Reconcile(ctx, env.GetName()); err == nil { + t.Fatal("foreign PVC ownership mismatch was not returned") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if uid := textNested(got.Object, "status", "pvc", "uid"); uid != "" { + t.Fatalf("foreign PVC UID persisted as authoritative: %q", uid) + } + pods, err := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if err != nil || len(pods.Items) != 0 { + t.Fatalf("executor created over foreign PVC: pods=%d err=%v", len(pods.Items), err) + } +} + +func TestReconcileCreatesTokenlessNonRootPodAndRetainedPVC(t *testing.T) { + ctx := context.Background() + env := testEnvironment() + scheme := runtime.NewScheme() + d := dynamicfake.NewSimpleDynamicClient(scheme, env) + k := kubefake.NewSimpleClientset() + r := NewReconciler(d, k, "ns", testProfiles()) + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + pvcs, _ := k.CoreV1().PersistentVolumeClaims("ns").List(ctx, metav1.ListOptions{}) + if len(pvcs.Items) != 1 { + t.Fatalf("pvcs=%d", len(pvcs.Items)) + } + if len(pvcs.Items[0].OwnerReferences) != 0 { + t.Fatal("PVC must not have owner reference") + } + pods, _ := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if len(pods.Items) != 1 { + t.Fatalf("pods=%d", len(pods.Items)) + } + p := pods.Items[0] + if p.Spec.AutomountServiceAccountToken == nil || *p.Spec.AutomountServiceAccountToken { + t.Fatal("service account token mounted") + } + if p.Spec.SecurityContext == nil || p.Spec.SecurityContext.RunAsNonRoot == nil || !*p.Spec.SecurityContext.RunAsNonRoot { + t.Fatal("pod is not non-root") + } + c := p.Spec.Containers[0] + if c.SecurityContext == nil || c.SecurityContext.ReadOnlyRootFilesystem == nil || !*c.SecurityContext.ReadOnlyRootFilesystem || c.SecurityContext.AllowPrivilegeEscalation == nil || *c.SecurityContext.AllowPrivilegeEscalation { + t.Fatal("container security context is not hardened") + } + if p.Spec.RuntimeClassName == nil || *p.Spec.RuntimeClassName != "sandboxed" || c.Resources.Requests.Cpu().IsZero() || c.Resources.Requests.Memory().IsZero() || c.Resources.Requests.StorageEphemeral().IsZero() || c.Resources.Limits.StorageEphemeral().IsZero() { + t.Fatal("runtime class or resource bounds are missing") + } + if p.Spec.Volumes[1].EmptyDir == nil || p.Spec.Volumes[1].EmptyDir.SizeLimit == nil || p.Spec.Volumes[1].EmptyDir.SizeLimit.String() != "256Mi" { + t.Fatal("tmp volume is not profile-bounded") + } +} +func TestTerminatingPodIsUnavailableDuringReconcile(t *testing.T) { + ctx := t.Context() + env := testEnvironment() + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + r := NewReconciler(d, k, "ns", testProfiles()) + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + pod, err := k.CoreV1().Pods("ns").Get(ctx, "executor-test", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + now := metav1.Now() + pod.DeletionTimestamp = &now + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if _, err := k.CoreV1().Pods("ns").Update(ctx, pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err == nil { + t.Fatal("terminating Pod was not rejected") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if conditionTrue(got, "Ready") { + t.Fatalf("terminating pod remained ready: %v", got.Object["status"]) + } +} + +func TestRepeatedReconcileDoesNotRewriteUnchangedStatus(t *testing.T) { + ctx := context.Background() + env := testEnvironment() + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + r := NewReconciler(d, k, "ns", testProfiles()) + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + writes := statusUpdateCount(d.Actions()) + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + transition := conditionTransition(got, "Ready") + if transition == "" { + t.Fatal("Ready condition has no transition timestamp") + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if extra := statusUpdateCount(d.Actions()) - writes; extra != 0 { + t.Fatalf("unchanged reconcile performed %d status writes", extra) + } + got, err = d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if current := conditionTransition(got, "Ready"); current != transition { + t.Fatalf("lastTransitionTime changed from %q to %q", transition, current) + } + + pods, err := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if err != nil || len(pods.Items) != 1 { + t.Fatalf("pods=%d err=%v", len(pods.Items), err) + } + pod := pods.Items[0].DeepCopy() + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if _, err := k.CoreV1().Pods("ns").UpdateStatus(ctx, pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + got, err = d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !conditionTrue(got, "Ready") { + t.Fatalf("Ready did not follow Pod readiness: %v", got.Object["status"]) + } + if statusUpdateCount(d.Actions()) != writes+1 { + t.Fatalf("status writes=%d, want %d", statusUpdateCount(d.Actions()), writes+1) + } +} + +func TestActiveOperationExpiryRequeuesWithoutAnotherEvent(t *testing.T) { + ctx := t.Context() + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + kubeClient := kubefake.NewSimpleClientset() + r := NewReconciler(dynamicClient, kubeClient, "ns", testProfiles()) + initialQueue := r.queue + t.Cleanup(initialQueue.ShutDown) + + // Establish a fully reconciled, ready environment so only operation expiry can + // cause the transition below. + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + pod, err := kubeClient.CoreV1().Pods("ns").Get(ctx, "executor-test", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if _, err := kubeClient.CoreV1().Pods("ns").UpdateStatus(ctx, pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + initialQueue.ShutDown() + + now := time.Date(2026, 9, 21, 12, 0, 0, 0, time.UTC) + deadline := now.Add(100 * time.Millisecond) + current, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedMap(current.Object, map[string]any{ + "id": "operation", "expiresAt": deadline.Format(time.RFC3339Nano), + }, "status", "activeOperation"); err != nil { + t.Fatal(err) + } + if _, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(ctx, current, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + + fakeClock := clocktesting.NewFakeClock(now) + delayingQueue := workqueue.NewTypedDelayingQueueWithConfig[string](workqueue.TypedDelayingQueueConfig[string]{Clock: fakeClock}) + replacementQueue := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[string](), + workqueue.TypedRateLimitingQueueConfig[string]{DelayingQueue: delayingQueue}, + ) + t.Cleanup(replacementQueue.ShutDown) + r.queue = replacementQueue + r.now = fakeClock.Now + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + beforeExpiry, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !conditionTrue(beforeExpiry, "Ready") || textNested(beforeExpiry.Object, "status", "fenceState") != fenceHealthy { + t.Fatalf("future operation lease disturbed ready state: %v", beforeExpiry.Object["status"]) + } + + waitUntil := time.Now().Add(time.Second) + for fakeClock.Waiters() < 2 && time.Now().Before(waitUntil) { + time.Sleep(time.Millisecond) + } + if fakeClock.Waiters() < 2 { + t.Fatal("reconcile did not register the operation-expiry deadline with the delayed queue") + } + + done := make(chan struct{}) + go func() { + r.worker(ctx) + close(done) + }() + t.Cleanup(func() { + replacementQueue.ShutDown() + <-done + }) + + fakeClock.Step(99 * time.Millisecond) + preDeadline, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !conditionTrue(preDeadline, "Ready") || textNested(preDeadline.Object, "status", "fenceState") != fenceHealthy { + t.Fatalf("operation fenced before its deadline: %v", preDeadline.Object["status"]) + } + + fakeClock.Step(time.Millisecond) + workerDeadline := time.Now().Add(2 * time.Second) + for time.Now().Before(workerDeadline) { + got, getErr := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if getErr != nil { + t.Fatal(getErr) + } + if textNested(got.Object, "status", "fenceState") == "FenceUnknown" { + if textNested(got.Object, "status", "activeOperation", "id") != "operation" || conditionTrue(got, "Ready") { + t.Fatalf("expiry did not retain unresolved operation identity and clear readiness: %v", got.Object["status"]) + } + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("worker did not reconcile the active-operation deadline without another Kubernetes event") +} + +func TestRuntimeAndRelevantStatusUpdatesEnqueue(t *testing.T) { + r := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kubefake.NewSimpleClientset(), "ns", testProfiles()) + r.enqueueRuntime(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"execution.mecatl.dev/environment": "exec-pod"}}}) + r.enqueueRuntime(&corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"execution.mecatl.dev/environment": "exec-pvc"}}}) + if r.queue.Len() != 2 { + t.Fatalf("runtime queue length=%d", r.queue.Len()) + } + + oldEnv := testEnvironment() + newEnv := oldEnv.DeepCopy() + _ = unstructured.SetNestedField(newEnv.Object, "op", "status", "activeOperation", "id") + r.enqueueUpdate(oldEnv, newEnv) + if r.queue.Len() != 3 { + t.Fatalf("relevant status update was not queued: len=%d", r.queue.Len()) + } + lifecycle := newEnv.DeepCopy() + _ = unstructured.SetNestedMap(lifecycle.Object, map[string]any{"id": "retire", "phase": "Quiescing"}, "status", "lifecycleOperation") + lifecycleQueue := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kubefake.NewSimpleClientset(), "ns", testProfiles()) + lifecycleQueue.enqueueUpdate(newEnv, lifecycle) + if lifecycleQueue.queue.Len() != 1 { + t.Fatalf("lifecycle start was not queued: len=%d", lifecycleQueue.queue.Len()) + } + advanced := lifecycle.DeepCopy() + _ = unstructured.SetNestedField(advanced.Object, "WaitingForTermination", "status", "lifecycleOperation", "phase") + phaseQueue := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kubefake.NewSimpleClientset(), "ns", testProfiles()) + phaseQueue.enqueueUpdate(lifecycle, advanced) + if phaseQueue.queue.Len() != 1 { + t.Fatalf("lifecycle phase change was not queued: len=%d", phaseQueue.queue.Len()) + } + controllerStatus := advanced.DeepCopy() + _ = unstructured.SetNestedField(controllerStatus.Object, "pod", "status", "pod", "name") + r.enqueueUpdate(advanced, controllerStatus) + if r.queue.Len() != 3 { + t.Fatalf("controller-owned status update was queued: len=%d", r.queue.Len()) + } +} + +func statusUpdateCount(actions []k8stesting.Action) int { + count := 0 + for _, action := range actions { + if action.GetVerb() == "update" && action.GetSubresource() == "status" { + count++ + } + } + return count +} + +func conditionTransition(env *unstructured.Unstructured, name string) string { + conditions, _, _ := unstructured.NestedSlice(env.Object, "status", "conditions") + for _, value := range conditions { + condition, ok := value.(map[string]any) + if ok && text(condition, "type") == name { + return text(condition, "lastTransitionTime") + } + } + return "" +} + +func TestStartupDoesNotFenceLivePeerOperation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + env := testEnvironment() + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "old", "operation": "file.replace"}, "status", "activeOperation") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + r := NewReconciler(d, profileResourceClient(), "ns", testProfiles()) + peer := NewReconciler(d, profileResourceClient(), "ns", testProfiles()) + if r.Ready() || peer.Ready() { + t.Fatal("reconciler reported ready before cache synchronization") + } + if err := r.Initialize(ctx); err != nil { + t.Fatal(err) + } + if err := peer.Initialize(ctx); err != nil { + t.Fatal(err) + } + if !r.Ready() || !peer.Ready() { + t.Fatal("reconcilers did not report ready after cache synchronization") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(got.Object, "status", "fenceState") != "Healthy" || textNested(got.Object, "status", "activeOperation", "id") != "old" { + t.Fatalf("startup mutated a potentially live peer operation: status=%v", got.Object["status"]) + } +} +func TestInitializeRefusesMissingRuntimeClassBeforeCreatingPods(t *testing.T) { + kube := kubefake.NewSimpleClientset(&storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "standard"}}) + r := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kube, "ns", testProfiles()) + err := r.Initialize(t.Context()) + if err == nil || !strings.Contains(err.Error(), `profile preflight: RuntimeClass "sandboxed" unavailable`) { + t.Fatalf("Initialize error = %v", err) + } + if r.Ready() { + t.Fatal("reconciler reported ready after failed profile preflight") + } + for _, action := range kube.Actions() { + if action.GetVerb() == "create" && action.GetResource().Resource == "pods" { + t.Fatal("profile preflight created a Pod") + } + } +} + +func profileResourceClient() *kubefake.Clientset { + return kubefake.NewSimpleClientset( + &nodev1.RuntimeClass{ObjectMeta: metav1.ObjectMeta{Name: "sandboxed"}}, + &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "standard"}}, + ) +} + +func testProfiles() *Profiles { + spec := ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100} + profile, err := validateProfile("go", spec) + if err != nil { + panic(err) + } + profile.Spec = spec + profile.Digest = "sha256:profile" + return &Profiles{byName: map[string]resolvedProfile{"go": profile}} +} +func testEnvironment() *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "exec-test", "namespace": "ns", "uid": string(types.UID("uid"))}, "spec": map[string]any{"schemaVersion": int64(2), "profile": "go", "profileDigest": "sha256:profile", "revision": "rev", "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "references": []any{}, "fenceState": "Healthy"}}} +} diff --git a/internal/adapter/executioncontroller/crd_schema_test.go b/internal/adapter/executioncontroller/crd_schema_test.go new file mode 100644 index 0000000000..4ccfed4d8f --- /dev/null +++ b/internal/adapter/executioncontroller/crd_schema_test.go @@ -0,0 +1,90 @@ +package executioncontroller + +import ( + "os" + "testing" + + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsvalidation "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning" + apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +func loadExecutionEnvironmentCRD(t *testing.T) (*apiextensionsv1.CustomResourceDefinition, *apiextensions.CustomResourceDefinition) { + t.Helper() + raw, err := os.ReadFile("../../../deploy/helm/mecatl-execution/crds/executionenvironment.yaml") + if err != nil { + t.Fatal(err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(raw, &crd); err != nil { + t.Fatal(err) + } + apiextensionsv1.SetDefaults_CustomResourceDefinition(&crd) + var internal apiextensions.CustomResourceDefinition + if err := apiextensionsv1.Convert_v1_CustomResourceDefinition_To_apiextensions_CustomResourceDefinition(&crd, &internal, nil); err != nil { + t.Fatal(err) + } + return &crd, &internal +} + +func TestExecutionEnvironmentCRDIsAcceptedByKubernetesValidation(t *testing.T) { + _, internal := loadExecutionEnvironmentCRD(t) + if errs := apiextensionsvalidation.ValidateCustomResourceDefinition(t.Context(), internal); len(errs) != 0 { + t.Fatalf("CRD is not installable: %v", errs.ToAggregate()) + } +} + +func TestMigrationReceiptSchemaAdmission(t *testing.T) { + crd, _ := loadExecutionEnvironmentCRD(t) + field := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"].Properties["lastMigrationFromSchema"] + var internal apiextensions.JSONSchemaProps + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(&field, &internal, nil); err != nil { + t.Fatal(err) + } + validator, _, err := apiservervalidation.NewSchemaValidator(&internal) + if err != nil { + t.Fatal(err) + } + for _, value := range []any{int64(0), int64(1), int64(-1), int64(2), "0"} { + errs := apiservervalidation.ValidateCustomResource(nil, value, validator) + valid := value == int64(0) || value == int64(1) + if (len(errs) == 0) != valid { + t.Fatalf("receipt source schema %v: valid=%t errors=%v", value, valid, errs) + } + } +} + +func TestExecutionEnvironmentCRDPreservesControllerStatus(t *testing.T) { + crd, _ := loadExecutionEnvironmentCRD(t) + var internal apiextensions.JSONSchemaProps + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(crd.Spec.Versions[0].Schema.OpenAPIV3Schema, &internal, nil); err != nil { + t.Fatal(err) + } + structural, err := schema.NewStructural(&internal) + if err != nil { + t.Fatal(err) + } + obj := map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env"}, "spec": map[string]any{}, "status": map[string]any{ + "schemaVersion": int64(2), "observedGeneration": int64(9), "epoch": int64(3), "grantGeneration": int64(4), "fenceState": "Healthy", + "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "op", "sourceBindingID": "source", "createdAt": "2026-09-17T00:00:00Z"}}, + "pvc": map[string]any{"name": "pvc", "uid": "pvc-uid"}, "pod": map[string]any{"name": "pod", "uid": "pod-uid"}, + "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "ownerHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "clientHash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "epoch": int64(3), "grantGeneration": int64(4), "expiresAt": "2026-09-17T00:01:00Z"}, + "renewReceipts": []any{map[string]any{"operationID": "renew", "fingerprint": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "expiresAt": "2026-09-17T00:01:00Z"}}, + "activeOperation": map[string]any{"id": "file", "operation": "file.read", "startedAt": "2026-09-17T00:00:00Z", "claimID": "claim", "runID": "run", "epoch": int64(3), "holderID": "holder", "renewedAt": "2026-09-17T00:00:01Z", "expiresAt": "2026-09-17T00:01:00Z"}, + "lifecycleOperation": map[string]any{"id": "life", "type": "ReplaceExecutor", "phase": "Quiescing", "expectedEpoch": int64(3), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid", "createdAt": "2026-09-17T00:00:00Z"}, + "terminationProof": map[string]any{"operationID": "life", "podUID": "pod-uid", "pvcUID": "pvc-uid", "epoch": int64(3), "podPhase": "Failed", "observedAt": "2026-09-17T00:00:00Z"}, + "migrationOperation": map[string]any{"id": "migration", "fromSchema": int64(1), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid"}, "lastMigrationOperationID": "migration", "lastMigrationFromSchema": int64(0), + "conditions": []any{map[string]any{"type": "Ready", "status": "True", "reason": "Reconciled", "message": "ready", "observedGeneration": int64(9), "lastTransitionTime": "2026-09-17T00:00:00Z"}}, + }} + pruning.Prune(obj, structural, true) + for _, path := range [][]string{{"status", "schemaVersion"}, {"status", "observedGeneration"}, {"status", "epoch"}, {"status", "grantGeneration"}, {"status", "references"}, {"status", "pvc"}, {"status", "pod"}, {"status", "activeRun", "grantGeneration"}, {"status", "renewReceipts"}, {"status", "activeOperation"}, {"status", "lifecycleOperation"}, {"status", "terminationProof"}, {"status", "migrationOperation"}, {"status", "lastMigrationOperationID"}, {"status", "lastMigrationFromSchema"}, {"status", "conditions"}} { + if _, found, err := unstructured.NestedFieldNoCopy(obj, path...); err != nil || !found { + t.Fatalf("CRD pruning removed controller field %v (found=%t err=%v)", path, found, err) + } + } +} diff --git a/internal/adapter/executioncontroller/disabled_composition_test.go b/internal/adapter/executioncontroller/disabled_composition_test.go new file mode 100644 index 0000000000..3de7bfe3da --- /dev/null +++ b/internal/adapter/executioncontroller/disabled_composition_test.go @@ -0,0 +1,95 @@ +package executioncontroller + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/internal/app" +) + +func TestDisabledCompositionPreservesAllocationsAndIndependentReconciliation(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + t.Setenv("XDG_STATE_HOME", t.TempDir()) + ctx := t.Context() + env := testEnvironment() + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewClientset() + provider := NewReconciler(d, k, "ns", testProfiles()) + // First persist the finalizer, then provision the runtime. + if err := provider.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if err := provider.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + resources := d.Resource(ExecutionEnvironmentGVR).Namespace("ns") + before, err := resources.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + pods, err := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if err != nil || len(pods.Items) != 1 { + t.Fatalf("pods=%v err=%v", pods, err) + } + pvcs, err := k.CoreV1().PersistentVolumeClaims("ns").List(ctx, metav1.ListOptions{}) + if err != nil || len(pvcs.Items) != 1 { + t.Fatalf("pvcs=%v err=%v", pvcs, err) + } + d.ClearActions() + k.ClearActions() + // No execution integration is passed to the real composition root. Existing + // runtime objects belong to the independent provider, not the harness lifetime. + built, err := app.Build(ctx, app.Config{UseMock: true, NoSoul: true, NoUserModel: true}) + if err != nil { + t.Fatal(err) + } + sess, err := built.Service.CreateSession(ctx, session.ModeDefault, session.Limits{}) + if err != nil { + built.Close() + t.Fatal(err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "offline") + if err != nil { + built.Close() + t.Fatal(err) + } + for range run.Events() { + } + built.Service.FinishRun(sess.ID, run) + built.Close() + if len(d.Actions()) != 0 || len(k.Actions()) != 0 { + t.Fatal("disabled composition touched provider resources") + } + after, err := resources.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil || !reflect.DeepEqual(before, after) { + t.Fatalf("allocation changed: %v", err) + } + afterPods, err := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if err != nil || !reflect.DeepEqual(pods.Items, afterPods.Items) { + t.Fatalf("executor changed: %v", err) + } + afterPVCs, err := k.CoreV1().PersistentVolumeClaims("ns").List(ctx, metav1.ListOptions{}) + if err != nil || !reflect.DeepEqual(pvcs.Items, afterPVCs.Items) { + t.Fatalf("storage changed: %v", err) + } + pod := pods.Items[0].DeepCopy() + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if _, err := k.CoreV1().Pods("ns").UpdateStatus(ctx, pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := provider.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + ready, err := resources.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil || !conditionTrue(ready, "Ready") { + t.Fatalf("independent provider failed to converge: %v", err) + } +} diff --git a/internal/adapter/executioncontroller/file_authorization_test.go b/internal/adapter/executioncontroller/file_authorization_test.go new file mode 100644 index 0000000000..d7558b6048 --- /dev/null +++ b/internal/adapter/executioncontroller/file_authorization_test.go @@ -0,0 +1,101 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type countedFileBackend struct { + *fakeBackend + calls int +} + +func (b *countedFileBackend) File(context.Context, string, string, executionenv.FileRequest) (executionenv.FileResponse, error) { + b.calls++ + return executionenv.FileResponse{}, nil +} + +func TestEveryFileOperationRequiresExactCurrentGrant(t *testing.T) { + public, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + const client = "spiffe://cluster/ns/mecak8s" + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + for value, name := range executionv1.FileOperation_name { + op, ok := operationFromProto(executionv1.FileOperation(value)) + if !ok { + continue + } + t.Run(name, func(t *testing.T) { + for _, failure := range []string{"client", "owner", "environment", "revision", "epoch", "operation", "expired", "revoked"} { + t.Run(failure, func(t *testing.T) { + backend := &countedFileBackend{fakeBackend: newFakeBackend()} + verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"key": public}, Issuer: "provider", Audience: "executor", MaxLifetime: time.Minute, Now: func() time.Time { return now }} + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{client: {MayAttestOwner: true}}, Verifier: verifier}, backend) + claims := executionenv.GrantClaims{KeyID: "key", Issuer: "provider", Audience: "executor", Client: client, OwnerHash: ownerHash(owner), BindingID: "binding", RunID: "run", ClaimID: "claim", Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, Epoch: 1, GrantGeneration: 1, Operations: []executionenv.Operation{op}, NotBefore: now.Add(-time.Second), ExpiresAt: now.Add(30 * time.Second), Nonce: "nonce"} + sign := func() string { + token, err := executionenv.SignGrant(private, claims) + if err != nil { + t.Fatal(err) + } + return token + } + req := &executionv1.FileRequest{Context: &executionv1.RequestContext{Environment: refToProto(claims.Environment), Owner: &executionv1.Owner{Issuer: owner.Issuer, Subject: owner.Subject}, BindingId: "binding", RunId: "run", ClaimId: "claim", Epoch: 1, GrantGeneration: 1, Grant: sign()}, Operation: executionv1.FileOperation(value), Path: "file"} + switch op { + case executionenv.OpFileReplace: + req.Version = []byte("version") + case executionenv.OpFileCopy, executionenv.OpFileRename: + req.Destination = "destination" + case executionenv.OpFileGlob: + req.Path = "" + req.Pattern = "*" + case executionenv.OpFileGrep: + req.Pattern = "text" + } + ctx := authenticatedContext(client) + if _, err := h.Files(ctx, req); err != nil || backend.calls != 1 { + t.Fatalf("positive control: calls=%d err=%v", backend.calls, err) + } + negative := proto.Clone(req).(*executionv1.FileRequest) + want := codes.PermissionDenied + switch failure { + case "client": + claims.Client = "spiffe://cluster/ns/other" + case "owner": + claims.OwnerHash = "other" + case "environment": + claims.Environment.ID = "other" + case "revision": + claims.Environment.Revision = "other" + case "epoch": + claims.Epoch++ + case "operation": + claims.Operations = []executionenv.Operation{executionenv.OpCommandStart} + case "expired": + claims.NotBefore = now.Add(-time.Minute) + claims.ExpiresAt = now.Add(-time.Second) + want = codes.Unauthenticated + case "revoked": + h.cfg.Verifier.RevokedNonces = map[string]struct{}{"nonce": {}} + } + negative.Context.Grant = sign() + if _, err := h.Files(ctx, negative); status.Code(err) != want || backend.calls != 1 { + t.Fatalf("negative control: calls=%d code=%v err=%v", backend.calls, status.Code(err), err) + } + }) + } + }) + } +} diff --git a/internal/adapter/executioncontroller/handler.go b/internal/adapter/executioncontroller/handler.go new file mode 100644 index 0000000000..a929fcfdd1 --- /dev/null +++ b/internal/adapter/executioncontroller/handler.go @@ -0,0 +1,910 @@ +// Package executioncontroller implements the authenticated provider gRPC boundary. +// +//nolint:revive // Generated gRPC method names define this private handler surface. +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "errors" + "io" + "math" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +// ClientPolicy defines capabilities assigned to an authenticated provider client. +type ClientPolicy struct { + MayAttestOwner, Administrator bool + AdministratorFor []string +} + +// GrantSigner configures short-lived environment grant issuance. +type GrantSigner struct { + KeyID string + PrivateKey ed25519.PrivateKey + Issuer, Audience string + Lifetime time.Duration +} + +// HandlerConfig configures authentication and capability grants. +type HandlerConfig struct { + Clients map[string]ClientPolicy + Signer GrantSigner + Verifier executionenv.GrantVerifier + Ready func() bool + Security *SecurityManager +} + +// Profile is the externally visible immutable execution profile. +type Profile struct { + Name, Digest string + MaxFileBytes, MaxCommandBytes int64 + MaxCommandDuration time.Duration + Capabilities []string +} + +// Allocation is an exact environment allocation and authorization binding. +type Allocation struct { + Environment executionenv.EnvironmentRef + Epoch, GrantGeneration uint64 + OwnerHash, BindingID, Client string + Ready bool +} + +// Backend implements provider-side authorization state and executor dispatch. +type Backend interface { + ValidateProfile(context.Context, string) (Profile, error) + Ensure(context.Context, string, string, string, string, string) (Allocation, error) + Attach(context.Context, executionenv.EnvironmentRef, string, string, string) (Allocation, error) + ReleaseReference(context.Context, executionenv.EnvironmentRef, string, string, string) error + File(context.Context, string, string, executionenv.FileRequest) (executionenv.FileResponse, error) + StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) + CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) + CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) +} + +type runClaimValidator interface { + ValidateRunClaim(context.Context, string, string, executionenv.RequestContext) error +} + +type lifecycleBackend interface { + EnsurePending(context.Context, string, string, string, string, string, string) (Allocation, error) + AcquireRun(context.Context, executionenv.EnvironmentRef, string, string, string, string, string, time.Duration) (executionenv.RunClaim, error) + RenewRun(context.Context, executionenv.EnvironmentRef, string, string, executionenv.RunClaimRequest) (executionenv.RunClaim, error) + ReleaseRun(context.Context, executionenv.EnvironmentRef, string, string, executionenv.RunClaimRequest) error + CommitReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + AbortReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + ReserveSuccessor(context.Context, executionenv.EnvironmentRef, string, string, string, string, string) error + PrepareReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + ConfirmReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + CancelReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) + FindReferenceIntent(context.Context, executionenv.EnvironmentRef, string, string, string) (executionenv.ReferenceIntent, error) +} + +type ownerIntentBackend interface { + EnsurePendingOwned(context.Context, string, string, executionenv.Owner, string, string, string, string) (Allocation, error) + ListReferenceIntentsForClient(context.Context, string, int) ([]executionenv.ReferenceIntent, error) +} + +type adminLifecycleBackend interface { + ReplaceExecutor(context.Context, adminLifecycleRequest) error + RetireExact(context.Context, adminLifecycleRequest) error + RecoverEnvironment(context.Context, adminLifecycleRequest) error + DeleteRetiredEnvironment(context.Context, adminLifecycleRequest) error + MigrateEnvironment(context.Context, adminLifecycleRequest) error + RevokeEnvironment(context.Context, adminLifecycleRequest, uint64) (uint64, error) +} + +// Handler is the authenticated private execution-provider gRPC service. +type Handler struct { + executionv1.UnimplementedExecutionProviderServiceServer + cfg HandlerConfig + backend Backend + lifecycleBackend lifecycleBackend + adminLifecycleBackend adminLifecycleBackend +} + +// NewHandler constructs the authenticated private gRPC service. +func NewHandler(cfg HandlerConfig, b Backend) *Handler { + lifecycle, _ := b.(lifecycleBackend) + adminLifecycle, _ := b.(adminLifecycleBackend) + return &Handler{cfg: cfg, backend: b, lifecycleBackend: lifecycle, adminLifecycleBackend: adminLifecycle} +} + +type authenticatedClient struct { + id string + policy ClientPolicy +} + +func (h *Handler) client(ctx context.Context) (authenticatedClient, error) { + if h.backend == nil || (h.cfg.Ready != nil && !h.cfg.Ready()) { + return authenticatedClient{}, wireError(executionenv.CodeNotReady, true) + } + chain, handshakeVerified, err := clientCertificates(ctx) + if err != nil || h.cfg.Security == nil && !handshakeVerified { + return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) + } + leaf := chain[0] + if h.cfg.Security != nil { + if !h.cfg.Security.Ready() { + return authenticatedClient{}, wireError(executionenv.CodeNotReady, true) + } + id, policy, err := h.cfg.Security.authorize(ctx, chain) + if err != nil { + return authenticatedClient{}, securityAuthorizationError(err) + } + return authenticatedClient{id: id, policy: policy}, nil + } + id, err := canonicalClientIdentity(leaf) + if err != nil { + return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) + } + policy, ok := h.cfg.Clients[id] + if !ok { + return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) + } + return authenticatedClient{id: id, policy: policy}, nil +} + +func canonicalClientIdentity(c *x509.Certificate) (string, error) { + if len(c.URIs) != 1 { + return "", errors.New("certificate must have exactly one URI SAN") + } + ids := make([]string, 0, 1) + for _, u := range c.URIs { + if u == nil || u.Scheme == "" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + continue + } + v := *u + v.Scheme, v.Host = strings.ToLower(v.Scheme), strings.ToLower(v.Host) + ids = append(ids, v.String()) + } + if len(ids) != 1 { + return "", errors.New("certificate must have exactly one canonical URI SAN") + } + return ids[0], nil +} + +func (h *Handler) lifecycle() (lifecycleBackend, error) { + if h.lifecycleBackend == nil { + return nil, wireError(executionenv.CodeNotReady, false) + } + return h.lifecycleBackend, nil +} + +// ValidateProfile validates one operator-defined execution profile. +func (h *Handler) ValidateProfile(ctx context.Context, q *executionv1.ValidateProfileRequest) (*executionv1.ValidateProfileResponse, error) { + if _, err := h.client(ctx); err != nil { + return nil, err + } + if q == nil || q.Profile == "" || len(q.Profile) > 63 { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + p, err := h.backend.ValidateProfile(ctx, q.Profile) + if err != nil { + return nil, backendError(err) + } + return &executionv1.ValidateProfileResponse{Profile: valid(p.Name), Digest: valid(p.Digest), Capabilities: validAll(p.Capabilities), MaxFileBytes: p.MaxFileBytes, MaxCommandBytes: p.MaxCommandBytes, MaxCommandDurationMillis: p.MaxCommandDuration.Milliseconds()}, nil +} + +// EnsureEnvironment idempotently resolves or allocates an environment. +func (h *Handler) EnsureEnvironment(ctx context.Context, q *executionv1.EnsureEnvironmentRequest) (*executionv1.EnsureEnvironmentResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + owner, ok := ownerFromProto(q.GetOwner()) + if q == nil || !c.policy.MayAttestOwner || !ok || q.BindingId == "" || len(q.BindingId) > executionenv.MaxBindingBytes || q.Profile == "" || len(q.Profile) > 63 || !validOperationID(q.OperationId) { + return nil, wireError(executionenv.CodePermissionDenied, false) + } + oh := ownerHash(owner) + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + fp := fingerprint(c.id, oh, q.BindingId, q.Profile) + var a Allocation + if owned, ok := h.backend.(ownerIntentBackend); ok { + a, err = owned.EnsurePendingOwned(ctx, c.id, oh, owner, q.BindingId, q.Profile, fp, q.OperationId) + } else { + a, err = lifecycle.EnsurePending(ctx, c.id, oh, q.BindingId, q.Profile, fp, q.OperationId) + } + if err != nil { + return nil, backendError(err) + } + return ensureResponse(a, "", time.Time{}), nil +} + +// AttachEnvironment exactly reattaches and refreshes a short-lived grant. +func (h *Handler) AttachEnvironment(ctx context.Context, q *executionv1.AttachEnvironmentRequest) (*executionv1.AttachEnvironmentResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + rc, ok := attachContext(q.GetContext()) + if q == nil || q.Purpose != executionenv.PurposeSession || !c.policy.MayAttestOwner || !ok { + return nil, wireError(executionenv.CodePermissionDenied, false) + } + oh := ownerHash(rc.Owner) + a, err := h.backend.Attach(ctx, rc.Environment, c.id, oh, rc.BindingID) + if err != nil { + return nil, backendError(err) + } + return attachResponse(a, "", time.Time{}), nil +} + +// AcquireRun takes environment-wide execution ownership and issues a run-bound grant. +func (h *Handler) AcquireRun(ctx context.Context, q *executionv1.AcquireRunRequest) (*executionv1.RunClaimResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + owner, ok := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if q == nil || !c.policy.MayAttestOwner || !ok || !validRef(ref) || !validBinding(q.BindingId) || !validIdentity(q.RunId) || !validOperationID(q.OperationId) || q.TtlMillis < executionenv.MinRunTTL.Milliseconds() || q.TtlMillis > executionenv.MaxRunTTL.Milliseconds() { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + claim, err := lifecycle.AcquireRun(ctx, ref, c.id, ownerHash(owner), q.BindingId, q.RunId, q.OperationId, time.Duration(q.TtlMillis)*time.Millisecond) + if err != nil { + return nil, backendError(err) + } + return h.runClaimResponse(ctx, claim, c.id, ownerHash(owner)) +} + +func (h *Handler) RenewRun(ctx context.Context, q *executionv1.RenewRunRequest) (*executionv1.RunClaimResponse, error) { + c, owner, req, err := h.runClaimRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetBindingId(), q.GetRunId(), q.GetClaimId(), q.GetEpoch(), q.GetGrantGeneration(), q.GetOperationId(), q.GetTtlMillis()) + if err != nil { + return nil, err + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + claim, backendErr := lifecycle.RenewRun(ctx, req.Environment, c.id, owner, req) + if backendErr != nil { + return nil, backendError(backendErr) + } + return h.runClaimResponse(ctx, claim, c.id, owner) +} + +func (h *Handler) ReleaseRun(ctx context.Context, q *executionv1.ReleaseRunRequest) (*emptypb.Empty, error) { + c, owner, req, err := h.runClaimRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetBindingId(), q.GetRunId(), q.GetClaimId(), q.GetEpoch(), q.GetGrantGeneration(), q.GetOperationId(), executionenv.DefaultRunTTL.Milliseconds()) + if err != nil { + return nil, err + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + if backendErr := lifecycle.ReleaseRun(ctx, req.Environment, c.id, owner, req); backendErr != nil { + return nil, backendError(backendErr) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) runClaimRequest(ctx context.Context, pRef *executionv1.EnvironmentRef, pOwner *executionv1.Owner, binding, runID, claimID string, epoch, generation uint64, operationID string, ttlMillis int64) (authenticatedClient, string, executionenv.RunClaimRequest, error) { + c, err := h.client(ctx) + if err != nil { + return c, "", executionenv.RunClaimRequest{}, err + } + owner, ok := ownerFromProto(pOwner) + ref := refFromProto(pRef) + if !c.policy.MayAttestOwner || !ok || !validRef(ref) || !validBinding(binding) || !validIdentity(runID) || !validIdentity(claimID) || epoch == 0 || epoch > math.MaxInt64 || generation == 0 || generation > math.MaxInt64 || !validOperationID(operationID) || ttlMillis < executionenv.MinRunTTL.Milliseconds() || ttlMillis > executionenv.MaxRunTTL.Milliseconds() { + return c, "", executionenv.RunClaimRequest{}, wireError(executionenv.CodeInvalidArgument, false) + } + return c, ownerHash(owner), executionenv.RunClaimRequest{Environment: ref, Owner: owner, BindingID: binding, RunID: runID, ClaimID: claimID, Epoch: epoch, GrantGeneration: generation, OperationID: operationID, TTL: time.Duration(ttlMillis) * time.Millisecond}, nil +} + +func (h *Handler) runClaimResponse(ctx context.Context, claim executionenv.RunClaim, client, owner string) (*executionv1.RunClaimResponse, error) { + grant, expiry, err := h.signClaim(ctx, claim, client, owner) + if err != nil { + if errors.Is(err, errAuthorityUnavailable) { + return nil, securityAuthorizationError(err) + } + return nil, wireError(executionenv.CodeInternal, false) + } + return &executionv1.RunClaimResponse{Environment: refToProto(claim.Environment), BindingId: valid(claim.BindingID), RunId: valid(claim.RunID), ClaimId: valid(claim.ClaimID), Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: valid(grant), ExpiresAt: timestamppb.New(expiry)}, nil +} + +func (h *Handler) referenceRequest(ctx context.Context, q *executionv1.ReferenceMutationRequest) (authenticatedClient, string, executionenv.EnvironmentRef, string, string, error) { + c, err := h.client(ctx) + if err != nil { + return c, "", executionenv.EnvironmentRef{}, "", "", err + } + owner, ok := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if q == nil || !c.policy.MayAttestOwner || !ok || !validRef(ref) || !validBinding(q.BindingId) || !validOperationID(q.OperationId) { + return c, "", ref, "", "", wireError(executionenv.CodeInvalidArgument, false) + } + return c, ownerHash(owner), ref, q.BindingId, q.OperationId, nil +} +func (h *Handler) referenceMutation(ctx context.Context, q *executionv1.ReferenceMutationRequest, operation executionenv.Operation) (*emptypb.Empty, error) { + c, owner, ref, binding, operationID, err := h.referenceRequest(ctx, q) + if err != nil { + return nil, err + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + switch operation { + case executionenv.OpReferenceCommit: + err = lifecycle.CommitReference(ctx, ref, c.id, owner, binding, operationID) + case executionenv.OpReferenceAbort: + err = lifecycle.AbortReference(ctx, ref, c.id, owner, binding, operationID) + case executionenv.OpReferenceDeletePrepare: + err = lifecycle.PrepareReferenceDelete(ctx, ref, c.id, owner, binding, operationID) + case executionenv.OpReferenceDeleteConfirm: + err = lifecycle.ConfirmReferenceDelete(ctx, ref, c.id, owner, binding, operationID) + case executionenv.OpReferenceDeleteCancel: + err = lifecycle.CancelReferenceDelete(ctx, ref, c.id, owner, binding, operationID) + default: + err = &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "invalid reference operation"} + } + if err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} +func (h *Handler) CommitReference(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceCommit) +} +func (h *Handler) AbortReference(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceAbort) +} +func (h *Handler) PrepareReferenceDelete(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceDeletePrepare) +} +func (h *Handler) ConfirmReferenceDelete(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceDeleteConfirm) +} +func (h *Handler) CancelReferenceDelete(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceDeleteCancel) +} + +func (h *Handler) ReserveSuccessor(ctx context.Context, q *executionv1.ReserveSuccessorRequest) (*executionv1.ReferenceReservationResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + owner, ok := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if q == nil || !c.policy.MayAttestOwner || !ok || !validRef(ref) || !validBinding(q.SourceBindingId) || !validBinding(q.DestinationBindingId) || q.SourceBindingId == q.DestinationBindingId || !validOperationID(q.OperationId) { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + if err = lifecycle.ReserveSuccessor(ctx, ref, c.id, ownerHash(owner), q.SourceBindingId, q.DestinationBindingId, q.OperationId); err != nil { + return nil, backendError(err) + } + return &executionv1.ReferenceReservationResponse{Environment: refToProto(ref)}, nil +} + +func (h *Handler) ListReferenceIntents(ctx context.Context, q *executionv1.ListReferenceIntentsRequest) (*executionv1.ListReferenceIntentsResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + if q == nil || !c.policy.MayAttestOwner || q.Limit < 0 || q.Limit > maxReferences { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + var intents []executionenv.ReferenceIntent + exactRef := refFromProto(q.GetEnvironment()) + exactLookup := q.GetEnvironment() != nil || q.GetBindingId() != "" + if exactLookup { + owner, ok := ownerFromProto(q.GetOwner()) + if !ok || !validRef(exactRef) || !validBinding(q.GetBindingId()) { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + intent, findErr := lifecycle.FindReferenceIntent(ctx, exactRef, c.id, ownerHash(owner), q.GetBindingId()) + if findErr != nil { + return nil, backendError(findErr) + } + intent.Owner = owner + intents = []executionenv.ReferenceIntent{intent} + } else if q.Owner != nil { + owner, ok := ownerFromProto(q.Owner) + if !ok { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + intents, err = lifecycle.ListReferenceIntents(ctx, c.id, ownerHash(owner), int(q.Limit)) + for i := range intents { + intents[i].Owner = owner + } + } else if owned, ok := h.backend.(ownerIntentBackend); ok { + intents, err = owned.ListReferenceIntentsForClient(ctx, c.id, int(q.Limit)) + } else { + return nil, wireError(executionenv.CodeNotReady, true) + } + if err != nil { + return nil, backendError(err) + } + out := &executionv1.ListReferenceIntentsResponse{} + for _, intent := range intents { + out.Intents = append(out.Intents, &executionv1.ReferenceIntent{Environment: refToProto(intent.Environment), BindingId: valid(intent.BindingID), State: valid(string(intent.State)), OperationId: valid(intent.OperationID), SourceBindingId: valid(intent.SourceBindingID), CreatedAt: timestamppb.New(intent.CreatedAt), Owner: &executionv1.Owner{Issuer: valid(intent.Owner.Issuer), Subject: valid(intent.Owner.Subject)}}) + } + return out, nil +} + +// ReleaseReference releases one authorized durable binding reference. +func (h *Handler) ReleaseReference(ctx context.Context, q *executionv1.ReleaseReferenceRequest) (*emptypb.Empty, error) { + c, owner, rc, err := h.authorize(ctx, q.GetContext(), executionenv.OpReferenceRelease) + if err != nil { + return nil, err + } + if err := h.backend.ReleaseReference(ctx, rc.Environment, c.id, owner, rc.BindingID); err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +// RetireEnvironment starts exact, proof-producing executor retirement while retaining the PVC. +func (h *Handler) RetireEnvironment(ctx context.Context, q *executionv1.RetireEnvironmentRequest) (*emptypb.Empty, error) { + req, err := h.adminRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetExpectedExecutionEpoch(), q.GetExpectedPodUid(), q.GetExpectedPvcUid(), q.GetOperationId()) + if err != nil { + return nil, err + } + if err := h.adminLifecycleBackend.RetireExact(ctx, req); err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) ReplaceExecutor(ctx context.Context, q *executionv1.ReplaceExecutorRequest) (*emptypb.Empty, error) { + req, err := h.adminRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetExpectedExecutionEpoch(), q.GetExpectedPodUid(), q.GetExpectedPvcUid(), q.GetOperationId()) + if err != nil { + return nil, err + } + if err := h.adminLifecycleBackend.ReplaceExecutor(ctx, req); err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) RecoverEnvironment(ctx context.Context, q *executionv1.RecoverEnvironmentRequest) (*emptypb.Empty, error) { + req, err := h.adminRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetExpectedExecutionEpoch(), q.GetExpectedPodUid(), q.GetExpectedPvcUid(), q.GetOperationId()) + if err != nil { + return nil, err + } + if err := h.adminLifecycleBackend.RecoverEnvironment(ctx, req); err != nil { + var controlled *executionenv.Error + if errors.As(err, &controlled) && controlled.Code == executionenv.CodeFenceUnknown { + return nil, wireError(executionenv.CodeFenceUnknown, false) + } + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) DeleteRetiredEnvironment(ctx context.Context, q *executionv1.DeleteRetiredEnvironmentRequest) (*emptypb.Empty, error) { + c, err := h.client(ctx) + owner, ownerOK := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if err != nil { + return nil, err + } + if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) || !validIdentity(q.GetExpectedPvcUid()) || !validOperationID(q.GetOperationId()) { + return nil, wireError(executionenv.CodePermissionDenied, false) + } + req := adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, AdministratorFor: c.policy.AdministratorFor, ExpectedPVCUID: q.GetExpectedPvcUid(), OperationID: q.GetOperationId()} + if err := h.adminLifecycleBackend.DeleteRetiredEnvironment(ctx, req); err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) MigrateEnvironment(ctx context.Context, q *executionv1.MigrateEnvironmentRequest) (*emptypb.Empty, error) { + req, err := h.adminRequest(ctx, q.GetEnvironment(), q.GetOwner(), 1, q.GetExpectedPodUid(), q.GetExpectedPvcUid(), q.GetOperationId()) + if err != nil { + return nil, err + } + if q == nil || q.ExpectedSchemaVersion == nil || q.GetExpectedSchemaVersion() > 1 { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + req.ExpectedEpoch = 0 + req.ExpectedSchema = int64(q.GetExpectedSchemaVersion()) + if err := h.adminLifecycleBackend.MigrateEnvironment(ctx, req); err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) RevokeEnvironment(ctx context.Context, q *executionv1.RevokeEnvironmentRequest) (*executionv1.RevokeEnvironmentResponse, error) { + c, err := h.client(ctx) + owner, ownerOK := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if err != nil { + return nil, err + } + if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) { + return nil, wireError(executionenv.CodePermissionDenied, false) + } + if q.GetExpectedGrantGeneration() == 0 || q.GetExpectedGrantGeneration() > math.MaxInt64 || !validOperationID(q.GetOperationId()) { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + req := adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, AdministratorFor: c.policy.AdministratorFor, OperationID: q.GetOperationId()} + generation, err := h.adminLifecycleBackend.RevokeEnvironment(ctx, req, q.GetExpectedGrantGeneration()) + if err != nil { + return nil, backendError(err) + } + return &executionv1.RevokeEnvironmentResponse{GrantGeneration: generation}, nil +} + +func (h *Handler) adminRequest(ctx context.Context, pRef *executionv1.EnvironmentRef, pOwner *executionv1.Owner, epoch uint64, podUID, pvcUID, operationID string) (adminLifecycleRequest, error) { + c, err := h.client(ctx) + if err != nil { + return adminLifecycleRequest{}, err + } + owner, ownerOK := ownerFromProto(pOwner) + ref := refFromProto(pRef) + if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) || epoch == 0 || epoch > math.MaxInt64 || !validIdentity(podUID) || !validIdentity(pvcUID) || !validOperationID(operationID) { + return adminLifecycleRequest{}, wireError(executionenv.CodePermissionDenied, false) + } + return adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, AdministratorFor: c.policy.AdministratorFor, ExpectedEpoch: epoch, ExpectedPodUID: podUID, ExpectedPVCUID: pvcUID, OperationID: operationID}, nil +} + +// Files executes one bounded and authorized filesystem operation. +func (h *Handler) Files(ctx context.Context, q *executionv1.FileRequest) (*executionv1.FileResponse, error) { + if _, err := h.client(ctx); err != nil { + return nil, err + } + op, ok := operationFromProto(q.GetOperation()) + if q == nil || !ok || !validFileRequest(q) { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + c, owner, rc, err := h.authorize(ctx, q.Context, op) + if err != nil { + return nil, err + } + req := executionenv.FileRequest{Context: rc, Operation: op, Path: q.Path, Destination: q.Destination, Pattern: q.Pattern, Data: q.Data, Version: string(q.Version), Limit: int(q.Limit)} + out, err := h.backend.File(ctx, c.id, owner, req) + if err != nil { + return nil, backendError(err) + } + return fileResponse(out), nil +} + +// StartCommand executes one authorized foreground command. +func (h *Handler) StartCommand(ctx context.Context, q *executionv1.CommandStartRequest) (*executionv1.CommandStartResponse, error) { + if _, err := h.client(ctx); err != nil { + return nil, err + } + if q == nil || len(q.Command) == 0 || len(q.Command) > executionenv.MaxCommandBytes || q.TimeoutMillis < 0 { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + c, owner, rc, err := h.authorize(ctx, q.Context, executionenv.OpCommandStart) + if err != nil { + return nil, err + } + out, err := h.backend.StartCommand(ctx, c.id, owner, executionenv.CommandStartRequest{Context: rc, Command: q.Command, TimeoutMillis: q.TimeoutMillis}) + if err != nil { + return nil, backendError(err) + } + response, ok := commandStartResponse(out) + if !ok { + return nil, wireError(executionenv.CodeInternal, false) + } + return response, nil +} + +// CommandStatus authorizes then reports the detached API as unsupported. +func (h *Handler) CommandStatus(ctx context.Context, q *executionv1.CommandQueryRequest) (*executionv1.CommandStatusResponse, error) { + return h.commandQuery(ctx, q, executionenv.OpCommandStatus) +} + +// CancelCommand authorizes then reports the detached API as unsupported. +func (h *Handler) CancelCommand(ctx context.Context, q *executionv1.CommandQueryRequest) (*executionv1.CommandStatusResponse, error) { + return h.commandQuery(ctx, q, executionenv.OpCommandCancel) +} +func (h *Handler) commandQuery(ctx context.Context, q *executionv1.CommandQueryRequest, op executionenv.Operation) (*executionv1.CommandStatusResponse, error) { + _, _, _, err := h.authorize(ctx, q.GetContext(), op) + if err != nil { + return nil, err + } + if q.CommandId == "" || q.Offset < 0 { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + return nil, status.Error(codes.Unimplemented, "foreground commands have no detached control API") +} + +func (h *Handler) authorize(ctx context.Context, p *executionv1.RequestContext, op executionenv.Operation) (authenticatedClient, string, executionenv.RequestContext, error) { + c, err := h.client(ctx) + if err != nil { + return c, "", executionenv.RequestContext{}, err + } + rc, ok := requestContextFromProto(p) + if !ok { + return c, "", rc, wireError(executionenv.CodePermissionDenied, false) + } + oh := ownerHash(rc.Owner) + verifier := h.cfg.Verifier + if h.cfg.Security != nil { + now := h.cfg.Security.now() + material, materialErr := h.cfg.Security.authoritativeAt(ctx, now) + if materialErr != nil { + return c, "", rc, wireError(executionenv.CodeNotReady, true) + } + verifier = material.verifierAt(now) + } + if _, err := verifier.Verify(rc.Grant, executionenv.GrantExpectation{Client: c.id, OwnerHash: oh, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: rc.GrantGeneration, Operation: op}); err != nil { + if errors.Is(err, executionenv.ErrGrantExpired) { + return c, "", rc, wireError(executionenv.CodeUnauthenticated, true) + } + return c, "", rc, wireError(executionenv.CodePermissionDenied, false) + } + if validator, ok := h.backend.(runClaimValidator); ok { + if err := validator.ValidateRunClaim(ctx, c.id, oh, rc); err != nil { + return c, "", rc, backendError(err) + } + } + return c, oh, rc, nil +} + +func requestContextFromProto(p *executionv1.RequestContext) (executionenv.RequestContext, bool) { + owner, ownerOK := ownerFromProto(p.GetOwner()) + ref := refFromProto(p.GetEnvironment()) + rc := executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: p.GetBindingId(), RunID: p.GetRunId(), ClaimID: p.GetClaimId(), Epoch: p.GetEpoch(), GrantGeneration: p.GetGrantGeneration(), Grant: p.GetGrant()} + ok := p != nil && ownerOK && ref.ID != "" && ref.Revision != "" && len(ref.ID) <= executionenv.MaxIdentityBytes && len(ref.Revision) <= executionenv.MaxIdentityBytes && rc.BindingID != "" && len(rc.BindingID) <= executionenv.MaxBindingBytes && validIdentity(rc.RunID) && validIdentity(rc.ClaimID) && rc.Epoch != 0 && rc.GrantGeneration != 0 && rc.Grant != "" && len(rc.Grant) <= executionenv.MaxGrantBytes + return rc, ok +} +func attachContext(p *executionv1.RequestContext) (executionenv.RequestContext, bool) { + owner, ownerOK := ownerFromProto(p.GetOwner()) + ref := refFromProto(p.GetEnvironment()) + rc := executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: p.GetBindingId()} + return rc, p != nil && ownerOK && ref.ID != "" && ref.Revision != "" && len(ref.ID) <= executionenv.MaxIdentityBytes && len(ref.Revision) <= executionenv.MaxIdentityBytes && rc.BindingID != "" && len(rc.BindingID) <= executionenv.MaxBindingBytes +} +func ownerFromProto(p *executionv1.Owner) (executionenv.Owner, bool) { + o := executionenv.Owner{Issuer: p.GetIssuer(), Subject: p.GetSubject()} + return o, p != nil && o.Issuer != "" && o.Subject != "" && len(o.Issuer) <= executionenv.MaxIdentityBytes && len(o.Subject) <= executionenv.MaxIdentityBytes +} +func refFromProto(p *executionv1.EnvironmentRef) executionenv.EnvironmentRef { + return executionenv.EnvironmentRef{ID: p.GetId(), Revision: p.GetRevision()} +} +func refToProto(r executionenv.EnvironmentRef) *executionv1.EnvironmentRef { + return &executionv1.EnvironmentRef{Id: valid(r.ID), Revision: valid(r.Revision)} +} + +func validFileRequest(q *executionv1.FileRequest) bool { //nolint:gocyclo // Closed operation/field matrix is intentionally explicit. + if len(q.Data) > executionenv.MaxFileBytes || len(q.Path) > executionenv.MaxPathBytes || len(q.Destination) > executionenv.MaxPathBytes || len(q.Pattern) > executionenv.MaxPathBytes || q.Limit < 0 || q.Limit > executionenv.MaxListEntries { + return false + } + path, dest, pattern, data, version, limit := q.Path != "", q.Destination != "", q.Pattern != "", len(q.Data) != 0, len(q.Version) != 0, q.Limit != 0 + switch q.Operation { + case executionv1.FileOperation_FILE_OPERATION_READ, executionv1.FileOperation_FILE_OPERATION_RESOLVE_AUTHORITY, executionv1.FileOperation_FILE_OPERATION_STAT, executionv1.FileOperation_FILE_OPERATION_REMOVE: + return path && !dest && !pattern && !data && !version && !limit + case executionv1.FileOperation_FILE_OPERATION_CREATE: + return path && !dest && !pattern && !version && !limit + case executionv1.FileOperation_FILE_OPERATION_REPLACE: + return path && !dest && !pattern && version && !limit + case executionv1.FileOperation_FILE_OPERATION_LIST: + return path && !dest && !pattern && !data && !version + case executionv1.FileOperation_FILE_OPERATION_RENAME, executionv1.FileOperation_FILE_OPERATION_COPY: + return path && dest && !pattern && !data && !version && !limit + case executionv1.FileOperation_FILE_OPERATION_GLOB: + return !path && !dest && pattern && !data && !version + case executionv1.FileOperation_FILE_OPERATION_GREP: + return path && !dest && pattern && !data && !version + default: + return false + } +} + +func operationFromProto(op executionv1.FileOperation) (executionenv.Operation, bool) { + m := map[executionv1.FileOperation]executionenv.Operation{executionv1.FileOperation_FILE_OPERATION_READ: executionenv.OpFileRead, executionv1.FileOperation_FILE_OPERATION_RESOLVE_AUTHORITY: executionenv.OpFileResolveAuthority, executionv1.FileOperation_FILE_OPERATION_STAT: executionenv.OpFileStat, executionv1.FileOperation_FILE_OPERATION_CREATE: executionenv.OpFileCreate, executionv1.FileOperation_FILE_OPERATION_REPLACE: executionenv.OpFileReplace, executionv1.FileOperation_FILE_OPERATION_LIST: executionenv.OpFileList, executionv1.FileOperation_FILE_OPERATION_REMOVE: executionenv.OpFileRemove, executionv1.FileOperation_FILE_OPERATION_RENAME: executionenv.OpFileRename, executionv1.FileOperation_FILE_OPERATION_COPY: executionenv.OpFileCopy, executionv1.FileOperation_FILE_OPERATION_GLOB: executionenv.OpFileGlob, executionv1.FileOperation_FILE_OPERATION_GREP: executionenv.OpFileGrep} + v, ok := m[op] + return v, ok +} + +func ensureResponse(a Allocation, grant string, expiry time.Time) *executionv1.EnsureEnvironmentResponse { + return &executionv1.EnsureEnvironmentResponse{Environment: refToProto(a.Environment), Epoch: a.Epoch, Ready: a.Ready, GrantGeneration: a.GrantGeneration, Grant: valid(grant), GrantExpiresAt: timestamppb.New(expiry)} +} +func attachResponse(a Allocation, grant string, expiry time.Time) *executionv1.AttachEnvironmentResponse { + return &executionv1.AttachEnvironmentResponse{Environment: refToProto(a.Environment), Epoch: a.Epoch, Ready: a.Ready, GrantGeneration: a.GrantGeneration, Grant: valid(grant), GrantExpiresAt: timestamppb.New(expiry)} +} +func fileResponse(v executionenv.FileResponse) *executionv1.FileResponse { + r := &executionv1.FileResponse{Data: v.Data, Version: []byte(v.Version), Paths: validAll(v.Paths), AuthorityTarget: valid(v.AuthorityTarget), AuthorityWorkspace: valid(v.AuthorityWorkspace)} + if v.Info != nil { + r.Info = fileInfoToProto(*v.Info) + } + for _, x := range v.Entries { + r.Entries = append(r.Entries, fileInfoToProto(x)) + } + for _, x := range v.Matches { + r.Matches = append(r.Matches, &executionv1.GrepMatch{Path: valid(x.Path), Line: boundedInt32(x.Line), Text: valid(x.Text)}) + } + return r +} +func fileInfoToProto(v executionenv.FileInfo) *executionv1.FileInfo { + return &executionv1.FileInfo{Name: valid(v.Name), Size: v.Size, Mode: v.Mode, ModTime: timestamppb.New(v.ModTime), IsDir: v.IsDir} +} +func commandStartResponse(v executionenv.CommandStartResponse) (*executionv1.CommandStartResponse, bool) { + state, ok := commandStateToProto(v.State) + if !ok { + return nil, false + } + result, ok := commandStatusResponse(v.Result) + if !ok { + return nil, false + } + return &executionv1.CommandStartResponse{CommandId: valid(v.CommandID), State: state, Result: result}, true +} +func commandStatusResponse(v executionenv.CommandStatusResponse) (*executionv1.CommandStatusResponse, bool) { + state, ok := commandStateToProto(v.State) + if !ok { + return nil, false + } + return &executionv1.CommandStatusResponse{CommandId: valid(v.CommandID), State: state, ExitCode: boundedInt32(v.ExitCode), Stdout: v.Stdout, Stderr: v.Stderr, NextOffset: v.NextOffset, Truncated: v.Truncated, TerminalReceipt: valid(v.TerminalReceipt)}, true +} +func boundedInt32(v int) int32 { + if v > math.MaxInt32 { + return math.MaxInt32 + } + if v < math.MinInt32 { + return math.MinInt32 + } + return int32(v) //nolint:gosec // bounds checked above +} + +func commandStateToProto(v executionenv.CommandState) (executionv1.CommandState, bool) { + switch v { + case executionenv.CommandRunning: + return executionv1.CommandState_COMMAND_STATE_RUNNING, true + case executionenv.CommandSucceeded: + return executionv1.CommandState_COMMAND_STATE_SUCCEEDED, true + case executionenv.CommandFailed: + return executionv1.CommandState_COMMAND_STATE_FAILED, true + case executionenv.CommandCancelled: + return executionv1.CommandState_COMMAND_STATE_CANCELLED, true + case executionenv.CommandFenceUnknown: + return executionv1.CommandState_COMMAND_STATE_FENCE_UNKNOWN, true + default: + return executionv1.CommandState_COMMAND_STATE_UNSPECIFIED, false + } +} + +func validIdentity(v string) bool { return v != "" && len(v) <= executionenv.MaxIdentityBytes } +func validBinding(v string) bool { return v != "" && len(v) <= executionenv.MaxBindingBytes } +func validOperationID(v string) bool { return validIdentity(v) } +func validRef(v executionenv.EnvironmentRef) bool { + return validIdentity(v.ID) && validIdentity(v.Revision) +} + +func (h *Handler) signClaim(ctx context.Context, claim executionenv.RunClaim, client, owner string) (string, time.Time, error) { + signer := h.cfg.Signer + now := time.Now().UTC() + var signingDeadline time.Time + if h.cfg.Security != nil { + now = h.cfg.Security.now() + material, err := h.cfg.Security.authoritativeAt(ctx, now) + if err != nil { + return "", time.Time{}, err + } + signer = material.signer + signingDeadline = material.activeWindow.verifyUntil + } + life := signer.Lifetime + if life <= 0 { + life = time.Minute + } + expiry := now.Add(life) + if !signingDeadline.IsZero() && signingDeadline.Before(expiry) { + expiry = signingDeadline + } + if !now.Before(expiry) { + return "", time.Time{}, errors.New("active signing key is expired") + } + nonce := make([]byte, 16) + if _, err := rand.Read(nonce); err != nil { + return "", time.Time{}, err + } + ops := []executionenv.Operation{executionenv.OpFileRead, executionenv.OpFileResolveAuthority, executionenv.OpFileStat, executionenv.OpFileCreate, executionenv.OpFileReplace, executionenv.OpFileList, executionenv.OpFileRemove, executionenv.OpFileRename, executionenv.OpFileCopy, executionenv.OpFileGlob, executionenv.OpFileGrep, executionenv.OpCommandStart, executionenv.OpCommandStatus, executionenv.OpCommandCancel} + grant, err := executionenv.SignGrant(signer.PrivateKey, executionenv.GrantClaims{KeyID: signer.KeyID, Issuer: signer.Issuer, Audience: signer.Audience, Client: client, OwnerHash: owner, BindingID: claim.BindingID, RunID: claim.RunID, ClaimID: claim.ClaimID, Environment: claim.Environment, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Operations: ops, NotBefore: now, ExpiresAt: expiry, Nonce: hex.EncodeToString(nonce)}) + return grant, expiry, err +} +func ownerHash(o executionenv.Owner) string { + s := sha256.Sum256([]byte(o.Issuer + "\x00" + o.Subject)) + return hex.EncodeToString(s[:]) +} +func fingerprint(fields ...string) string { + h := sha256.New() + for _, f := range fields { + _, _ = h.Write([]byte{0}) + _, _ = io.WriteString(h, f) + } + return hex.EncodeToString(h.Sum(nil)) +} + +func securityAuthorizationError(err error) error { + if errors.Is(err, errAuthorityUnavailable) { + return wireError(executionenv.CodeNotReady, true) + } + return wireError(executionenv.CodeUnauthenticated, false) +} + +func backendError(err error) error { + var e *executionenv.Error + if errors.As(err, &e) && e.Code.Valid() { + return wireError(e.Code, e.Retryable) + } + return wireError(executionenv.CodeInternal, false) +} +func wireError(code executionenv.ErrorCode, retry bool) error { + if !code.Valid() || (retry && code != executionenv.CodeNotReady && code != executionenv.CodeUnauthenticated) { + code, retry = executionenv.CodeInternal, false + } + grpcCode := codes.Internal + switch code { + case executionenv.CodeInvalidArgument: + grpcCode = codes.InvalidArgument + case executionenv.CodeUnauthenticated: + grpcCode = codes.Unauthenticated + case executionenv.CodePermissionDenied: + grpcCode = codes.PermissionDenied + case executionenv.CodeNotFound: + grpcCode = codes.NotFound + case executionenv.CodeAlreadyExists: + grpcCode = codes.AlreadyExists + case executionenv.CodeConflict, executionenv.CodeVersionMismatch, executionenv.CodeDirectoryNotEmpty: + grpcCode = codes.Aborted + case executionenv.CodeNotReady: + grpcCode = codes.Unavailable + case executionenv.CodeFenceUnknown: + grpcCode = codes.FailedPrecondition + case executionenv.CodeResourceExhausted: + grpcCode = codes.ResourceExhausted + } + message := "execution provider request failed" + if code == executionenv.CodeFenceUnknown { + message = "execution provider request failed; verify termination independently or use external fencing" + } + st := status.New(grpcCode, message) + with, err := st.WithDetails(&executionv1.ErrorDetail{Code: string(code), Retryable: retry}) + if err == nil { + st = with + } + return st.Err() +} +func valid(s string) string { + if utf8.ValidString(s) { + return s + } + return strings.ToValidUTF8(s, "�") +} +func validAll(in []string) []string { + out := make([]string, len(in)) + for i := range in { + out[i] = valid(in[i]) + } + return out +} + +// TLSConfig returns the provider's TLS 1.3 mutual-authentication policy. +func TLSConfig(server tls.Certificate, clientCAs *x509.CertPool) *tls.Config { + return &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{server}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: clientCAs} +} diff --git a/internal/adapter/executioncontroller/handler_test.go b/internal/adapter/executioncontroller/handler_test.go new file mode 100644 index 0000000000..314292c856 --- /dev/null +++ b/internal/adapter/executioncontroller/handler_test.go @@ -0,0 +1,370 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "net/url" + "strings" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type fakeBackend struct { + ensureCalls, created int + allocations map[string]Allocation + commandState executionenv.CommandState + intents []executionenv.ReferenceIntent +} + +func newFakeBackend() *fakeBackend { return &fakeBackend{allocations: map[string]Allocation{}} } +func (*fakeBackend) ValidateProfile(context.Context, string) (Profile, error) { + return Profile{Name: "go", Digest: "sha256:test", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute}, nil +} +func (f *fakeBackend) Ensure(_ context.Context, client, owner, binding, _, fp string) (Allocation, error) { + f.ensureCalls++ + if a, ok := f.allocations[fp]; ok { + return a, nil + } + f.created++ + a := Allocation{Environment: executionenv.EnvironmentRef{ID: "env-1", Revision: "rev-1"}, Epoch: 1, OwnerHash: owner, BindingID: binding, Client: client, Ready: true} + f.allocations[fp] = a + return a, nil +} +func (*fakeBackend) Attach(context.Context, executionenv.EnvironmentRef, string, string, string) (Allocation, error) { + panic("unused") +} +func (*fakeBackend) ReleaseReference(context.Context, executionenv.EnvironmentRef, string, string, string) error { + return nil +} +func (*fakeBackend) Retire(context.Context, executionenv.EnvironmentRef, string) error { return nil } +func (*fakeBackend) File(context.Context, string, string, executionenv.FileRequest) (executionenv.FileResponse, error) { + return executionenv.FileResponse{}, nil +} +func (f *fakeBackend) StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + state := f.commandState + if state == "" { + state = executionenv.CommandSucceeded + } + return executionenv.CommandStartResponse{CommandID: "command", State: state, Result: executionenv.CommandStatusResponse{CommandID: "command", State: state}}, nil +} +func (*fakeBackend) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, nil +} +func (*fakeBackend) CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, nil +} + +func (f *fakeBackend) EnsurePending(ctx context.Context, client, owner, binding, profile, fp, _ string) (Allocation, error) { + return f.Ensure(ctx, client, owner, binding, profile, fp) +} +func (*fakeBackend) AcquireRun(_ context.Context, ref executionenv.EnvironmentRef, _ string, _ string, binding, run, _ string, ttl time.Duration) (executionenv.RunClaim, error) { + return executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: run, ClaimID: "claim", Epoch: 2, GrantGeneration: 1, ExpiresAt: time.Now().Add(ttl)}, nil +} +func (*fakeBackend) RenewRun(_ context.Context, _ executionenv.EnvironmentRef, _ string, _ string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + return executionenv.RunClaim{Environment: req.Environment, BindingID: req.BindingID, RunID: req.RunID, ClaimID: req.ClaimID, Epoch: req.Epoch, GrantGeneration: 1, ExpiresAt: time.Now().Add(req.TTL)}, nil +} +func (*fakeBackend) ReleaseRun(context.Context, executionenv.EnvironmentRef, string, string, executionenv.RunClaimRequest) error { + return nil +} +func (*fakeBackend) CommitReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) AbortReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) ReserveSuccessor(context.Context, executionenv.EnvironmentRef, string, string, string, string, string) error { + return nil +} +func (*fakeBackend) PrepareReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) ConfirmReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) CancelReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (f *fakeBackend) ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) { + return f.intents, nil +} +func (*fakeBackend) FindReferenceIntent(context.Context, executionenv.EnvironmentRef, string, string, string) (executionenv.ReferenceIntent, error) { + return executionenv.ReferenceIntent{}, nil +} + +type adminFakeBackend struct { + *fakeBackend + migratedSchema int64 +} + +func (*adminFakeBackend) ReplaceExecutor(context.Context, adminLifecycleRequest) error { return nil } +func (*adminFakeBackend) RetireExact(context.Context, adminLifecycleRequest) error { return nil } +func (*adminFakeBackend) RecoverEnvironment(context.Context, adminLifecycleRequest) error { + return nil +} +func (*adminFakeBackend) DeleteRetiredEnvironment(context.Context, adminLifecycleRequest) error { + return nil +} +func (b *adminFakeBackend) MigrateEnvironment(_ context.Context, req adminLifecycleRequest) error { + b.migratedSchema = req.ExpectedSchema + return nil +} +func (*adminFakeBackend) RevokeEnvironment(context.Context, adminLifecycleRequest, uint64) (uint64, error) { + return 2, nil +} + +func authenticatedContext(id string) context.Context { + u, _ := url.Parse(id) + cert := &x509.Certificate{URIs: []*url.URL{u}} + return peer.NewContext(context.Background(), &peer.Peer{AuthInfo: credentials.TLSInfo{State: structTLSState(cert)}}) +} +func structTLSState(cert *x509.Certificate) (s tls.ConnectionState) { + s.PeerCertificates = []*x509.Certificate{cert} + s.VerifiedChains = [][]*x509.Certificate{{cert}} + return s +} + +func TestListReferenceIntentsProjectsAttestedOwner(t *testing.T) { + backend := newFakeBackend() + backend.intents = []executionenv.ReferenceIntent{{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", State: executionenv.ReferencePendingDelete, OperationID: "delete", CreatedAt: time.Now().UTC()}} + id := "spiffe://cluster/ns/mecak8s" + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{id: {MayAttestOwner: true}}}, backend) + owner := &executionv1.Owner{Issuer: "issuer", Subject: "alice"} + out, err := h.ListReferenceIntents(authenticatedContext(id), &executionv1.ListReferenceIntentsRequest{Owner: owner, Limit: 64}) + if err != nil { + t.Fatal(err) + } + if len(out.GetIntents()) != 1 || out.GetIntents()[0].GetOwner().GetIssuer() != owner.GetIssuer() || out.GetIntents()[0].GetOwner().GetSubject() != owner.GetSubject() { + t.Fatal("owner-scoped reference intent omitted its attested owner") + } +} + +func TestMigrateEnvironmentRequiresExpectedSchemaPresence(t *testing.T) { + backend := &adminFakeBackend{fakeBackend: newFakeBackend()} + id := "spiffe://cluster/ns/admin" + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{id: {Administrator: true}}}, backend) + ctx := authenticatedContext(id) + base := &executionv1.MigrateEnvironmentRequest{Environment: &executionv1.EnvironmentRef{Id: "env", Revision: "rev"}, Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, ExpectedPodUid: "pod", ExpectedPvcUid: "pvc", OperationId: "migrate"} + if _, err := h.MigrateEnvironment(ctx, base); status.Code(err) != codes.InvalidArgument { + t.Fatalf("omitted schema code=%v", status.Code(err)) + } + zero := uint32(0) + base.ExpectedSchemaVersion = &zero + if _, err := h.MigrateEnvironment(ctx, base); err != nil || backend.migratedSchema != 0 { + t.Fatalf("explicit zero rejected: schema=%d err=%v", backend.migratedSchema, err) + } + unknown := uint32(2) + base.ExpectedSchemaVersion = &unknown + if _, err := h.MigrateEnvironment(ctx, base); status.Code(err) != codes.InvalidArgument { + t.Fatalf("unknown schema code=%v", status.Code(err)) + } +} + +func TestHandlerBlocksAllRequestsUntilStartupReady(t *testing.T) { + backend := newFakeBackend() + h := NewHandler(HandlerConfig{Ready: func() bool { return false }}, backend) + _, err := h.EnsureEnvironment(authenticatedContext("spiffe://cluster/ns/mecak8s"), &executionv1.EnsureEnvironmentRequest{}) + if status.Code(err) != codes.Unavailable { + t.Fatalf("code=%v", status.Code(err)) + } + if backend.ensureCalls != 0 { + t.Fatalf("backend calls before readiness=%d", backend.ensureCalls) + } +} + +func TestHandlerRequiresAllowlistedCanonicalURISAN(t *testing.T) { + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {}}}, newFakeBackend()) + _, err := h.ValidateProfile(authenticatedContext("spiffe://cluster/ns/other"), &executionv1.ValidateProfileRequest{Profile: "go"}) + if status.Code(err) != codes.Unauthenticated { + t.Fatalf("code=%v", status.Code(err)) + } +} + +func TestHandlerValidateIsReadOnlyAndEnsureIdempotent(t *testing.T) { + pub, priv, _ := ed25519.GenerateKey(rand.Reader) + backend := newFakeBackend() + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {MayAttestOwner: true}}, Signer: GrantSigner{KeyID: "k1", PrivateKey: priv, Issuer: "provider", Audience: "execution", Lifetime: time.Minute}, Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: "provider", Audience: "execution"}}, backend) + ctx := authenticatedContext("spiffe://cluster/ns/mecak8s") + if _, err := h.ValidateProfile(ctx, &executionv1.ValidateProfileRequest{Profile: "go"}); err != nil { + t.Fatal(err) + } + if backend.ensureCalls != 0 { + t.Fatal("validation allocated") + } + q := &executionv1.EnsureEnvironmentRequest{BindingId: "session-1", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, OperationId: "create"} + first, err := h.EnsureEnvironment(ctx, q) + if err != nil { + t.Fatal(err) + } + second, err := h.EnsureEnvironment(ctx, q) + if err != nil { + t.Fatal(err) + } + if backend.created != 1 || first.Environment.Id != second.Environment.Id || first.Epoch != second.Epoch { + t.Fatalf("created=%d", backend.created) + } +} + +func TestHandlerWithNilBackendRejectsEveryRPCWithoutPanicking(t *testing.T) { + h := NewHandler(HandlerConfig{}, nil) + ctx := context.Background() + calls := []func(bool) error{ + func(valid bool) error { + var q *executionv1.ValidateProfileRequest + if valid { + q = &executionv1.ValidateProfileRequest{Profile: "go"} + } + _, err := h.ValidateProfile(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.EnsureEnvironmentRequest + if valid { + q = &executionv1.EnsureEnvironmentRequest{BindingId: "binding", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, OperationId: "create"} + } + _, err := h.EnsureEnvironment(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.AttachEnvironmentRequest + if valid { + q = &executionv1.AttachEnvironmentRequest{Purpose: executionenv.PurposeSession} + } + _, err := h.AttachEnvironment(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.ReleaseReferenceRequest + if valid { + q = &executionv1.ReleaseReferenceRequest{Context: &executionv1.RequestContext{}} + } + _, err := h.ReleaseReference(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.RetireEnvironmentRequest + if valid { + q = &executionv1.RetireEnvironmentRequest{Environment: &executionv1.EnvironmentRef{}, Owner: &executionv1.Owner{}} + } + _, err := h.RetireEnvironment(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.FileRequest + if valid { + q = &executionv1.FileRequest{Operation: executionv1.FileOperation_FILE_OPERATION_READ, Path: "x"} + } + _, err := h.Files(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.CommandStartRequest + if valid { + q = &executionv1.CommandStartRequest{Command: "true"} + } + _, err := h.StartCommand(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.CommandQueryRequest + if valid { + q = &executionv1.CommandQueryRequest{CommandId: "command"} + } + _, err := h.CommandStatus(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.CommandQueryRequest + if valid { + q = &executionv1.CommandQueryRequest{CommandId: "command"} + } + _, err := h.CancelCommand(ctx, q) + return err + }, + } + for i, call := range calls { + for _, valid := range []bool{false, true} { + if err := call(valid); status.Code(err) != codes.Unavailable { + t.Fatalf("RPC %d valid=%t code=%v", i, valid, status.Code(err)) + } + } + } +} + +func TestHandlerRejectsUnknownBackendCommandState(t *testing.T) { + pub, priv, _ := ed25519.GenerateKey(rand.Reader) + backend := newFakeBackend() + backend.commandState = executionenv.CommandState("future") + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {MayAttestOwner: true}}, Signer: GrantSigner{KeyID: "k1", PrivateKey: priv, Issuer: "provider", Audience: "execution", Lifetime: time.Minute}, Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: "provider", Audience: "execution"}}, backend) + ctx := authenticatedContext("spiffe://cluster/ns/mecak8s") + ensured, err := h.EnsureEnvironment(ctx, &executionv1.EnsureEnvironmentRequest{BindingId: "session-1", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, OperationId: "create"}) + if err != nil { + t.Fatal(err) + } + claim, err := h.AcquireRun(ctx, &executionv1.AcquireRunRequest{Environment: ensured.Environment, Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, BindingId: "session-1", RunId: "run", OperationId: "acquire", TtlMillis: time.Minute.Milliseconds()}) + if err != nil { + t.Fatal(err) + } + q := &executionv1.CommandStartRequest{Context: &executionv1.RequestContext{Environment: claim.Environment, Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, BindingId: "session-1", RunId: claim.RunId, ClaimId: claim.ClaimId, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant}, Command: "true"} + if _, err := h.StartCommand(ctx, q); status.Code(err) != codes.Internal || status.Convert(err).Message() != "execution provider request failed" { + t.Fatalf("unexpected invalid-state result: code=%v message=%q", status.Code(err), status.Convert(err).Message()) + } +} + +func TestCanonicalClientIdentityRejectsCNAndAmbiguousURIs(t *testing.T) { + if _, err := canonicalClientIdentity(&x509.Certificate{Subject: pkix.Name{CommonName: "trusted"}}); err == nil { + t.Fatal("CN accepted") + } + u1, _ := url.Parse("spiffe://b/x") + u2, _ := url.Parse("spiffe://a/x") + if _, err := canonicalClientIdentity(&x509.Certificate{URIs: []*url.URL{u1, u2}}); err == nil { + t.Fatal("ambiguous URI SANs accepted") + } +} + +func TestRunClaimResponseKeepsNonAuthoritySigningFailureInternal(t *testing.T) { + h := NewHandler(HandlerConfig{Signer: GrantSigner{KeyID: "k1", Issuer: "issuer", Audience: "audience", Lifetime: time.Minute}}, newFakeBackend()) + claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1} + _, err := h.runClaimResponse(t.Context(), claim, "spiffe://example/client", "owner") + st := status.Convert(err) + if st.Code() != codes.Internal { + t.Fatalf("code=%v", st.Code()) + } + details := st.Details() + if len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeInternal) || details[0].(*executionv1.ErrorDetail).Retryable { + t.Fatalf("details=%v", details) + } +} + +func TestWireErrorDoesNotExposeBackendMessage(t *testing.T) { + err := backendError(&executionenv.Error{Code: executionenv.CodeInternal, Message: "kubectl exec --token=secret"}) + if got := status.Convert(err).Message(); got != "execution provider request failed" { + t.Fatalf("message=%q", got) + } + unknown := backendError(&executionenv.Error{Code: executionenv.ErrorCode("future"), Message: "sensitive path"}) + st := status.Convert(unknown) + if st.Code() != codes.Internal || st.Message() != "execution provider request failed" { + t.Fatalf("unknown error escaped: code=%v message=%q", st.Code(), st.Message()) + } + if details := st.Details(); len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeInternal) { + t.Fatalf("unknown error detail=%v", details) + } + fenced := status.Convert(backendError(&executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "/var/run/provider/private-key"})) + if fenced.Code() != codes.FailedPrecondition || strings.Contains(fenced.Message(), "/var/") { + t.Fatalf("fence error leaked or mapped incorrectly: code=%v message=%q", fenced.Code(), fenced.Message()) + } + if details := fenced.Details(); len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeFenceUnknown) { + t.Fatalf("fence error detail=%v", details) + } +} diff --git a/internal/adapter/executioncontroller/legacy_fixture_kind.go b/internal/adapter/executioncontroller/legacy_fixture_kind.go new file mode 100644 index 0000000000..5266a19c2c --- /dev/null +++ b/internal/adapter/executioncontroller/legacy_fixture_kind.go @@ -0,0 +1,202 @@ +//go:build kind_execution_e2e + +package executioncontroller + +import ( + "context" + "errors" + "fmt" + "slices" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// LegacyMigrationSeed identifies one pre-upgrade API-server fixture. +type LegacyMigrationSeed struct { + Environment executionenv.EnvironmentRef `json:"environment"` + Owner executionenv.Owner `json:"owner"` + Binding string `json:"binding"` + PodUID string `json:"pod_uid"` + PVCUID string `json:"pvc_uid"` + Malformed executionenv.EnvironmentRef `json:"malformed_environment"` + MalformedPodUID string `json:"malformed_pod_uid"` + MalformedPVCUID string `json:"malformed_pvc_uid"` + Insecure executionenv.EnvironmentRef `json:"insecure_environment"` + InsecurePodUID string `json:"insecure_pod_uid"` + InsecurePVCUID string `json:"insecure_pvc_uid"` +} + +// SeedLegacyMigrationFixture creates the narrow security-compatible prototype +// accepted by MigrateEnvironment plus malformed and insecure negative controls. +// It is excluded from regular builds. +func SeedLegacyMigrationFixture(ctx context.Context, d dynamic.Interface, kube kubernetes.Interface, namespace, profilesPath string) (LegacyMigrationSeed, error) { + profiles, err := LoadProfiles(profilesPath) + if err != nil { + return LegacyMigrationSeed{}, err + } + profile, ok := profiles.get("go") + if !ok { + return LegacyMigrationSeed{}, fmt.Errorf("go profile unavailable") + } + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "production-migration"} + client := "spiffe://mecatl.test/client/mecak8s" + recognized, err := seedOneLegacyEnvironment(ctx, d, kube, namespace, profile, owner, client, "legacy-migration", "legacy-migration-binding", []any{"legacy-migration-binding"}, false) + if err != nil { + return LegacyMigrationSeed{}, err + } + malformed, err := seedOneLegacyEnvironment(ctx, d, kube, namespace, profile, owner, client, "legacy-migration-malformed", "legacy-malformed-binding", []any{""}, false) + if err != nil { + return LegacyMigrationSeed{}, err + } + insecure, err := seedOneLegacyEnvironment(ctx, d, kube, namespace, profile, owner, client, "legacy-migration-insecure", "legacy-insecure-binding", []any{"legacy-insecure-binding"}, true) + if err != nil { + return LegacyMigrationSeed{}, err + } + return LegacyMigrationSeed{Environment: recognized.ref, Owner: owner, Binding: "legacy-migration-binding", PodUID: recognized.podUID, PVCUID: recognized.pvcUID, Malformed: malformed.ref, MalformedPodUID: malformed.podUID, MalformedPVCUID: malformed.pvcUID, Insecure: insecure.ref, InsecurePodUID: insecure.podUID, InsecurePVCUID: insecure.pvcUID}, nil +} + +// Kubernetes defaults resource-quota usage resync to five minutes. Discovery of +// a new CRD need not enqueue a quota that has not accounted for that resource. +const legacyQuotaWait = 6 * time.Minute + +type seededLegacyEnvironment struct { + ref executionenv.EnvironmentRef + podUID, pvcUID string +} + +func seedOneLegacyEnvironment(ctx context.Context, d dynamic.Interface, kube kubernetes.Interface, namespace string, profile resolvedProfile, owner executionenv.Owner, client, name, binding string, references []any, insecure bool) (seededLegacyEnvironment, error) { + // Deployment readiness does not imply that quota admission has initialized + // accounting, especially for the freshly installed custom resource. + started := time.Now() + var missing []string + if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, legacyQuotaWait, true, func(ctx context.Context) (bool, error) { + quota, err := kube.CoreV1().ResourceQuotas(namespace).Get(ctx, "mecatl-execution", metav1.GetOptions{}) + if err != nil { + return false, err + } + missing = nil + ready := true + for name, configured := range quota.Spec.Hard { + hard, hasHard := quota.Status.Hard[name] + _, hasUsed := quota.Status.Used[name] + if !hasHard || hard.Cmp(configured) != 0 || !hasUsed { + ready = false + switch name { + case "pods", "persistentvolumeclaims", "count/executionenvironments.execution.mecatl.dev", "requests.cpu", "requests.memory", "requests.storage", "requests.ephemeral-storage", "limits.cpu", "limits.memory", "limits.ephemeral-storage": + missing = append(missing, string(name)) + } + } + } + return ready, nil + }); err != nil { + if errors.Is(err, context.DeadlineExceeded) { + err = context.DeadlineExceeded + } else if errors.Is(err, context.Canceled) { + err = context.Canceled + } else if apierrors.IsForbidden(err) { + err = &apierrors.StatusError{ErrStatus: metav1.Status{Reason: metav1.StatusReasonForbidden, Code: 403, Message: "forbidden"}} + } else { + err = errors.New("api_error") + } + // The fixture CLI prints this diagnostic; never wrap raw API response data. + slices.Sort(missing) + return seededLegacyEnvironment{}, fmt.Errorf("wait for legacy fixture quota accounting: missing_or_mismatched_keys=%v elapsed=%s: %w", missing, time.Since(started).Round(time.Millisecond), err) + } + revision, err := randomID() + if err != nil { + return seededLegacyEnvironment{}, err + } + spec := map[string]any{ + "schemaVersion": int64(1), "allocationID": name, "revision": revision, + "ownerHash": ownerHash(owner), "ownerIssuer": owner.Issuer, "ownerSubject": owner.Subject, + "clientHash": hashText(client), "bindingID": binding, "requestFingerprint": hashText("legacy-fixture-" + name), + "profile": "go", "profileDigest": profile.Digest, "image": profile.Spec.Image, + "storageClass": profile.Spec.StorageClass, "storageSize": profile.Spec.StorageSize, + "resources": map[string]any{"cpuRequest": profile.Spec.CPURequest, "memoryRequest": profile.Spec.MemoryRequest, "cpuLimit": profile.Spec.CPULimit, "memoryLimit": profile.Spec.MemoryLimit}, + "desired": "Active", + } + object := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", + "metadata": map[string]any{"name": name, "namespace": namespace, "finalizers": []any{environmentFinalizer}}, + "spec": spec, + }} + resources := d.Resource(ExecutionEnvironmentGVR).Namespace(namespace) + created, err := resources.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + created, err = resources.Create(ctx, object, metav1.CreateOptions{}) + } else if err == nil { + revision = textNested(created.Object, "spec", "revision") + podUID := textNested(created.Object, "status", "pod", "uid") + pvcUID := textNested(created.Object, "status", "pvc", "uid") + if revision != "" && podUID != "" && pvcUID != "" { + return seededLegacyEnvironment{ref: executionenv.EnvironmentRef{ID: name, Revision: revision}, podUID: podUID, pvcUID: pvcUID}, nil + } + } + if err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("create legacy environment %s: %w", name, err) + } + r := NewReconciler(d, kube, namespace, &Profiles{byName: map[string]resolvedProfile{"go": profile}}) + pvcName, podName := resourceName("workspace", name), resourceName("executor", name) + pvc, err := r.ensurePVC(ctx, created, profile, pvcName) + if err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("create legacy PVC %s: %w", name, err) + } + pod, err := r.ensurePod(ctx, created, profile, podName, pvcName) + if err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("create legacy Pod %s: %w", name, err) + } + if insecure { + pod, err = replaceWithInsecureLegacyPod(ctx, kube, namespace, pod) + if err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("create insecure legacy Pod %s: %w", name, err) + } + } + now := time.Now().UTC().Format(time.RFC3339Nano) + created.Object["status"] = map[string]any{ + "schemaVersion": int64(1), "observedGeneration": created.GetGeneration(), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": fenceHealthy, + "references": references, "pvc": map[string]any{"name": pvc.Name, "uid": string(pvc.UID)}, "pod": map[string]any{"name": pod.Name, "uid": string(pod.UID)}, + "conditions": []any{map[string]any{"type": "Ready", "status": "True", "reason": "LegacyFixtureReady", "message": "security-compatible prototype runtime created before CRD upgrade", "observedGeneration": created.GetGeneration(), "lastTransitionTime": now}}, + } + if _, err := resources.UpdateStatus(ctx, created, metav1.UpdateOptions{}); err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("persist legacy status %s: %w", name, err) + } + return seededLegacyEnvironment{ref: executionenv.EnvironmentRef{ID: name, Revision: revision}, podUID: string(pod.UID), pvcUID: string(pvc.UID)}, nil +} + +func replaceWithInsecureLegacyPod(ctx context.Context, kube kubernetes.Interface, namespace string, pod *corev1.Pod) (*corev1.Pod, error) { + pods := kube.CoreV1().Pods(namespace) + if _, err := pods.Patch(ctx, pod.Name, types.MergePatchType, []byte(`{"metadata":{"finalizers":[]}}`), metav1.PatchOptions{}); err != nil { + return nil, err + } + zero := int64(0) + if err := pods.Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &zero}); err != nil { + return nil, err + } + for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); { + if _, err := pods.Get(ctx, pod.Name, metav1.GetOptions{}); apierrors.IsNotFound(err) { + break + } else if err != nil { + return nil, err + } + time.Sleep(100 * time.Millisecond) + } + if _, err := pods.Get(ctx, pod.Name, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("secure fixture Pod deletion did not complete: %w", err) + } + candidate := pod.DeepCopy() + candidate.ObjectMeta = metav1.ObjectMeta{Name: pod.Name, Namespace: namespace, Labels: pod.Labels, Finalizers: pod.Finalizers, OwnerReferences: pod.OwnerReferences} + candidate.Status = corev1.PodStatus{} + automount := true + candidate.Spec.AutomountServiceAccountToken = &automount + return pods.Create(ctx, candidate, metav1.CreateOptions{}) +} diff --git a/internal/adapter/executioncontroller/legacy_fixture_kind_test.go b/internal/adapter/executioncontroller/legacy_fixture_kind_test.go new file mode 100644 index 0000000000..fc710b19b4 --- /dev/null +++ b/internal/adapter/executioncontroller/legacy_fixture_kind_test.go @@ -0,0 +1,128 @@ +//go:build kind_execution_e2e + +package executioncontroller + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestLegacyQuotaWaitCoversDefaultControllerResync(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + kube := kubefake.NewClientset() + kube.PrependReactor("get", "resourcequotas", func(ktesting.Action) (bool, runtime.Object, error) { + cancel() + return true, nil, context.Canceled + }) + if legacyQuotaWait < 6*time.Minute { + t.Fatal("fixture must cover the five-minute quota resync plus margin") + } + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + _, err := seedOneLegacyEnvironment(ctx, d, kube, "test", resolvedProfile{}, executionenv.Owner{}, "client", "legacy", "binding", nil, false) + if !errors.Is(err, context.Canceled) || len(d.Actions()) != 0 { + t.Fatal("cancellation must stop before creating fixtures") + } +} + +func TestLegacyFixtureWaitsForQuotaAccountingBeforeCreate(t *testing.T) { + for _, mode := range []string{"delayed", "timeout", "cancel", "forbidden", "api-error"} { + t.Run(mode, func(t *testing.T) { + quota := &corev1.ResourceQuota{ObjectMeta: metav1.ObjectMeta{Name: "mecatl-execution", Namespace: "test"}, Spec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{corev1.ResourceName("count/executionenvironments.execution.mecatl.dev"): resource.MustParse("10"), corev1.ResourcePods: resource.MustParse("10"), corev1.ResourceName("private-quota-key"): resource.MustParse("12345")}}} + kube := kubefake.NewClientset() + reads, creates := 0, 0 + ctx, cancel := context.WithTimeout(context.Background(), 350*time.Millisecond) + defer cancel() + const hostile = "https://sentinel.invalid/private?token=sk-fake-quota-secret" + kube.PrependReactor("get", "resourcequotas", func(ktesting.Action) (bool, runtime.Object, error) { + reads++ + if mode == "forbidden" { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Resource: "resourcequotas"}, quota.Name, errors.New(hostile)) + } + if mode == "api-error" { + return true, nil, apierrors.NewInternalError(errors.New(hostile)) + } + // Partially initialized status must not pass: all configured resources matter. + quota.Status.Hard = quota.Spec.Hard.DeepCopy() + quota.Status.Used = corev1.ResourceList{corev1.ResourcePods: resource.MustParse("0")} + if mode == "timeout" { + quota.Status.Hard[corev1.ResourcePods] = resource.MustParse("20") + } + if mode == "delayed" && reads >= 2 { + quota.Status.Used = quota.Spec.Hard.DeepCopy() + } + if mode == "cancel" { + cancel() + } + return true, quota.DeepCopy(), nil + }) + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + stop := errors.New("create reached") + d.PrependReactor("create", "executionenvironments", func(ktesting.Action) (bool, runtime.Object, error) { + creates++ + if reads < 2 || mode != "delayed" { + t.Error("create before quota accounting initialized") + } + return true, nil, stop + }) + _, err := seedOneLegacyEnvironment(ctx, d, kube, "test", resolvedProfile{}, executionenv.Owner{}, "client", "legacy", "binding", nil, false) + if mode == "delayed" { + if !errors.Is(err, stop) || creates != 1 { + t.Fatalf("creates=%d reads=%d error=%v", creates, reads, err) + } + return + } + if err == nil || creates != 0 { + t.Fatalf("creates=%d error=%v", creates, err) + } + if mode == "forbidden" || mode == "api-error" { + var reported strings.Builder + fmt.Fprintln(&reported, err) // Same reporting boundary as legacyfixture.fail. + for _, secret := range []string{"https://sentinel.invalid", "sk-fake-quota-secret"} { + if strings.Contains(err.Error(), secret) || strings.Contains(reported.String(), secret) { + t.Fatal("quota error disclosed API response data") + } + } + if reads != 1 { + t.Fatalf("API error retried: reads=%d", reads) + } + } + switch mode { + case "timeout": + if strings.Contains(err.Error(), "private-quota-key") || strings.Contains(err.Error(), "12345") || !strings.Contains(err.Error(), "pods") { + t.Fatalf("quota diagnostics lost mismatch or leaked data: %v", err) + } + if !strings.Contains(err.Error(), "count/executionenvironments.execution.mecatl.dev") || !strings.Contains(err.Error(), "elapsed=") { + t.Fatalf("missing bounded quota diagnostic: %v", err) + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatal(err) + } + case "cancel": + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + case "forbidden": + if !apierrors.IsForbidden(err) || reads != 1 { + t.Fatalf("reads=%d error=%v", reads, err) + } + } + }) + } +} diff --git a/internal/adapter/executioncontroller/operation_lease.go b/internal/adapter/executioncontroller/operation_lease.go new file mode 100644 index 0000000000..5f0bcfe38f --- /dev/null +++ b/internal/adapter/executioncontroller/operation_lease.go @@ -0,0 +1,50 @@ +package executioncontroller + +import ( + "context" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func operationMatches(o *unstructured.Unstructured, operationID, holderID, claimID string, epoch uint64) bool { + m, found, err := unstructured.NestedMap(o.Object, "status", "activeOperation") + return err == nil && found && text(m, "id") == operationID && text(m, "holderID") == holderID && text(m, "claimID") == claimID && intNested(m, "epoch") == int64(epoch) //nolint:gosec // epochs are bounded before persistence. +} + +func (s *Store) renewOperationLease(ctx context.Context, environment, operationID, claimID string, epoch uint64) error { + interval := s.opTTL / 3 + if interval <= 0 { + interval = operationLeaseTTL / 3 + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + err := s.retryUpdateStatus(ctx, environment, func(o *unstructured.Unstructured) error { + if !operationMatches(o, operationID, s.holderID, claimID, epoch) { + return incompatibleOperationError() + } + m, _, _ := unstructured.NestedMap(o.Object, "status", "activeOperation") + now := s.now() + m["renewedAt"] = now.Format(time.RFC3339Nano) + m["expiresAt"] = now.Add(s.opTTL).Format(time.RFC3339Nano) + return unstructured.SetNestedMap(o.Object, m, "status", "activeOperation") + }) + if err != nil { + return err + } + } + } +} + +func incompatibleOperationError() error { + return &operationLeaseError{} +} + +type operationLeaseError struct{} + +func (*operationLeaseError) Error() string { return "operation lease ownership changed" } diff --git a/internal/adapter/executioncontroller/podexec.go b/internal/adapter/executioncontroller/podexec.go new file mode 100644 index 0000000000..2ed4ce2995 --- /dev/null +++ b/internal/adapter/executioncontroller/podexec.go @@ -0,0 +1,120 @@ +package executioncontroller + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "sync" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// PodExecutor invokes the fixed helper in an already-selected executor Pod. +type PodExecutor struct { + config *rest.Config + kube kubernetes.Interface + namespace string +} + +// NewPodExecutor constructs a namespace-scoped Pod executor transport. +func NewPodExecutor(config *rest.Config, kube kubernetes.Interface, namespace string) *PodExecutor { + return &PodExecutor{config: rest.CopyConfig(config), kube: kube, namespace: namespace} +} + +// Execute runs one bounded helper request in pod. +func (p *PodExecutor) Execute(ctx context.Context, pod string, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + body, err := json.Marshal(q) + if err != nil { + return executionenv.ExecutorResponse{}, err + } + if len(body) > executionenv.MaxJSONBody { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "executor request exceeds limit"} + } + req := p.kube.CoreV1().RESTClient().Post().Resource("pods").Name(pod).Namespace(p.namespace).SubResource("exec").VersionedParams(&corev1.PodExecOptions{Container: "executor", Command: []string{"/mecatl-executor"}, Stdin: true, Stdout: true, Stderr: true, TTY: false}, scheme.ParameterCodec) + exec, err := remotecommand.NewSPDYExecutor(p.config, "POST", req.URL()) + if err != nil { + return executionenv.ExecutorResponse{}, fmt.Errorf("create executor stream: %w", err) + } + var stdout, stderr limitedBuffer + stdout.limit = executionenv.MaxJSONBody + stderr.limit = 64 << 10 + var stdin io.Reader = bytes.NewReader(body) + var closeInput func() + if q.Operation == executionenv.OpCommandStart { + stdin, closeInput = commandInput(ctx, body) + defer closeInput() + } + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{Stdin: stdin, Stdout: &stdout, Stderr: &stderr}) + if err != nil { + return executionenv.ExecutorResponse{}, fmt.Errorf("executor stream failed: %w", err) + } + var envelope executionenv.ExecutorEnvelope + if err := executionenv.DecodeStrict(stdout.Bytes(), &envelope); err != nil { + return executionenv.ExecutorResponse{}, fmt.Errorf("invalid executor response: %w", err) + } + if envelope.Error != nil && envelope.Response != nil { + return executionenv.ExecutorResponse{}, errors.New("executor returned both result and error") + } + if envelope.Error != nil { + return executionenv.ExecutorResponse{}, envelope.Error + } + if envelope.Response == nil { + return executionenv.ExecutorResponse{}, errors.New("executor returned no result") + } + return *envelope.Response, nil +} + +func commandInput(ctx context.Context, body []byte) (io.Reader, func()) { + reader, writer := io.Pipe() + done := make(chan struct{}) + go func() { + payload := append(append(make([]byte, 0, len(body)+1), body...), '\n') + if _, err := writer.Write(payload); err != nil { + _ = writer.CloseWithError(err) + } + }() + go func() { + select { + case <-ctx.Done(): + _ = writer.CloseWithError(ctx.Err()) + case <-done: + } + }() + var once sync.Once + return reader, func() { + once.Do(func() { + close(done) + _ = writer.Close() + _ = reader.Close() + }) + } +} + +type limitedBuffer struct { + bytes.Buffer + limit int + overflow bool +} + +func (b *limitedBuffer) Write(p []byte) (int, error) { + if b.Len()+len(p) > b.limit { + remaining := b.limit - b.Len() + if remaining > 0 { + _, _ = b.Buffer.Write(p[:remaining]) + } + b.overflow = true + return len(p), errors.New("executor output exceeds limit") + } + return b.Buffer.Write(p) +} + +var _ ExecutorTransport = (*PodExecutor)(nil) diff --git a/internal/adapter/executioncontroller/podexec_test.go b/internal/adapter/executioncontroller/podexec_test.go new file mode 100644 index 0000000000..4108f48c93 --- /dev/null +++ b/internal/adapter/executioncontroller/podexec_test.go @@ -0,0 +1,36 @@ +package executioncontroller + +import ( + "context" + "io" + "testing" + "time" +) + +func TestCommandInputClosesIndependentlyOnCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + reader, closeInput := commandInput(ctx, []byte(`{"operation":"command.start"}`)) + defer closeInput() + + line := make([]byte, len(`{"operation":"command.start"}`)+1) + if _, err := io.ReadFull(reader, line); err != nil { + t.Fatal(err) + } + cancel() + readDone := make(chan error, 1) + go func() { + var one [1]byte + _, err := reader.Read(one[:]) + readDone <- err + }() + deadline := time.NewTimer(2 * time.Second) + defer deadline.Stop() + select { + case err := <-readDone: + if err == nil { + t.Fatal("command control pipe remained open after cancellation") + } + case <-deadline.C: + t.Fatal("command control pipe did not close after cancellation") + } +} diff --git a/internal/adapter/executioncontroller/production_lifecycle_test.go b/internal/adapter/executioncontroller/production_lifecycle_test.go new file mode 100644 index 0000000000..c62a9910c2 --- /dev/null +++ b/internal/adapter/executioncontroller/production_lifecycle_test.go @@ -0,0 +1,573 @@ +package executioncontroller + +import ( + "context" + "errors" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestPodTerminalRequiresEveryDeclaredContainerByName(t *testing.T) { + terminated := corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}} + base := &corev1.Pod{Spec: corev1.PodSpec{InitContainers: []corev1.Container{{Name: "init"}}, Containers: []corev1.Container{{Name: "executor"}}}, Status: corev1.PodStatus{Phase: corev1.PodSucceeded, InitContainerStatuses: []corev1.ContainerStatus{{Name: "init", State: terminated}}, ContainerStatuses: []corev1.ContainerStatus{{Name: "executor", State: terminated}}}} + if !podTerminal(base) { + t.Fatal("exact terminated init and regular containers were rejected") + } + cases := map[string]func(*corev1.Pod){ + "init waiting": func(p *corev1.Pod) { + p.Status.InitContainerStatuses[0].State = corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{}} + }, + "init running": func(p *corev1.Pod) { + p.Status.InitContainerStatuses[0].State = corev1.ContainerState{Running: &corev1.ContainerStateRunning{}} + }, + "missing init": func(p *corev1.Pod) { p.Status.InitContainerStatuses = nil }, + "duplicate regular": func(p *corev1.Pod) { + p.Status.ContainerStatuses = append(p.Status.ContainerStatuses, p.Status.ContainerStatuses[0]) + }, + "wrong regular name": func(p *corev1.Pod) { p.Status.ContainerStatuses[0].Name = "other" }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + pod := base.DeepCopy() + mutate(pod) + if podTerminal(pod) { + t.Fatal("invalid container evidence proved terminal") + } + }) + } +} + +func lifecycleAdminEnvironment(schema int64, refs []any) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", + "metadata": map[string]any{"name": "env", "namespace": "ns", "uid": "env-uid", "finalizers": []any{environmentFinalizer}}, + "spec": map[string]any{"schemaVersion": schema, "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, + "status": map[string]any{"schemaVersion": schema, "epoch": int64(4), "grantGeneration": int64(1), "fenceState": fenceHealthy, "references": refs, + "pvc": map[string]any{"name": "workspace", "uid": "pvc-uid"}, "pod": map[string]any{"name": "executor", "uid": "pod-uid"}, + "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}, + }} +} + +func terminalExecutor() *corev1.Pod { + profile, _ := testProfiles().get("go") + noPriv, nonroot, ro := false, true, true + uid := int64(65532) + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "executor", Namespace: "ns", UID: types.UID("pod-uid"), Finalizers: []string{executorFinalizer}, Labels: map[string]string{"execution.mecatl.dev/environment": "env", "execution.mecatl.dev/profile": hashText("go")[:16]}, OwnerReferences: []metav1.OwnerReference{{APIVersion: "execution.mecatl.dev/v1alpha1", Kind: "ExecutionEnvironment", Name: "env", UID: types.UID("env-uid"), Controller: &nonroot}}}, Spec: corev1.PodSpec{AutomountServiceAccountToken: &noPriv, RuntimeClassName: &profile.Spec.RuntimeClassName, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &nonroot, RunAsUser: &uid, RunAsGroup: &uid, FSGroup: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "executor", Image: profile.Spec.Image, Command: []string{"/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &noPriv, ReadOnlyRootFilesystem: &ro, RunAsNonRoot: &nonroot, RunAsUser: &uid, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: profile.CPURequest, corev1.ResourceMemory: profile.MemoryRequest, corev1.ResourceEphemeralStorage: profile.EphemeralStorageRequest}, Limits: corev1.ResourceList{corev1.ResourceCPU: profile.CPULimit, corev1.ResourceMemory: profile.MemoryLimit, corev1.ResourceEphemeralStorage: profile.EphemeralStorageLimit}}, VolumeMounts: []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}, {Name: "tmp", MountPath: "/tmp"}}}}, Volumes: []corev1.Volume{{Name: "workspace", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "workspace"}}}, {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &profile.TmpSizeLimit}}}}}, Status: corev1.PodStatus{Phase: corev1.PodFailed, ContainerStatuses: []corev1.ContainerStatus{{Name: "executor", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 137}}}}}} +} + +func retainedPVC() *corev1.PersistentVolumeClaim { + profile, _ := testProfiles().get("go") + mode := corev1.PersistentVolumeFilesystem + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "workspace", Namespace: "ns", UID: types.UID("pvc-uid"), Labels: map[string]string{"execution.mecatl.dev/environment": "env", "execution.mecatl.dev/revision": "rev", "execution.mecatl.dev/allocation-uid": "env-uid"}}, Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: &profile.Spec.StorageClass, AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, VolumeMode: &mode, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: profile.StorageSize}}}} +} + +func adminRequestFixture() adminLifecycleRequest { + return adminLifecycleRequest{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, OwnerHash: "owner", Client: "client", ExpectedEpoch: 4, ExpectedPodUID: "pod-uid", ExpectedPVCUID: "pvc-uid", OperationID: "admin-operation"} +} + +func TestHealthyTerminalPodCanStartLifecycleWithoutReadyCondition(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Ready", false, "PodTerminated", "executor exited naturally") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + q := adminRequestFixture() + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatal(err) + } + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatalf("retry was not idempotent: %v", err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "terminationProof", operationIDField) != q.OperationID || textNested(got.Object, "status", "lifecycleOperation", "id") != q.OperationID { + t.Fatalf("terminal proof and lifecycle operation were not persisted: %v", got.Object["status"]) + } +} + +func TestHealthyTerminalLifecycleRejectsIncompleteEvidence(t *testing.T) { + cases := map[string]func(*corev1.Pod){ + "missing pod": func(*corev1.Pod) {}, + "init running": func(p *corev1.Pod) { + p.Spec.InitContainers = []corev1.Container{{Name: "init"}} + p.Status.InitContainerStatuses = []corev1.ContainerStatus{{Name: "init", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}} + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Ready", false, "PodTerminated", "executor exited naturally") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + objects := []runtime.Object{retainedPVC()} + if name != "missing pod" { + pod := terminalExecutor() + mutate(pod) + objects = append(objects, pod) + } + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(objects...)) + if err := store.RetireExact(t.Context(), adminRequestFixture()); err == nil { + t.Fatal("incomplete terminal evidence admitted lifecycle") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "terminationProof", "podUID") != "" || textNested(got.Object, "status", "lifecycleOperation", "id") != "" { + t.Fatalf("rejected evidence changed lifecycle state: %v", got.Object["status"]) + } + }) + } +} + +func TestRetirementCompletionReceiptReplaysOnlyExactOperation(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, terminationProof(q, terminalExecutor()), "status", "terminationProof") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": q.OperationID, "type": "RetireEnvironment", "phase": "WaitingForPodDeletion", "expectedEpoch": int64(4), "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID, "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + r := NewReconciler(dynamicClient, kubefake.NewSimpleClientset(retainedPVC()), "ns", testProfiles()) + if err := r.finishRetirement(t.Context(), env, q.OperationID); err != nil { + t.Fatal(err) + } + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatalf("lost-reply retry failed: %v", err) + } + q.OperationID = "different-operation" + if err := store.RetireExact(t.Context(), q); err == nil { + t.Fatal("different operation silently succeeded against retired environment") + } + q = adminRequestFixture() + q.OwnerHash = "other-owner" + if err := store.RetireExact(t.Context(), q); err == nil { + t.Fatal("wrong subject replayed retirement receipt") + } +} + +func TestOldSchemaFailsClosedAndExplicitMigrationConvertsReferences(t *testing.T) { + env := lifecycleAdminEnvironment(0, []any{"session-a"}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + kube := kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC()) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kube) + _, err := store.Attach(t.Context(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "session-a") + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeNotReady { + t.Fatalf("old schema attach error=%v", err) + } + q := adminRequestFixture() + q.ExpectedSchema = 0 + if err := store.MigrateEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + if err := store.MigrateEnvironment(t.Context(), q); err != nil { + t.Fatalf("migration retry was not idempotent: %v", err) + } + got, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if intNested(got.Object, "spec", "schemaVersion") != 2 || intNested(got.Object, "status", "schemaVersion") != 2 || intNested(got.Object, "status", "epoch") != 5 { + t.Fatalf("migration did not advance schema and epoch: %v", got.Object) + } + refs, err := referenceRecords(got) + if err != nil || len(refs) != 1 || refs[0].BindingID != "session-a" || refs[0].State != executionenv.ReferencePublished { + t.Fatalf("migrated refs=%+v err=%v", refs, err) + } + fromSchema, found, err := unstructured.NestedInt64(got.Object, "status", "lastMigrationFromSchema") + if err != nil || !found || fromSchema != 0 { + t.Fatal("schema-zero migration lost exact receipt presence") + } + unstructured.RemoveNestedField(got.Object, "status", "lastMigrationFromSchema") + if _, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(t.Context(), got, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := store.MigrateEnvironment(t.Context(), q); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("old receipt without source schema replayed: %v", err) + } +} + +func TestMigrationReceiptExpiresAfterReconciledReplacement(t *testing.T) { + for name, schema := range map[string]int64{"legacy": 0, "prototype": 1} { + t.Run(name, func(t *testing.T) { + env := lifecycleAdminEnvironment(schema, []any{"session-a"}) + pod, pvc := terminalExecutor(), retainedPVC() + pod.Name, pvc.Name = resourceName("executor", "env"), resourceName("workspace", "env") + pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name + _ = unstructured.SetNestedField(env.Object, pod.Name, "status", "pod", "name") + _ = unstructured.SetNestedField(env.Object, pvc.Name, "status", "pvc", "name") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(pod, pvc) + // The fake API must retain a finalizer-protected Pod until the real + // reconciler has recorded terminal proof and removed its finalizer. + k.PrependReactor("delete", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + obj, err := k.Tracker().Get(corev1.SchemeGroupVersion.WithResource("pods"), "ns", action.(k8stesting.DeleteAction).GetName()) + if err != nil { + return true, nil, err + } + return len(obj.(*corev1.Pod).Finalizers) != 0, nil, nil + }) + k.PrependReactor("create", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + created := action.(k8stesting.CreateAction).GetObject().(*corev1.Pod) + created.UID = "replacement-pod" + created.Status = replacementExecutor(true).Status + return false, nil, nil + }) + store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) + q := adminRequestFixture() + q.ExpectedSchema = schema + if err := store.MigrateEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + if err := store.MigrateEnvironment(t.Context(), q); err != nil { + t.Fatalf("exact receipt replay before replacement: %v", err) + } + replace := q + replace.OperationID = "replace-after-migration" + replace.ExpectedEpoch++ + if err := store.ReplaceExecutor(t.Context(), replace); err != nil { + t.Fatal(err) + } + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + // Drive every replacement phase, then an ordinary ready reconcile. + for range 8 { + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !conditionTrue(got, "Ready") || textNested(got.Object, "status", "pod", "uid") != "replacement-pod" || textNested(got.Object, "status", "pvc", "uid") != q.ExpectedPVCUID || intNested(got.Object, "status", "epoch") != 6 { + t.Fatalf("replacement did not complete normally: %v", got.Object["status"]) + } + changed := q + changed.ExpectedPodUID = "replacement-pod" + for _, replay := range []adminLifecycleRequest{changed, q} { + var controlled *executionenv.Error + if err := store.MigrateEnvironment(t.Context(), replay); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("expired migration replay with Pod %q: %v", replay.ExpectedPodUID, err) + } + } + if textNested(got.Object, "status", "lastMigrationOperationID") != "" { + t.Fatal("replacement retained migration operation receipt") + } + if _, found, err := unstructured.NestedInt64(got.Object, "status", "lastMigrationFromSchema"); err != nil || found { + t.Fatal("replacement retained migration source schema receipt") + } + }) + } +} + +func TestInsecurePrototypeMigrationRejectedWithoutRewrite(t *testing.T) { + env := lifecycleAdminEnvironment(1, []any{"session-a"}) + pod := terminalExecutor() + automount := true + pod.Spec.AutomountServiceAccountToken = &automount + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(pod, retainedPVC())) + q := adminRequestFixture() + q.ExpectedSchema = 1 + if err := store.MigrateEnvironment(t.Context(), q); err == nil { + t.Fatal("insecure prototype executor migrated") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if intNested(got.Object, "spec", "schemaVersion") != 1 || intNested(got.Object, "status", "schemaVersion") != 1 || intNested(got.Object, "status", "epoch") != 4 { + t.Fatalf("insecure prototype was rewritten: %v", got.Object) + } +} + +func TestUnknownSchemaMigrationRejectedWithoutRewrite(t *testing.T) { + env := lifecycleAdminEnvironment(7, []any{}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + q := adminRequestFixture() + q.ExpectedSchema = 7 + if err := store.MigrateEnvironment(t.Context(), q); err == nil { + t.Fatal("unknown schema migrated") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if intNested(got.Object, "spec", "schemaVersion") != 7 || intNested(got.Object, "status", "epoch") != 4 { + t.Fatalf("unknown schema was rewritten: %v", got.Object) + } +} + +func TestExpiredOperationLeaseFencesWithoutClearingIdentity(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "peer-op", "operation": "file.replace", "claimID": "claim", "runID": "run", "epoch": int64(4), "holderID": "peer", "renewedAt": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano), "expiresAt": time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano)}, "status", "activeOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + r := NewReconciler(dynamicClient, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "fenceState") != "FenceUnknown" || textNested(got.Object, "status", "activeOperation", "id") != "peer-op" { + t.Fatalf("expired operation was not retained and fenced: %v", got.Object["status"]) + } +} + +func TestRecoverRequiresExactTerminalPodAndPersistsProof(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "uncertain", "claimID": "claim", "epoch": int64(4)}, "status", "activeOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + q := adminRequestFixture() + q.OperationID = "recover-op" + if err := store.RecoverEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "fenceState") != fenceHealthy || textNested(got.Object, "status", "activeOperation", "id") != "" || textNested(got.Object, "status", "terminationProof", "podUID") != "pod-uid" { + t.Fatalf("recovery proof/status=%v", got.Object["status"]) + } +} + +func TestRecoveredTerminalProofCanStartExactReplacement(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") + setConditionObject(env, "Ready", false, "FenceUnknown", "holder lost") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "uncertain", "claimID": "claim", "epoch": int64(4)}, "status", "activeOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + q := adminRequestFixture() + q.OperationID = "recover-op" + if err := store.RecoverEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + q.OperationID = "replace-op" + if err := store.ReplaceExecutor(t.Context(), q); err != nil { + t.Fatalf("exact replacement after terminal recovery: %v", err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "lifecycleOperation", "id") != "replace-op" { + t.Fatalf("replacement operation missing after recovery: %v", got.Object["status"]) + } +} + +func TestRecoverMissingPodRemainsFenceUnknown(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(retainedPVC())) + if err := store.RecoverEnvironment(t.Context(), adminRequestFixture()); err == nil { + t.Fatal("missing executor accepted as termination proof") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "fenceState") != "FenceUnknown" || textNested(got.Object, "status", "terminationProof", "podUID") != "" { + t.Fatalf("missing executor changed fenced state: %v", got.Object["status"]) + } +} + +func TestCompletedReplacementReceiptReplaysExactOperation(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}}) + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, map[string]any{operationIDField: q.OperationID, "previousPodUID": q.ExpectedPodUID, "replacementPodUID": "new-pod", "pvcUID": q.ExpectedPVCUID, "previousEpoch": int64(q.ExpectedEpoch), "replacementEpoch": int64(q.ExpectedEpoch + 1)}, "status", "lastReplacement") + _ = unstructured.SetNestedField(env.Object, int64(q.ExpectedEpoch+1), "status", "epoch") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"name": "executor", "uid": "new-pod"}, "status", "pod") + store := NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env), "ns", testProfiles(), nil) + if err := store.ReplaceExecutor(t.Context(), q); err != nil { + t.Fatalf("exact completed replacement did not replay: %v", err) + } + q.OperationID = "different" + if err := store.ReplaceExecutor(t.Context(), q); err == nil { + t.Fatal("different operation replayed completed replacement") + } +} + +func TestReplacementQuiescesAndPendingReferenceBlocksRetirement(t *testing.T) { + published := map[string]any{"bindingID": "session", "state": "Published", "operationID": "publish", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)} + env := lifecycleAdminEnvironment(2, []any{published}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + q := adminRequestFixture() + if err := store.ReplaceExecutor(t.Context(), q); err != nil { + t.Fatal(err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "lifecycleOperation", "phase") != "Quiescing" || conditionTrue(got, "Ready") { + t.Fatalf("replacement did not consume admission: %v", got.Object["status"]) + } + if _, err := store.AcquireRun(t.Context(), q.Environment, "client", "owner", "session", "run", "acquire", time.Minute); err == nil { + t.Fatal("run admitted during replacement") + } + + pending := map[string]any{"bindingID": "pending", "state": "PendingDelete", "operationID": "delete", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)} + env = lifecycleAdminEnvironment(2, []any{pending}) + store = NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env), "ns", testProfiles(), nil) + if err := store.RetireExact(t.Context(), q); err == nil { + t.Fatal("pending reference did not block retirement") + } +} + +func TestReplacementPersistsTerminalProofBeforeRemovingPodFinalizer(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "replace", "type": "ReplaceExecutor", "phase": "WaitingForTermination", "expectedEpoch": int64(4), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + kube := kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC()) + r := NewReconciler(dynamicClient, kube, "ns", testProfiles()) + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "terminationProof", "podUID") != "pod-uid" || textNested(got.Object, "status", "lifecycleOperation", "phase") != "RemovingPodFinalizer" { + t.Fatalf("terminal proof was not persisted first: %v", got.Object["status"]) + } + pod, _ := kube.CoreV1().Pods("ns").Get(t.Context(), "executor", metav1.GetOptions{}) + if !contains(pod.Finalizers, executorFinalizer) { + t.Fatal("executor finalizer was removed in the proof-persistence transition") + } + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + pod, err := kube.CoreV1().Pods("ns").Get(t.Context(), "executor", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if contains(pod.Finalizers, executorFinalizer) { + t.Fatal("executor finalizer remained after durable exact proof") + } + got, _ = dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "lifecycleOperation", "phase") != "WaitingForPodDeletion" { + t.Fatalf("phase=%v", got.Object["status"]) + } +} + +func replacementExecutor(ready bool) *corev1.Pod { + pod := terminalExecutor() + pod.UID = types.UID("new-pod") + pod.Status = corev1.PodStatus{Phase: corev1.PodRunning} + if ready { + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + } + return pod +} + +func replacementLifecycleEnvironment() *unstructured.Unstructured { + env := lifecycleAdminEnvironment(2, []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}}) + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, terminationProof(q, terminalExecutor()), "status", "terminationProof") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": q.OperationID, "type": "ReplaceExecutor", "phase": "CreatingReplacement", "expectedEpoch": int64(4), "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID, "createdAt": "2026-09-21T00:00:00Z"}, "status", "lifecycleOperation") + unstructured.RemoveNestedField(env.Object, "status", "pod") + setConditionObject(env, "Ready", false, "ReplacementStarting", "waiting") + return env +} + +func TestStaleReplacementObservationCannotOverwriteCompletion(t *testing.T) { + stale := replacementLifecycleEnvironment() + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), stale.DeepCopy()) + kube := kubefake.NewSimpleClientset(replacementExecutor(false), retainedPVC()) + observed, release := make(chan struct{}), make(chan struct{}) + kube.PrependReactor("get", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + close(observed) + <-release + return false, nil, nil + }) + r := NewReconciler(dynamicClient, kube, "ns", testProfiles()) + done := make(chan error, 1) + go func() { done <- r.finishReplacement(t.Context(), stale, adminRequestFixture().OperationID) }() + <-observed + + current, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + _ = unstructured.SetNestedField(current.Object, int64(5), "status", "epoch") + _ = unstructured.SetNestedMap(current.Object, map[string]any{"name": "executor", "uid": "new-pod"}, "status", "pod") + _ = unstructured.SetNestedMap(current.Object, map[string]any{operationIDField: adminRequestFixture().OperationID, "previousPodUID": "pod-uid", "replacementPodUID": "new-pod", "pvcUID": "pvc-uid", "previousEpoch": int64(4), "replacementEpoch": int64(5)}, "status", "lastReplacement") + unstructured.RemoveNestedField(current.Object, "status", "lifecycleOperation") + setConditionObject(current, "Ready", true, "ReplacementReady", "replacement executor is ready on the retained workspace") + if _, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(t.Context(), current, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + close(release) + var controlled *executionenv.Error + if err := <-done; !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("stale replacement result=%v", err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if !conditionTrue(got, "Ready") || intNested(got.Object, "status", "epoch") != 5 || textNested(got.Object, "status", "pod", "uid") != "new-pod" { + t.Fatalf("stale unready write damaged completion: %v", got.Object["status"]) + } + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + if allocation, err := store.Attach(t.Context(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "binding"); err != nil || !allocation.Ready || allocation.Epoch != 5 { + t.Fatalf("attach after completed replacement: allocation=%+v err=%v", allocation, err) + } +} + +func TestLifecycleStatusWritersRejectStaleObservations(t *testing.T) { + t.Run("phase regression", func(t *testing.T) { + stale := replacementLifecycleEnvironment() + _ = unstructured.SetNestedField(stale.Object, "Quiescing", "status", "lifecycleOperation", "phase") + current := stale.DeepCopy() + _ = unstructured.SetNestedField(current.Object, "RemovingPodFinalizer", "status", "lifecycleOperation", "phase") + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), current) + r := NewReconciler(client, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if err := r.setLifecyclePhase(t.Context(), stale, adminRequestFixture().OperationID, "Quiescing", "WaitingForTermination"); err == nil { + t.Fatal("stale phase transition succeeded") + } + got, _ := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if phase := textNested(got.Object, "status", "lifecycleOperation", "phase"); phase != "RemovingPodFinalizer" { + t.Fatalf("phase regressed to %q", phase) + } + }) + + t.Run("waiting and fence writes", func(t *testing.T) { + stale := replacementLifecycleEnvironment() + _ = unstructured.SetNestedField(stale.Object, "WaitingForTermination", "status", "lifecycleOperation", "phase") + current := stale.DeepCopy() + _ = unstructured.SetNestedField(current.Object, "RemovingPodFinalizer", "status", "lifecycleOperation", "phase") + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), current) + r := NewReconciler(client, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if err := r.setLifecycleCondition(t.Context(), stale, "Ready", false, "AwaitingTerminalExecutor", "waiting"); err == nil { + t.Fatal("stale waiting condition succeeded") + } + if err := r.setLifecycleFenceUnknown(t.Context(), stale, "stale failure"); err == nil { + t.Fatal("stale fence write succeeded") + } + got, _ := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "fenceState") != fenceHealthy || textNested(got.Object, "status", "lifecycleOperation", "phase") != "RemovingPodFinalizer" { + t.Fatalf("stale writer changed lifecycle status: %v", got.Object["status"]) + } + }) + + t.Run("ordinary reconcile after admission", func(t *testing.T) { + stale := lifecycleAdminEnvironment(2, []any{}) + current := stale.DeepCopy() + _ = unstructured.SetNestedMap(current.Object, map[string]any{"id": "replace", "type": "ReplaceExecutor", "phase": "Quiescing", "expectedEpoch": int64(4), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid", "createdAt": "2026-09-21T00:00:00Z"}, "status", "lifecycleOperation") + setConditionObject(current, "Ready", false, "Quiescing", "replacement admitted") + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), current) + r := NewReconciler(client, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if err := r.updateRuntimeStatus(t.Context(), stale, retainedPVC(), terminalExecutor(), true); err == nil { + t.Fatal("ordinary reconcile overwrote lifecycle admission") + } + got, _ := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if conditionTrue(got, "Ready") || textNested(got.Object, "status", "lifecycleOperation", "id") != "replace" { + t.Fatalf("ordinary reconcile damaged lifecycle admission: %v", got.Object["status"]) + } + }) +} + +func TestWrongPVCUIDCannotStartRetainedDeletion(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + q := adminRequestFixture() + q.ExpectedPVCUID = "wrong" + if err := store.DeleteRetiredEnvironment(t.Context(), q); err == nil { + t.Fatal("wrong PVC UID admitted for deletion") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "lifecycleOperation", "id") != "" { + t.Fatalf("wrong UID created delete operation: %v", got.Object["status"]) + } +} diff --git a/internal/adapter/executioncontroller/profiles.go b/internal/adapter/executioncontroller/profiles.go new file mode 100644 index 0000000000..ea9a46d2cd --- /dev/null +++ b/internal/adapter/executioncontroller/profiles.go @@ -0,0 +1,198 @@ +package executioncontroller + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "sort" + "strings" + "time" + + "github.com/goccy/go-yaml" + "k8s.io/apimachinery/pkg/api/resource" +) + +// ProfilesFile is the strict operator profile-file schema. +type ProfilesFile struct { + Profiles map[string]ProfileSpec `yaml:"profiles"` +} + +// ProfileSpec defines immutable image, storage, resource, and operation bounds. +type ProfileSpec struct { + Image string `yaml:"image"` + StorageClass string `yaml:"storageClass"` + StorageSize string `yaml:"storageSize"` + CPURequest string `yaml:"cpuRequest"` + MemoryRequest string `yaml:"memoryRequest"` + CPULimit string `yaml:"cpuLimit"` + MemoryLimit string `yaml:"memoryLimit"` + EphemeralStorageRequest string `yaml:"ephemeralStorageRequest"` + EphemeralStorageLimit string `yaml:"ephemeralStorageLimit"` + TmpSizeLimit string `yaml:"tmpSizeLimit"` + RuntimeClassName string `yaml:"runtimeClassName"` + MaxFileBytes int64 `yaml:"maxFileBytes"` + MaxCommandBytes int64 `yaml:"maxCommandBytes"` + MaxCommandDuration time.Duration `yaml:"maxCommandDuration"` + MaxEnvironments int `yaml:"maxEnvironments"` +} + +// Profiles is a validated immutable profile registry. +type Profiles struct{ byName map[string]resolvedProfile } +type resolvedProfile struct { + Spec ProfileSpec + Digest string + StorageSize resource.Quantity + CPURequest resource.Quantity + MemoryRequest resource.Quantity + CPULimit resource.Quantity + MemoryLimit resource.Quantity + EphemeralStorageRequest resource.Quantity + EphemeralStorageLimit resource.Quantity + TmpSizeLimit resource.Quantity +} + +// LoadProfiles reads and strictly validates an operator profile file. +func LoadProfiles(path string) (*Profiles, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read profiles: %w", err) + } + var f ProfilesFile + if err := yaml.UnmarshalWithOptions(b, &f, yaml.DisallowUnknownField()); err != nil { + return nil, fmt.Errorf("decode profiles: %w", err) + } + if len(f.Profiles) == 0 { + return nil, errors.New("profiles file has no profiles") + } + p := &Profiles{byName: map[string]resolvedProfile{}} + for name, s := range f.Profiles { + quantities, err := validateProfile(name, s) + if err != nil { + return nil, err + } + canonical, err := yaml.Marshal(s) + if err != nil { + return nil, err + } + sum := sha256.Sum256(canonical) + quantities.Spec = s + quantities.Digest = "sha256:" + hex.EncodeToString(sum[:]) + p.byName[name] = quantities + } + return p, nil +} +func validateProfile(name string, s ProfileSpec) (resolvedProfile, error) { + if name == "" || strings.ContainsAny(name, "/\\") { + return resolvedProfile{}, fmt.Errorf("invalid profile name %q", name) + } + if !strings.Contains(s.Image, "@sha256:") { + return resolvedProfile{}, fmt.Errorf("profile %q image must be digest-pinned", name) + } + if s.StorageClass == "" || s.StorageSize == "" { + return resolvedProfile{}, fmt.Errorf("profile %q requires explicit storage class and size", name) + } + if s.CPURequest == "" || s.MemoryRequest == "" || s.CPULimit == "" || s.MemoryLimit == "" || s.EphemeralStorageRequest == "" || s.EphemeralStorageLimit == "" || s.TmpSizeLimit == "" || s.RuntimeClassName == "" { + return resolvedProfile{}, fmt.Errorf("profile %q requires explicit resource requests, limits, tmp size, and runtime class", name) + } + quantities, err := resolveProfileQuantities(name, s) + if err != nil { + return resolvedProfile{}, err + } + if !validExecutionBounds(s) { + return resolvedProfile{}, fmt.Errorf("profile %q has invalid execution bounds", name) + } + return quantities, nil +} + +func resolveProfileQuantities(name string, s ProfileSpec) (resolvedProfile, error) { + storage, err := positiveQuantity(name, "storageSize", s.StorageSize) + if err != nil { + return resolvedProfile{}, err + } + cpuRequest, err := positiveQuantity(name, "cpuRequest", s.CPURequest) + if err != nil { + return resolvedProfile{}, err + } + memoryRequest, err := positiveQuantity(name, "memoryRequest", s.MemoryRequest) + if err != nil { + return resolvedProfile{}, err + } + cpuLimit, err := positiveQuantity(name, "cpuLimit", s.CPULimit) + if err != nil { + return resolvedProfile{}, err + } + memoryLimit, err := positiveQuantity(name, "memoryLimit", s.MemoryLimit) + if err != nil { + return resolvedProfile{}, err + } + ephemeralRequest, err := positiveQuantity(name, "ephemeralStorageRequest", s.EphemeralStorageRequest) + if err != nil { + return resolvedProfile{}, err + } + ephemeralLimit, err := positiveQuantity(name, "ephemeralStorageLimit", s.EphemeralStorageLimit) + if err != nil { + return resolvedProfile{}, err + } + tmpLimit, err := positiveQuantity(name, "tmpSizeLimit", s.TmpSizeLimit) + if err != nil { + return resolvedProfile{}, err + } + if cpuRequest.Cmp(cpuLimit) > 0 { + return resolvedProfile{}, fmt.Errorf("profile %q cpuRequest exceeds cpuLimit", name) + } + if memoryRequest.Cmp(memoryLimit) > 0 { + return resolvedProfile{}, fmt.Errorf("profile %q memoryRequest exceeds memoryLimit", name) + } + if ephemeralRequest.Cmp(ephemeralLimit) > 0 || tmpLimit.Cmp(ephemeralLimit) > 0 { + return resolvedProfile{}, fmt.Errorf("profile %q ephemeral storage request or tmp limit exceeds ephemeral storage limit", name) + } + return resolvedProfile{StorageSize: storage, CPURequest: cpuRequest, MemoryRequest: memoryRequest, CPULimit: cpuLimit, MemoryLimit: memoryLimit, EphemeralStorageRequest: ephemeralRequest, EphemeralStorageLimit: ephemeralLimit, TmpSizeLimit: tmpLimit}, nil +} + +func validExecutionBounds(s ProfileSpec) bool { + return s.MaxFileBytes > 0 && s.MaxFileBytes <= 5<<20 && + s.MaxCommandBytes > 0 && s.MaxCommandBytes <= 1<<20 && + s.MaxCommandDuration > 0 && s.MaxCommandDuration <= 30*time.Minute && + s.MaxEnvironments > 0 && s.MaxEnvironments <= 10_000 +} + +func positiveQuantity(profile, field, value string) (resource.Quantity, error) { + quantity, err := resource.ParseQuantity(value) + if err != nil { + return resource.Quantity{}, fmt.Errorf("profile %q %s is invalid: %w", profile, field, err) + } + if quantity.Sign() <= 0 { + return resource.Quantity{}, fmt.Errorf("profile %q %s must be positive", profile, field) + } + return quantity, nil +} +func (p *Profiles) get(name string) (resolvedProfile, bool) { + if p == nil { + return resolvedProfile{}, false + } + v, ok := p.byName[name] + return v, ok +} + +func (p *Profiles) clusterResources() (runtimeClasses, storageClasses []string) { + if p == nil { + return nil, nil + } + runtimes := make(map[string]struct{}, len(p.byName)) + storage := make(map[string]struct{}, len(p.byName)) + for _, profile := range p.byName { + runtimes[profile.Spec.RuntimeClassName] = struct{}{} + storage[profile.Spec.StorageClass] = struct{}{} + } + for name := range runtimes { + runtimeClasses = append(runtimeClasses, name) + } + for name := range storage { + storageClasses = append(storageClasses, name) + } + sort.Strings(runtimeClasses) + sort.Strings(storageClasses) + return runtimeClasses, storageClasses +} diff --git a/internal/adapter/executioncontroller/profiles_test.go b/internal/adapter/executioncontroller/profiles_test.go new file mode 100644 index 0000000000..563f95ea20 --- /dev/null +++ b/internal/adapter/executioncontroller/profiles_test.go @@ -0,0 +1,101 @@ +package executioncontroller + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadProfilesStrictAndDigestPinned(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "profiles.yaml") + good := validProfileYAML() + if err := os.WriteFile(path, []byte(good), 0o600); err != nil { + t.Fatal(err) + } + p, err := LoadProfiles(path) + if err != nil { + t.Fatal(err) + } + v, ok := p.get("go") + if !ok || !strings.HasPrefix(v.Digest, "sha256:") { + t.Fatalf("profile=%+v", v) + } + if err := os.WriteFile(path, []byte(good+" unknown: true\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadProfiles(path); err == nil { + t.Fatal("unknown field accepted") + } + unpinned := strings.Replace(good, "@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "", 1) + _ = os.WriteFile(path, []byte(unpinned), 0o600) + if _, err := LoadProfiles(path); err == nil { + t.Fatal("unpinned image accepted") + } +} + +func TestLoadProfilesRejectsInvalidQuantities(t *testing.T) { + cases := map[string]string{ + "malformed": strings.Replace(validProfileYAML(), "cpuRequest: 100m", "cpuRequest: invalid", 1), + "zero": strings.Replace(validProfileYAML(), "memoryRequest: 128Mi", `memoryRequest: "0"`, 1), + "negative": strings.Replace(validProfileYAML(), `cpuLimit: "1"`, `cpuLimit: "-1"`, 1), + "cpu request over limit": strings.Replace(validProfileYAML(), "cpuRequest: 100m", "cpuRequest: 2", 1), + "memory request over limit": strings.Replace(validProfileYAML(), "memoryLimit: 1Gi", "memoryLimit: 64Mi", 1), + "ephemeral request over limit": strings.Replace(validProfileYAML(), "ephemeralStorageLimit: 1Gi", "ephemeralStorageLimit: 32Mi", 1), + "tmp over ephemeral limit": strings.Replace(validProfileYAML(), "tmpSizeLimit: 256Mi", "tmpSizeLimit: 2Gi", 1), + } + for name, content := range cases { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "profiles.yaml") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadProfiles(path); err == nil { + t.Fatal("invalid quantity accepted") + } + }) + } +} + +func TestLoadProfilesAcceptsQuantityUnitVariants(t *testing.T) { + content := validProfileYAML() + content = strings.Replace(content, "storageSize: 2Gi", "storageSize: 500M", 1) + content = strings.Replace(content, "cpuRequest: 100m", "cpuRequest: 0.1", 1) + content = strings.Replace(content, `cpuLimit: "1"`, "cpuLimit: 250m", 1) + content = strings.Replace(content, "memoryRequest: 128Mi", "memoryRequest: 1M", 1) + content = strings.Replace(content, "memoryLimit: 1Gi", "memoryLimit: 2M", 1) + path := filepath.Join(t.TempDir(), "profiles.yaml") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + profiles, err := LoadProfiles(path) + if err != nil { + t.Fatal(err) + } + profile, ok := profiles.get("go") + if !ok || profile.CPURequest.String() != "100m" || profile.MemoryRequest.String() != "1M" || profile.StorageSize.String() != "500M" { + t.Fatalf("parsed quantities=%+v", profile) + } +} + +func validProfileYAML() string { + return `profiles: + go: + image: ghcr.io/example/workload@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + storageClass: standard + storageSize: 2Gi + cpuRequest: 100m + memoryRequest: 128Mi + cpuLimit: "1" + memoryLimit: 1Gi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: sandboxed + maxFileBytes: 1048576 + maxCommandBytes: 65536 + maxCommandDuration: 1m + maxEnvironments: 100 +` +} diff --git a/internal/adapter/executioncontroller/provisioning_retry_test.go b/internal/adapter/executioncontroller/provisioning_retry_test.go new file mode 100644 index 0000000000..d69a7f1476 --- /dev/null +++ b/internal/adapter/executioncontroller/provisioning_retry_test.go @@ -0,0 +1,135 @@ +package executioncontroller + +import ( + "errors" + "sync/atomic" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/util/workqueue" +) + +func TestProvisioningWorkerConvergesWithoutAnotherEvent(t *testing.T) { + for _, resource := range []string{"persistentvolumeclaims", "pods"} { + t.Run(resource, func(t *testing.T) { + ctx := t.Context() + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + var attempts, pvcCreates, podCreates atomic.Int32 + var restored atomic.Bool + k.PrependReactor("create", "*", func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.GetResource().Resource == resource { + attempts.Add(1) + if !restored.Load() { + return true, nil, errors.New("transient quota saturation") + } + } + switch obj := action.(k8stesting.CreateAction).GetObject().(type) { + case *corev1.PersistentVolumeClaim: + pvcCreates.Add(1) + obj.UID = "pvc-stable" + case *corev1.Pod: + podCreates.Add(1) + obj.UID = "pod-stable" + obj.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + } + return false, nil, nil + }) + r := NewReconciler(d, k, "ns", testProfiles()) + r.queue.ShutDown() + r.queue = workqueue.NewTypedRateLimitingQueue(workqueue.NewTypedItemExponentialFailureRateLimiter[string](10*time.Millisecond, 10*time.Millisecond)) + done := make(chan struct{}) + go func() { r.worker(ctx); close(done) }() + t.Cleanup(func() { r.queue.ShutDown(); <-done }) + start := time.Now() + r.queue.Add(env.GetName()) + // Exceed a customary finite retry budget before restoring admission. + waitProvisioning(t, func() bool { return attempts.Load() >= 24 }) + if time.Since(start) < 200*time.Millisecond { + t.Fatal("provisioning retries hot-looped instead of rate limiting") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil || conditionTrue(got, "Ready") || conditionTransition(got, "Ready") == "" { + t.Fatalf("failed provisioning did not retain Ready=false: env=%v err=%v", got, err) + } + if writes := statusUpdateCount(d.Actions()); writes != 1 { + t.Fatalf("unchanged failure rewrote status: writes=%d", writes) + } + // No CR, runtime, or queue event: only the failed external prerequisite changes. + restored.Store(true) + waitProvisioning(t, func() bool { + got, err = d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + return err == nil && conditionTrue(got, "Ready") && r.queue.NumRequeues(env.GetName()) == 0 + }) + if textNested(got.Object, "status", "pvc", "uid") != "pvc-stable" || textNested(got.Object, "status", "pod", "uid") != "pod-stable" || pvcCreates.Load() != 1 || podCreates.Load() != 1 { + t.Fatalf("duplicate or changed runtime identity: pvc=%d pod=%d status=%v", pvcCreates.Load(), podCreates.Load(), got.Object["status"]) + } + }) + } +} + +func waitProvisioning(t *testing.T, ready func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if ready() { + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("worker did not converge without another event") +} + +func TestProvisioningPreservesCreateAndConditionErrors(t *testing.T) { + for _, resource := range []string{"persistentvolumeclaims", "pods"} { + t.Run(resource, func(t *testing.T) { + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + createErr, statusErr := errors.New("create unavailable"), errors.New("status unavailable") + k.PrependReactor("create", resource, func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, createErr }) + d.PrependReactor("update", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, statusErr }) + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + if err := r.Reconcile(t.Context(), env.GetName()); !errors.Is(err, createErr) || !errors.Is(err, statusErr) { + t.Fatalf("lost provisioning/status error: %v", err) + } + }) + } +} + +func TestProvisioningRetriesNeverReplaceAuthoritativeResources(t *testing.T) { + for _, resource := range []string{"pvc", "pod"} { + t.Run(resource, func(t *testing.T) { + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + if err := unstructured.SetNestedField(env.Object, "missing-authority", "status", resource, "uid"); err != nil { + t.Fatal(err) + } + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + for range 3 { + if err := r.Reconcile(t.Context(), env.GetName()); err == nil { + t.Fatal("missing authoritative resource was not rejected") + } + } + for _, action := range k.Actions() { + if action.GetVerb() == "create" && (resource == "pvc" || action.GetResource().Resource == "pods") { + t.Fatalf("replaced missing authority: %v", action) + } + } + }) + } +} diff --git a/internal/adapter/executioncontroller/retained_delete_race_test.go b/internal/adapter/executioncontroller/retained_delete_race_test.go new file mode 100644 index 0000000000..3be3ce5011 --- /dev/null +++ b/internal/adapter/executioncontroller/retained_delete_race_test.go @@ -0,0 +1,215 @@ +package executioncontroller + +import ( + "errors" + "strconv" + "testing" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" +) + +func TestRetainedDeletePeerBetweenFinalizerUpdateAndDelete(t *testing.T) { + for _, mode := range []string{"peer-before-delete", "already-terminating", "stale-resource-version"} { + t.Run(mode, func(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + env.SetResourceVersion("1") + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + // The tracker does not implement API-server resource versions or CAS. + versionedUpdate := func(a ktesting.Action) (bool, runtime.Object, error) { + obj, err := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + if err != nil { + return true, nil, err + } + cur := obj.(*unstructured.Unstructured) + next := a.(ktesting.UpdateAction).GetObject().(*unstructured.Unstructured).DeepCopy() + version, err := strconv.Atoi(cur.GetResourceVersion()) + if err != nil || version < 1 || next.GetResourceVersion() == "" { + t.Fatal("fixture lost API-server resource version") + } + if next.GetResourceVersion() != cur.GetResourceVersion() { + return true, nil, apierrors.NewConflict(ExecutionEnvironmentGVR.GroupResource(), "env", errors.New("stale update")) + } + if !contains(cur.GetFinalizers(), environmentFinalizer) && contains(next.GetFinalizers(), environmentFinalizer) { + t.Fatal("peer re-added finalizer during admitted retained deletion") + } + next.SetResourceVersion(strconv.Itoa(version + 1)) + return true, next, d.Tracker().Update(ExecutionEnvironmentGVR, next, "ns") + } + d.PrependReactor("update", "executionenvironments", versionedUpdate) + k := kubefake.NewSimpleClientset(retainedPVC(), &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{profileAllocationKey("go"): `["env"]`}}) + store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) + if err := store.DeleteRetiredEnvironment(t.Context(), adminRequestFixture()); err != nil { + t.Fatal(err) + } + // Separate fake clients avoid the fake's per-client reactor lock, while both + // reconcilers observe the same API-server tracker. + peerClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + peerClient.PrependReactor("*", "*", ktesting.ObjectReaction(d.Tracker())) + peerClient.PrependReactor("update", "executionenvironments", versionedUpdate) + peer := NewReconciler(peerClient, k, "ns", testProfiles()) + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(peer.queue.ShutDown) + t.Cleanup(r.queue.ShutDown) + pvcsDeleted, envsDeleted := 0, 0 + k.PrependReactor("delete", "persistentvolumeclaims", func(a ktesting.Action) (bool, runtime.Object, error) { + opts := a.(ktesting.DeleteAction).GetDeleteOptions() + if opts.Preconditions == nil || opts.Preconditions.UID == nil || *opts.Preconditions.UID != "pvc-uid" { + t.Fatal("PVC deletion lost exact UID") + } + pvcsDeleted++ + return false, nil, nil + }) + staleDeletes := 0 + deletion := func(a ktesting.Action) (bool, runtime.Object, error) { + opts := a.(ktesting.DeleteAction).GetDeleteOptions() + if opts.Preconditions == nil || opts.Preconditions.ResourceVersion == nil || *opts.Preconditions.ResourceVersion == "" { + t.Fatal("CR deletion lost nonempty resource version") + } + obj, err := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + if err != nil { + return true, nil, err + } + cur := obj.(*unstructured.Unstructured) + if opts.Preconditions.UID == nil || *opts.Preconditions.UID != cur.GetUID() { + t.Fatal("CR deletion lost exact UID") + } + if *opts.Preconditions.ResourceVersion != cur.GetResourceVersion() { + staleDeletes++ + return true, nil, apierrors.NewConflict(ExecutionEnvironmentGVR.GroupResource(), "env", errors.New("stale delete")) + } + if contains(cur.GetFinalizers(), environmentFinalizer) { + t.Fatal("admitted retained deletion re-added finalizer") + } + envsDeleted++ + return true, nil, d.Tracker().Delete(ExecutionEnvironmentGVR, "ns", "env") + } + peerClient.PrependReactor("delete", "executionenvironments", deletion) + interleaved := false + d.PrependReactor("delete", "executionenvironments", func(a ktesting.Action) (bool, runtime.Object, error) { + if !interleaved { + interleaved = true + if mode == "stale-resource-version" { + res := peerClient.Resource(ExecutionEnvironmentGVR).Namespace("ns") + cur, err := res.Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + cur.SetAnnotations(map[string]string{"peer-observation": "updated"}) + if _, err := res.Update(t.Context(), cur, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + } else if err := peer.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + } + return deletion(a) + }) + if mode == "already-terminating" { + obj, _ := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + cur := obj.(*unstructured.Unstructured) + now := metav1.Now() + cur.SetDeletionTimestamp(&now) + if _, err := peerClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Update(t.Context(), cur, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + } + conflicts := 0 + for range 5 { + if err := r.Reconcile(t.Context(), "env"); err != nil { + if mode != "stale-resource-version" || !apierrors.IsConflict(err) { + t.Fatal(err) + } + conflicts++ + cur, getErr := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + if getErr != nil || contains(cur.(*unstructured.Unstructured).GetFinalizers(), environmentFinalizer) || envsDeleted != 0 { + t.Fatal("stale deletion must preserve CR without re-adding finalizer") + } + } + } + wantConflicts := 0 + if mode == "stale-resource-version" { + wantConflicts = 1 + } + if !interleaved || staleDeletes != wantConflicts || conflicts != wantConflicts { + t.Fatalf("interleaved=%v stale deletes=%d returned conflicts=%d", interleaved, staleDeletes, conflicts) + } + if _, err := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env"); !apierrors.IsNotFound(err) { + t.Fatalf("authorized deletion stranded: %v", err) + } + if pvcsDeleted != 1 || envsDeleted != 1 { + t.Fatalf("successful deletes PVC=%d CR=%d", pvcsDeleted, envsDeleted) + } + if len(profileSlots(t, k)) != 0 { + t.Fatal("capacity was not released") + } + }) + } +} + +func TestRetainedDeleteDoesNotRemoveForeignCR(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) + if err := store.DeleteRetiredEnvironment(t.Context(), adminRequestFixture()); err != nil { + t.Fatal(err) + } + env, _ = d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + _ = unstructured.SetNestedField(env.Object, "ReleasingSlot", "status", "lifecycleOperation", "phase") + foreign := env.DeepCopy() + foreign.SetUID("foreign") + if err := d.Tracker().Update(ExecutionEnvironmentGVR, foreign, "ns"); err != nil { + t.Fatal(err) + } + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + op, _, _ := unstructured.NestedMap(env.Object, "status", "lifecycleOperation") + if err := r.reconcileRetainedDelete(t.Context(), env, op, "workspace", "pvc-uid"); err == nil { + t.Fatal("foreign CR did not conflict") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil || got.GetUID() != "foreign" || !contains(got.GetFinalizers(), environmentFinalizer) { + t.Fatal("foreign CR touched") + } + for _, a := range k.Actions() { + if a.GetVerb() != "get" { + t.Fatal("foreign CR released capacity") + } + } +} + +func TestDirectCRDeleteStillBlocked(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + now := metav1.Now() + env.SetDeletionTimestamp(&now) + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(retainedPVC(), terminalExecutor()) + k.PrependReactor("delete", "*", func(ktesting.Action) (bool, runtime.Object, error) { + t.Error("direct delete destroyed runtime") + return true, nil, apierrors.NewForbidden(schema.GroupResource{}, "", errors.New("unexpected")) + }) + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil || !contains(got.GetFinalizers(), environmentFinalizer) { + t.Fatal("direct deletion lost finalizer") + } + if !conditionTrue(got, "DeletionBlocked") { + t.Fatal("direct deletion not blocked") + } +} diff --git a/internal/adapter/executioncontroller/security.go b/internal/adapter/executioncontroller/security.go new file mode 100644 index 0000000000..11021843de --- /dev/null +++ b/internal/adapter/executioncontroller/security.go @@ -0,0 +1,715 @@ +package executioncontroller + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "maps" + "math" + "net/url" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync/atomic" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + securityManifestVersion = 1 + maxSecurityKeys = 64 + maxSecurityClients = 256 + securityStateDataKey = "state.json" + keyStateRevoked = "revoked" +) + +type securityKeyManifest struct { + ID string `json:"id"` + Version uint64 `json:"version"` + File string `json:"file"` + PublicSHA256 string `json:"publicKeySHA256"` + ActivateAt time.Time `json:"activateAt"` + VerifyUntil time.Time `json:"verifyUntil"` + State string `json:"state"` +} + +type securityClientManifest struct { + URI string `json:"uri"` + MayAttestOwner bool `json:"mayAttestOwner"` + Administrator bool `json:"administrator"` + AdministratorFor []string `json:"administratorFor,omitempty"` +} + +type securityTLSManifest struct { + CertificateFile string `json:"certificateFile"` + PrivateKeyFile string `json:"privateKeyFile"` + ClientCAFile string `json:"clientCAFile"` +} + +type securityManifest struct { + Version int `json:"version"` + Generation uint64 `json:"generation"` + Issuer string `json:"issuer"` + Audience string `json:"audience"` + ActiveKeyID string `json:"activeKeyID"` + GrantTTL time.Duration `json:"-"` + GrantTTLText string `json:"grantTTL"` + ClockSkewText string `json:"clockSkew"` + Keys []securityKeyManifest `json:"keys"` + TLS securityTLSManifest `json:"tls"` + Clients []securityClientManifest `json:"clients"` +} + +type keyValidity struct { + activateAt, verifyUntil time.Time + state string +} + +type securitySnapshot struct { + generation uint64 + digest string + signer GrantSigner + verifier executionenv.GrantVerifier + keyValidity map[string]keyValidity + activeWindow keyValidity + tlsConfig *tls.Config + clientCAs *x509.CertPool + clients map[string]ClientPolicy + validUntil time.Time + fingerprints map[string]string +} + +type securityLedger struct { + Generation uint64 `json:"generation"` + Digest string `json:"digest"` + Fingerprints map[string]string `json:"fingerprints"` +} + +type securityState struct { + snapshot *securitySnapshot +} + +// SecurityManager reloads one immutable, versioned security snapshot from +// projected files and binds its generation to a durable ConfigMap high-water mark. +type SecurityManager struct { + manifestPath, keyDirectory, namespace, configMap string + kube kubernetes.Interface + now func() time.Time + state atomic.Pointer[securityState] +} + +// NewSecurityManager constructs a fail-closed security material reloader. +func NewSecurityManager(manifestPath, keyDirectory, namespace, configMap string, kube kubernetes.Interface) *SecurityManager { + return &SecurityManager{manifestPath: manifestPath, keyDirectory: keyDirectory, namespace: namespace, configMap: configMap, kube: kube, now: func() time.Time { return time.Now().UTC() }} +} + +// Ready reports whether the current snapshot is authoritative and unexpired. +func (m *SecurityManager) Ready() bool { + state := m.state.Load() + return state != nil && snapshotValidAt(state.snapshot, m.now()) +} + +// CheckReady verifies that the loaded snapshot still matches durable authority. +func (m *SecurityManager) CheckReady(ctx context.Context) bool { + _, err := m.authoritative(ctx) + return err == nil +} + +// Reload atomically validates and publishes a complete security snapshot. +func (m *SecurityManager) Reload(ctx context.Context) error { + observed := m.state.Load() + candidate, err := m.load() + if err != nil { + m.invalidateObservedUnlessReplaced(ctx, observed) + return err + } + if err := m.publishGeneration(ctx, candidate); err != nil { + m.invalidateObservedUnlessReplaced(ctx, observed) + return err + } + if err := m.verifyAuthority(ctx, candidate); err != nil { + m.invalidateObservedUnlessReplaced(ctx, observed) + return err + } + for { + current := m.state.Load() + if current != nil && current.snapshot.generation > candidate.generation { + return errors.New("security manifest generation rollback rejected") + } + if m.state.CompareAndSwap(current, &securityState{snapshot: candidate}) { + return nil + } + } +} + +// Run periodically reloads security material until ctx is cancelled. +func (m *SecurityManager) Run(ctx context.Context, interval time.Duration) { + if interval <= 0 { + interval = 2 * time.Second + } + if interval > time.Minute { + interval = time.Minute + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _ = m.Reload(ctx) + } + } +} + +// TLSConfig returns a dynamic TLS configuration backed by the current snapshot. +func (m *SecurityManager) TLSConfig() *tls.Config { + return &tls.Config{MinVersion: tls.VersionTLS13, ClientAuth: tls.RequireAnyClientCert, GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + state := m.state.Load() + if state == nil || !snapshotValidAt(state.snapshot, m.now()) { + return nil, errors.New("security material is not ready") + } + return state.snapshot.tlsConfig.Clone(), nil + }} +} + +func (m *SecurityManager) authorize(ctx context.Context, chain []*x509.Certificate) (string, ClientPolicy, error) { + if len(chain) == 0 || chain[0] == nil { + return "", ClientPolicy{}, errors.New("client certificate is unavailable") + } + now := m.now() + s, err := m.authoritativeAt(ctx, now) + if err != nil { + return "", ClientPolicy{}, err + } + intermediates := x509.NewCertPool() + for _, cert := range chain[1:] { + if cert != nil { + intermediates.AddCert(cert) + } + } + opts := x509.VerifyOptions{Roots: s.clientCAs, Intermediates: intermediates, CurrentTime: now, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + leaf := chain[0] + if _, err := leaf.Verify(opts); err != nil { + return "", ClientPolicy{}, err + } + id, err := canonicalClientIdentity(leaf) + if err != nil { + return "", ClientPolicy{}, err + } + policy, ok := s.clients[id] + if !ok { + return "", ClientPolicy{}, errors.New("client is not authorized") + } + return id, policy, nil +} + +func (m *SecurityManager) material(ctx context.Context) (*securitySnapshot, error) { + return m.authoritative(ctx) +} + +func (m *SecurityManager) authoritative(ctx context.Context) (*securitySnapshot, error) { + return m.authoritativeAt(ctx, m.now()) +} + +var errAuthorityUnavailable = errors.New("security authority is unavailable") + +func authorityUnavailable(err error) error { + if err == nil { + return errAuthorityUnavailable + } + return fmt.Errorf("%w: %v", errAuthorityUnavailable, err) +} + +func (m *SecurityManager) authoritativeAt(ctx context.Context, now time.Time) (*securitySnapshot, error) { + state := m.state.Load() + if state == nil || !snapshotValidAt(state.snapshot, now) || m.kube == nil { + return nil, authorityUnavailable(nil) + } + if err := m.verifyAuthority(ctx, state.snapshot); err != nil { + m.state.CompareAndSwap(state, nil) + return nil, authorityUnavailable(err) + } + return state.snapshot, nil +} + +func (m *SecurityManager) verifyAuthority(ctx context.Context, snapshot *securitySnapshot) error { + if snapshot == nil || m.kube == nil { + return errors.New("security material is not ready") + } + cm, err := m.kube.CoreV1().ConfigMaps(m.namespace).Get(ctx, m.configMap, metav1.GetOptions{}) + if err != nil { + return err + } + var ledger securityLedger + if err := executionenv.DecodeStrict([]byte(cm.Data[securityStateDataKey]), &ledger); err != nil { + return err + } + if ledger.Generation != snapshot.generation || ledger.Digest != snapshot.digest || !maps.Equal(ledger.Fingerprints, snapshot.fingerprints) { + return errors.New("loaded security generation is not authoritative") + } + return nil +} + +func (m *SecurityManager) invalidateObservedUnlessReplaced(ctx context.Context, observed *securityState) { + if m.state.CompareAndSwap(observed, nil) { + return + } + current := m.state.Load() + if current != nil && m.verifyAuthority(ctx, current.snapshot) != nil { + m.state.CompareAndSwap(current, nil) + } +} + +func snapshotValidAt(s *securitySnapshot, now time.Time) bool { + return s != nil && !now.Before(s.activeWindow.activateAt) && now.Before(s.activeWindow.verifyUntil) && now.Before(s.validUntil) +} + +func (s *securitySnapshot) verifierAt(now time.Time) executionenv.GrantVerifier { + v := s.verifier + v.Keys = make(map[string]ed25519.PublicKey, len(s.verifier.Keys)) + for id, key := range s.verifier.Keys { + window := s.keyValidity[id] + if window.state != keyStateRevoked && !now.Before(window.activateAt) && now.Before(window.verifyUntil) { + v.Keys[id] = key + } + } + return v +} + +func (m *SecurityManager) load() (*securitySnapshot, error) { + mf, ttl, skew, err := m.loadManifest() + if err != nil { + return nil, err + } + root, err := os.OpenRoot(m.keyDirectory) + if err != nil { + return nil, fmt.Errorf("open security directory: %w", err) + } + defer func() { _ = root.Close() }() + + keys, active, fingerprints, windows, activeWindow, err := m.loadGrantKeys(root, mf) + if err != nil { + return nil, err + } + tlsConfig, clientCAs, tlsValidUntil, caFingerprints, serverFingerprint, err := m.loadTLS(root, mf.TLS) + if err != nil { + return nil, err + } + clients, err := clientPolicies(mf.Clients) + if err != nil { + return nil, err + } + digest, err := authorityDigest(mf, ttl, skew, fingerprints, caFingerprints, serverFingerprint) + if err != nil { + return nil, err + } + validUntil := tlsValidUntil + now := m.now() + for _, window := range windows { + if window.state != keyStateRevoked && window.verifyUntil.Before(validUntil) { + validUntil = window.verifyUntil + } + if window.state != keyStateRevoked && now.Before(window.activateAt) && window.activateAt.Before(validUntil) { + validUntil = window.activateAt + } + } + return &securitySnapshot{ + generation: mf.Generation, digest: digest, + signer: GrantSigner{ + KeyID: mf.ActiveKeyID, PrivateKey: active, Issuer: mf.Issuer, + Audience: mf.Audience, Lifetime: ttl, + }, + verifier: executionenv.GrantVerifier{ + Keys: keys, Issuer: mf.Issuer, Audience: mf.Audience, MaxLifetime: ttl + skew, + }, + keyValidity: windows, activeWindow: activeWindow, + tlsConfig: tlsConfig, clientCAs: clientCAs, clients: clients, + validUntil: validUntil, fingerprints: fingerprints, + }, nil +} + +func (m *SecurityManager) loadManifest() (securityManifest, time.Duration, time.Duration, error) { + manifestBytes, err := os.ReadFile(m.manifestPath) + if err != nil { + return securityManifest{}, 0, 0, fmt.Errorf("read security manifest: %w", err) + } + var mf securityManifest + if err := executionenv.DecodeStrict(manifestBytes, &mf); err != nil { + return securityManifest{}, 0, 0, fmt.Errorf("decode security manifest: %w", err) + } + if mf.Version != securityManifestVersion || mf.Generation == 0 || mf.Generation > math.MaxInt64 || mf.Issuer == "" || mf.Audience == "" || mf.ActiveKeyID == "" || len(mf.Keys) == 0 || len(mf.Keys) > maxSecurityKeys || len(mf.Clients) == 0 || len(mf.Clients) > maxSecurityClients { + return securityManifest{}, 0, 0, errors.New("security manifest identity or bounds are invalid") + } + ttl, err := time.ParseDuration(mf.GrantTTLText) + if err != nil || ttl <= 0 || ttl > 5*time.Minute { + return securityManifest{}, 0, 0, errors.New("grantTTL must be positive and at most 5m") + } + skew, err := time.ParseDuration(mf.ClockSkewText) + if err != nil || skew < 0 || skew >= ttl { + return securityManifest{}, 0, 0, errors.New("clockSkew must be non-negative and less than grantTTL") + } + return mf, ttl, skew, nil +} + +func (m *SecurityManager) loadGrantKeys(root *os.Root, mf securityManifest) (map[string]ed25519.PublicKey, ed25519.PrivateKey, map[string]string, map[string]keyValidity, keyValidity, error) { + keys := make(map[string]ed25519.PublicKey, len(mf.Keys)) + fingerprints := make(map[string]string, len(mf.Keys)) + windows := make(map[string]keyValidity, len(mf.Keys)) + seenVersion := map[uint64]bool{} + seenID := map[string]bool{} + var active ed25519.PrivateKey + var activeWindow keyValidity + now := m.now() + for _, km := range mf.Keys { + if !validKeyManifest(km, seenID, seenVersion) { + return nil, nil, nil, nil, keyValidity{}, errors.New("security key entry is invalid or duplicated") + } + seenID[km.ID] = true + seenVersion[km.Version] = true + key, err := readEd25519(root, km.File) + if err != nil { + return nil, nil, nil, nil, keyValidity{}, fmt.Errorf("load security key %q: %w", km.ID, err) + } + pub := key.Public().(ed25519.PublicKey) + sum := sha256.Sum256(pub) + fp := hex.EncodeToString(sum[:]) + if !strings.EqualFold(fp, km.PublicSHA256) { + return nil, nil, nil, nil, keyValidity{}, fmt.Errorf("security key %q fingerprint mismatch", km.ID) + } + fingerprints[km.ID+":"+strconv.FormatUint(km.Version, 10)] = fp + window := keyValidity{activateAt: km.ActivateAt, verifyUntil: km.VerifyUntil, state: km.State} + windows[km.ID] = window + if km.State != keyStateRevoked { + keys[km.ID] = pub + } + if km.ID == mf.ActiveKeyID { + if km.State != "active" || now.Before(km.ActivateAt) || !now.Before(km.VerifyUntil) { + return nil, nil, nil, nil, keyValidity{}, errors.New("active grant key is outside its activation window") + } + active = key + activeWindow = window + } + } + if active == nil { + return nil, nil, nil, nil, keyValidity{}, errors.New("active grant key is missing") + } + return keys, active, fingerprints, windows, activeWindow, nil +} + +func validKeyManifest(km securityKeyManifest, seenID map[string]bool, seenVersion map[uint64]bool) bool { + return validKeyID(km.ID) && !seenID[km.ID] && km.Version != 0 && !seenVersion[km.Version] && validBaseName(km.File) && + (km.State == "active" || km.State == "verify-only" || km.State == keyStateRevoked) && + !km.VerifyUntil.IsZero() && km.VerifyUntil.After(km.ActivateAt) +} + +func (m *SecurityManager) loadTLS(root *os.Root, manifest securityTLSManifest) (*tls.Config, *x509.CertPool, time.Time, []string, string, error) { + certPEM, err := readRootFile(root, manifest.CertificateFile) + if err != nil { + return nil, nil, time.Time{}, nil, "", err + } + keyPEM, err := readRootFile(root, manifest.PrivateKeyFile) + if err != nil { + return nil, nil, time.Time{}, nil, "", err + } + serverCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, nil, time.Time{}, nil, "", fmt.Errorf("load TLS identity: %w", err) + } + leaf, err := x509.ParseCertificate(serverCert.Certificate[0]) + if err != nil { + return nil, nil, time.Time{}, nil, "", err + } + now := m.now() + if now.Before(leaf.NotBefore) || !now.Before(leaf.NotAfter) || !hasUsage(leaf.ExtKeyUsage, x509.ExtKeyUsageServerAuth) { + return nil, nil, time.Time{}, nil, "", errors.New("server certificate is not currently valid for server authentication") + } + serverCert.Leaf = leaf + caPEM, err := readRootFile(root, manifest.ClientCAFile) + if err != nil { + return nil, nil, time.Time{}, nil, "", err + } + pool := x509.NewCertPool() + caFingerprints, err := appendCAs(pool, caPEM) + if err != nil { + return nil, nil, time.Time{}, nil, "", err + } + serverSum := sha256.Sum256(leaf.Raw) + cfg := &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{serverCert}, ClientAuth: tls.RequireAnyClientCert} + return cfg, pool, leaf.NotAfter, caFingerprints, hex.EncodeToString(serverSum[:]), nil +} + +func appendCAs(pool *x509.CertPool, pemBytes []byte) ([]string, error) { + var fingerprints []string + for len(bytes.TrimSpace(pemBytes)) != 0 { + block, rest := pem.Decode(pemBytes) + if block == nil || block.Type != "CERTIFICATE" { + return nil, errors.New("client CA bundle is invalid") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil || !cert.IsCA { + return nil, errors.New("client CA bundle contains an invalid CA certificate") + } + pool.AddCert(cert) + sum := sha256.Sum256(cert.Raw) + fingerprints = append(fingerprints, hex.EncodeToString(sum[:])) + pemBytes = rest + } + if len(fingerprints) == 0 { + return nil, errors.New("client CA bundle has no certificates") + } + slices.Sort(fingerprints) + return fingerprints, nil +} + +func clientPolicies(entries []securityClientManifest) (map[string]ClientPolicy, error) { + clients := make(map[string]ClientPolicy, len(entries)) + for _, entry := range entries { + uri, err := url.Parse(entry.URI) + if err != nil { + return nil, errors.New("client URI is invalid") + } + canonical, err := canonicalClientIdentity(&x509.Certificate{URIs: []*url.URL{uri}}) + if err != nil || canonical != entry.URI { + return nil, errors.New("client URI is not canonical") + } + if _, exists := clients[entry.URI]; exists { + return nil, errors.New("client URI is duplicated") + } + if len(entry.AdministratorFor) > maxSecurityClients || len(entry.AdministratorFor) > 0 && !entry.Administrator { + return nil, errors.New("administrator scope requires administrator authority and at most 256 creators") + } + scope := slices.Clone(entry.AdministratorFor) + slices.Sort(scope) + for i, creator := range scope { + uri, err := url.Parse(creator) + if err != nil || strings.ContainsAny(creator, "*?") || strings.ContainsAny(uri.Path, "*?") { + return nil, errors.New("administrator creator URI is invalid") + } + canonical, err := canonicalClientIdentity(&x509.Certificate{URIs: []*url.URL{uri}}) + if err != nil || canonical != creator || i > 0 && scope[i-1] == creator { + return nil, errors.New("administrator creator URI is not canonical or is duplicated") + } + } + clients[entry.URI] = ClientPolicy{MayAttestOwner: entry.MayAttestOwner, Administrator: entry.Administrator, AdministratorFor: scope} + } + return clients, nil +} + +func authorityDigest(mf securityManifest, ttl, skew time.Duration, keyFingerprints map[string]string, caFingerprints []string, serverFingerprint string) (string, error) { + keys := slices.Clone(mf.Keys) + slices.SortFunc(keys, func(a, b securityKeyManifest) int { return strings.Compare(a.ID, b.ID) }) + clients := slices.Clone(mf.Clients) + for i := range clients { + clients[i].AdministratorFor = slices.Clone(clients[i].AdministratorFor) + slices.Sort(clients[i].AdministratorFor) + } + slices.SortFunc(clients, func(a, b securityClientManifest) int { return strings.Compare(a.URI, b.URI) }) + canonical := struct { + Version int + Generation uint64 + Issuer, Audience, ActiveKeyID string + GrantTTL, ClockSkew int64 + Keys []securityKeyManifest + TLS securityTLSManifest + Clients []securityClientManifest + KeyFingerprints map[string]string + CAFingerprints []string + ServerFingerprint string + }{ + Version: mf.Version, Generation: mf.Generation, Issuer: mf.Issuer, Audience: mf.Audience, ActiveKeyID: mf.ActiveKeyID, + GrantTTL: int64(ttl), ClockSkew: int64(skew), Keys: keys, TLS: mf.TLS, Clients: clients, + KeyFingerprints: keyFingerprints, CAFingerprints: slices.Clone(caFingerprints), ServerFingerprint: serverFingerprint, + } + raw, err := json.Marshal(canonical) + if err != nil { + return "", err + } + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]), nil +} + +func (m *SecurityManager) publishGeneration(ctx context.Context, s *securitySnapshot) error { + if m.kube == nil || m.configMap == "" { + return errors.New("security authority ConfigMap is required") + } + cms := m.kube.CoreV1().ConfigMaps(m.namespace) + for range 5 { + cm, exists, ledger, err := m.readSecurityLedger(ctx, cms) + if err != nil { + return err + } + changed, err := advanceSecurityLedger(&ledger, s) + if err != nil || !changed { + return err + } + raw, err := json.Marshal(ledger) + if err != nil { + return err + } + if cm.Data == nil { + cm.Data = map[string]string{} + } + cm.Data[securityStateDataKey] = string(raw) + if !exists { + _, err = cms.Create(ctx, cm, metav1.CreateOptions{}) + } else { + _, err = cms.Update(ctx, cm, metav1.UpdateOptions{}) + } + if err == nil { + return nil + } + if !apierrors.IsAlreadyExists(err) && !apierrors.IsConflict(err) { + return err + } + } + return errors.New("security authority state changed concurrently") +} + +func (m *SecurityManager) readSecurityLedger(ctx context.Context, cms corev1client.ConfigMapInterface) (*corev1.ConfigMap, bool, securityLedger, error) { + cm, err := cms.Get(ctx, m.configMap, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + cm = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: m.configMap, Namespace: m.namespace}, Data: map[string]string{}} + return cm, false, securityLedger{Fingerprints: map[string]string{}}, nil + } + if err != nil { + return nil, false, securityLedger{}, err + } + ledger := securityLedger{Fingerprints: map[string]string{}} + if raw := cm.Data[securityStateDataKey]; raw != "" { + if err := executionenv.DecodeStrict([]byte(raw), &ledger); err != nil || ledger.Generation == 0 || ledger.Digest == "" || ledger.Fingerprints == nil || len(ledger.Fingerprints) > maxSecurityKeys { + return nil, false, securityLedger{}, errors.New("security authority state is invalid") + } + } + return cm, true, ledger, nil +} + +func advanceSecurityLedger(ledger *securityLedger, s *securitySnapshot) (bool, error) { + if s.generation < ledger.Generation { + return false, errors.New("security manifest generation rollback rejected") + } + if s.generation == ledger.Generation && ledger.Generation != 0 { + if ledger.Digest != s.digest || !containsFingerprints(ledger.Fingerprints, s.fingerprints) { + return false, errors.New("security generation authority drift rejected") + } + s.fingerprints = maps.Clone(ledger.Fingerprints) + return false, nil + } + for identity, fp := range s.fingerprints { + old, exists := ledger.Fingerprints[identity] + if exists && old != fp { + return false, errors.New("security key identity reuse rejected") + } + if !exists && !keyVersionAdvances(ledger.Fingerprints, identity) { + return false, errors.New("security key version rollback rejected") + } + ledger.Fingerprints[identity] = fp + } + if len(ledger.Fingerprints) > maxSecurityKeys { + return false, errors.New("security key tombstone limit reached; rotate issuer") + } + ledger.Generation = s.generation + ledger.Digest = s.digest + s.fingerprints = maps.Clone(ledger.Fingerprints) + return true, nil +} + +func readRootFile(root *os.Root, name string) ([]byte, error) { + if !validBaseName(name) { + return nil, errors.New("security filename must be a basename") + } + f, err := root.Open(name) + if err != nil { + return nil, err + } + b, readErr := io.ReadAll(io.LimitReader(f, 1<<20)) + return b, errors.Join(readErr, f.Close()) +} +func readEd25519(root *os.Root, name string) (ed25519.PrivateKey, error) { + b, err := readRootFile(root, name) + if err != nil { + return nil, err + } + block, rest := pem.Decode(b) + if block == nil || block.Type != "PRIVATE KEY" || len(bytes.TrimSpace(rest)) != 0 { + return nil, errors.New("key must be one PKCS8 PEM block") + } + raw, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, errors.New("invalid PKCS8 key") + } + key, ok := raw.(ed25519.PrivateKey) + if !ok { + return nil, errors.New("key is not Ed25519") + } + return key, nil +} +func validBaseName(v string) bool { return v != "" && filepath.Base(v) == v && v != "." && v != ".." } +func validKeyID(v string) bool { + if len(v) == 0 || len(v) > 64 { + return false + } + for _, r := range v { + if r != '-' && r != '_' && (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r < '0' || r > '9') { + return false + } + } + return true +} +func hasUsage(usages []x509.ExtKeyUsage, wanted x509.ExtKeyUsage) bool { + for _, u := range usages { + if u == wanted || u == x509.ExtKeyUsageAny { + return true + } + } + return false +} +func containsFingerprints(authority, candidate map[string]string) bool { + for identity, fingerprint := range candidate { + if authority[identity] != fingerprint { + return false + } + } + return true +} + +func keyVersionAdvances(authority map[string]string, identity string) bool { + separator := strings.LastIndexByte(identity, ':') + if separator <= 0 { + return false + } + id := identity[:separator] + version, err := strconv.ParseUint(identity[separator+1:], 10, 64) + if err != nil { + return false + } + for previous := range authority { + previousSeparator := strings.LastIndexByte(previous, ':') + if previousSeparator <= 0 || previous[:previousSeparator] != id { + continue + } + previousVersion, err := strconv.ParseUint(previous[previousSeparator+1:], 10, 64) + if err != nil || version <= previousVersion { + return false + } + } + return true +} diff --git a/internal/adapter/executioncontroller/security_limit.go b/internal/adapter/executioncontroller/security_limit.go new file mode 100644 index 0000000000..d172248c50 --- /dev/null +++ b/internal/adapter/executioncontroller/security_limit.go @@ -0,0 +1,83 @@ +package executioncontroller + +import ( + "context" + "crypto/x509" + "errors" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +// RPCLimiter bounds active unary RPCs globally and per authorized client. +type RPCLimiter struct { + security *SecurityManager + globalMax int + clientMax int + mu sync.Mutex + global int + clients map[string]int +} + +// NewRPCLimiter constructs a limiter whose client map is bounded by the security allowlist. +func NewRPCLimiter(security *SecurityManager, globalMax, clientMax int) (*RPCLimiter, error) { + if security == nil || globalMax < 1 || clientMax < 1 || clientMax > globalMax { + return nil, status.Error(codes.InvalidArgument, "invalid RPC concurrency limits") + } + return &RPCLimiter{security: security, globalMax: globalMax, clientMax: clientMax, clients: map[string]int{}}, nil +} + +// UnaryInterceptor authenticates against current policy before reserving capacity. +func (l *RPCLimiter) UnaryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + chain, _, err := clientCertificates(ctx) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "execution provider request failed") + } + id, _, err := l.security.authorize(ctx, chain) + if err != nil { + return nil, securityAuthorizationError(err) + } + if !l.acquire(id) { + return nil, status.Error(codes.ResourceExhausted, "execution provider is at its concurrency limit") + } + defer l.release(id) + return handler(ctx, req) +} + +func clientCertificates(ctx context.Context) ([]*x509.Certificate, bool, error) { + p, ok := peer.FromContext(ctx) + if !ok { + return nil, false, errors.New("peer is unavailable") + } + tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) + if !ok || len(tlsInfo.State.PeerCertificates) == 0 { + return nil, false, errors.New("client certificate is unavailable") + } + return tlsInfo.State.PeerCertificates, len(tlsInfo.State.VerifiedChains) != 0, nil +} + +func (l *RPCLimiter) acquire(id string) bool { + l.mu.Lock() + defer l.mu.Unlock() + if l.global >= l.globalMax || l.clients[id] >= l.clientMax { + return false + } + l.global++ + l.clients[id]++ + return true +} + +func (l *RPCLimiter) release(id string) { + l.mu.Lock() + defer l.mu.Unlock() + l.global-- + if l.clients[id] == 1 { + delete(l.clients, id) + } else { + l.clients[id]-- + } +} diff --git a/internal/adapter/executioncontroller/security_limit_test.go b/internal/adapter/executioncontroller/security_limit_test.go new file mode 100644 index 0000000000..4600fce13c --- /dev/null +++ b/internal/adapter/executioncontroller/security_limit_test.go @@ -0,0 +1,29 @@ +package executioncontroller + +import "testing" + +func TestRPCLimiterBoundsGlobalPerClientAndIdentityMap(t *testing.T) { + limiter, err := NewRPCLimiter(&SecurityManager{}, 2, 1) + if err != nil { + t.Fatal(err) + } + if !limiter.acquire("client-a") || limiter.acquire("client-a") { + t.Fatal("per-client bound was not enforced") + } + if !limiter.acquire("client-b") || limiter.acquire("client-c") { + t.Fatal("global bound was not enforced") + } + limiter.release("client-a") + limiter.release("client-b") + if len(limiter.clients) != 0 || limiter.global != 0 { + t.Fatalf("released limiter retained cardinality: global=%d clients=%v", limiter.global, limiter.clients) + } +} + +func TestRPCLimiterRejectsInvalidBounds(t *testing.T) { + for _, limits := range [][2]int{{0, 1}, {1, 0}, {1, 2}} { + if _, err := NewRPCLimiter(&SecurityManager{}, limits[0], limits[1]); err == nil { + t.Fatalf("accepted limits %v", limits) + } + } +} diff --git a/internal/adapter/executioncontroller/security_rotation_test.go b/internal/adapter/executioncontroller/security_rotation_test.go new file mode 100644 index 0000000000..8595844b4c --- /dev/null +++ b/internal/adapter/executioncontroller/security_rotation_test.go @@ -0,0 +1,176 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "os" + "path/filepath" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +// These are synthetic fixture names, including the former mutable Secret keys. +func rotationSecurityFixture(t *testing.T) (*SecurityManager, securityManifest, context.Context, map[string][]byte) { + t.Helper() + dir := t.TempDir() + now := time.Now().UTC() + oldCA, oldCert, oldKey := syntheticPKI(t, now) + newCA, newCert, newKey, client := rpcSecurityPKI(t, now, "spiffe://example/client") + writeRotationMaterial(t, dir, map[string][]byte{"tls.crt": oldCert, "tls.key": oldKey, "clients.pem": append(oldCA, newCA...)}) + manifest := securityManifest{Version: 1, Generation: 2, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k2", GrantTTLText: "1m", ClockSkewText: "5s", TLS: securityTLSManifest{CertificateFile: "tls.crt", PrivateKeyFile: "tls.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + for i, id := range []string{"k1", "k2"} { + pub, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + name := "grant-" + id + ".pem" + writePKCS8(t, filepath.Join(dir, name), private) + fp := sha256.Sum256(pub) + manifest.Keys = append(manifest.Keys, securityKeyManifest{ID: id, Version: uint64(i + 1), File: name, PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}) + } + path := filepath.Join(dir, "manifest.json") + writeManifest(t, path, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(path, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + leaf, err := x509.ParseCertificate(client.Certificate[0]) + if err != nil { + t.Fatal(err) + } + ctx := peer.NewContext(t.Context(), &peer.Peer{AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}, VerifiedChains: [][]*x509.Certificate{{leaf}}}}}) + return manager, manifest, ctx, map[string][]byte{"tls.crt": newCert, "tls.key": newKey, "clients.pem": newCA} +} + +func writeRotationMaterial(t *testing.T, dir string, files map[string][]byte) { + t.Helper() + for name, contents := range files { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } +} + +func TestSecurityRotationPinnedReplicaReadConvergesBeforeDispatch(t *testing.T) { + manager, manifest, ctx, final := rotationSecurityFixture(t) + backend := &countedFileBackend{fakeBackend: newFakeBackend()} + h := NewHandler(HandlerConfig{Security: manager}, backend) + owner := &executionv1.Owner{Issuer: "issuer", Subject: "alice"} + claim, err := h.AcquireRun(ctx, &executionv1.AcquireRunRequest{Environment: &executionv1.EnvironmentRef{Id: "env", Revision: "rev"}, Owner: owner, BindingId: "binding", RunId: "run", OperationId: "acquire", TtlMillis: time.Minute.Milliseconds()}) + if err != nil { + t.Fatal("bridge k2 acquire failed") + } + request := &executionv1.FileRequest{Context: &executionv1.RequestContext{Environment: claim.Environment, Owner: owner, BindingId: claim.BindingId, RunId: claim.RunId, ClaimId: claim.ClaimId, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant}, Operation: executionv1.FileOperation_FILE_OPERATION_READ, Path: "sentinel"} + if _, err := h.Files(ctx, request); err != nil || backend.calls != 1 { + t.Fatal("bridge claim positive control failed") + } + manifest.Generation = 3 + manifest.Keys[0].State = "revoked" + manifest.TLS = securityTLSManifest{CertificateFile: "provider-new.crt", PrivateKeyFile: "provider-new.key", ClientCAFile: "final-clients.pem"} + writeRotationMaterial(t, manager.keyDirectory, map[string][]byte{"provider-new.crt": final["tls.crt"], "provider-new.key": final["tls.key"], "final-clients.pem": final["clients.pem"]}) + writeManifest(t, manager.manifestPath, manifest) + other := NewSecurityManager(manager.manifestPath, manager.keyDirectory, "ns", "authority", manager.kube) + other.now = manager.now + if err := other.Reload(t.Context()); err != nil { + t.Fatal(err) + } + backend.calls = 0 + _, err = h.Files(ctx, request) + st := status.Convert(err) + details := st.Details() + if st.Code() != codes.Unavailable || len(details) != 1 || backend.calls != 0 { + t.Fatalf("lagging replica: code=%s dispatches=%d", st.Code(), backend.calls) + } + detail, ok := details[0].(*executionv1.ErrorDetail) + if !ok || detail.Code != string(executionenv.CodeNotReady) || !detail.Retryable { + t.Fatal("lag was not structured retryable not_ready") + } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := h.Files(ctx, request); err != nil || backend.calls != 1 { + t.Fatalf("same current claim after reload: code=%s dispatches=%d", status.Code(err), backend.calls) + } +} + +func TestSecurityRotationMutableNamesPoisonSameGeneration(t *testing.T) { + manager, manifest, _, final := rotationSecurityFixture(t) + manifest.Generation = 3 + manifest.Keys[0].State = "revoked" + // ConfigMap projection arrives before the Secret: all old names still exist. + writeManifest(t, manager.manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatalf("mixed bundle did not reproduce publication: %v", err) + } + mixed := manager.state.Load().snapshot.digest + writeRotationMaterial(t, manager.keyDirectory, final) + complete, err := manager.load() + if err != nil || complete.digest == mixed { + t.Fatal("complete bundle did not differ from published mixed bundle") + } + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("same-generation material drift was not rejected") + } + if err := manager.Reload(t.Context()); err == nil { + t.Fatal("retry unexpectedly repaired the poisoned ledger") + } + // Recovery requires a forward generation, not a retry or ledger reset. + manifest.Generation++ + writeManifest(t, manager.manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } +} + +func TestSecurityRotationImmutableNamesHandleProjectionSkew(t *testing.T) { + for _, manifestFirst := range []bool{false, true} { + name := "material-first" + if manifestFirst { + name = "manifest-first" + } + t.Run(name, func(t *testing.T) { + manager, manifest, _, final := rotationSecurityFixture(t) + before := manager.state.Load().snapshot.digest + manifest.Generation = 3 + manifest.Keys[0].State = "revoked" + manifest.TLS = securityTLSManifest{CertificateFile: "provider-new.crt", PrivateKeyFile: "provider-new.key", ClientCAFile: "final-clients.pem"} + if manifestFirst { + writeManifest(t, manager.manifestPath, manifest) + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("missing immutable material did not fail closed") + } + } + writeRotationMaterial(t, manager.keyDirectory, map[string][]byte{"provider-new.crt": final["tls.crt"], "provider-new.key": final["tls.key"], "final-clients.pem": final["clients.pem"]}) + if !manifestFirst { + if err := manager.Reload(t.Context()); err != nil || manager.state.Load().snapshot.digest != before { + t.Fatal("staging new names changed the old manifest's authority") + } + writeManifest(t, manager.manifestPath, manifest) + } + if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { + t.Fatalf("complete immutable bundle did not converge: %v", err) + } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal("same-generation immutable reload drifted") + } + }) + } +} diff --git a/internal/adapter/executioncontroller/security_test.go b/internal/adapter/executioncontroller/security_test.go new file mode 100644 index 0000000000..9de09f5009 --- /dev/null +++ b/internal/adapter/executioncontroller/security_test.go @@ -0,0 +1,743 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "math/big" + "net" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + grpcpeer "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type durableClaimBackend struct { + *fakeBackend + mu sync.Mutex + acquired, renewed map[string]executionenv.RunClaim + acquireCalls, renewalCalls int +} + +func newDurableClaimBackend() *durableClaimBackend { + return &durableClaimBackend{fakeBackend: newFakeBackend(), acquired: make(map[string]executionenv.RunClaim), renewed: make(map[string]executionenv.RunClaim)} +} + +func (b *durableClaimBackend) AcquireRun(_ context.Context, ref executionenv.EnvironmentRef, _ string, _ string, binding, run, operation string, ttl time.Duration) (executionenv.RunClaim, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.acquireCalls++ + if claim, ok := b.acquired[operation]; ok { + return claim, nil + } + claim := executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: run, ClaimID: "claim-acquire", Epoch: 2, GrantGeneration: 1, ExpiresAt: time.Now().Add(ttl)} + b.acquired[operation] = claim + return claim, nil +} + +func (b *durableClaimBackend) RenewRun(_ context.Context, _ executionenv.EnvironmentRef, _ string, _ string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.renewalCalls++ + if claim, ok := b.renewed[req.OperationID]; ok { + return claim, nil + } + claim := executionenv.RunClaim{Environment: req.Environment, BindingID: req.BindingID, RunID: req.RunID, ClaimID: req.ClaimID, Epoch: req.Epoch, GrantGeneration: req.GrantGeneration + 1, ExpiresAt: time.Now().Add(req.TTL)} + b.renewed[req.OperationID] = claim + return claim, nil +} + +func (b *durableClaimBackend) claim(operation string, renew bool) (executionenv.RunClaim, int, bool) { + b.mu.Lock() + defer b.mu.Unlock() + if renew { + claim, ok := b.renewed[operation] + return claim, b.renewalCalls, ok + } + claim, ok := b.acquired[operation] + return claim, b.acquireCalls, ok +} + +func TestSecurityManagerReloadRollbackAndRecovery(t *testing.T) { + dir := t.TempDir() + data := filepath.Join(dir, "..data") + if err := os.Mkdir(data, 0o700); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(data, "grant.pem"), grantPriv) + caPEM, serverCert, serverKey := syntheticPKI(t, now) + if err := os.WriteFile(filepath.Join(data, "server.crt"), serverCert, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(data, "server.key"), serverKey, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(data, "clients.pem"), caPEM, 0o600); err != nil { + t.Fatal(err) + } + for _, name := range []string{"grant.pem", "server.crt", "server.key", "clients.pem"} { + if err := os.Symlink(filepath.Join("..data", name), filepath.Join(dir, name)); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true, Administrator: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if !manager.Ready() { + t.Fatal("valid snapshot is not ready") + } + manifest.Clients[0].Administrator = false + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("same-generation client policy drift was accepted") + } + manifest.Clients[0].Administrator = true + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatalf("restore same generation: %v", err) + } + manifest.Generation = 0 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("invalid candidate retained authorization readiness") + } + manifest.Generation = 2 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { + t.Fatalf("recovery reload: ready=%v err=%v", manager.Ready(), err) + } + firstCertificate := append([]byte(nil), manager.state.Load().snapshot.tlsConfig.Certificates[0].Certificate[0]...) + cm, err := kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + originalState := cm.Data[securityStateDataKey] + cm.Data[securityStateDataKey] = `{"generation":2,"fingerprints":{"k1:1":"forged"}}` + if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := manager.material(t.Context()); err == nil || manager.Ready() { + t.Fatal("authoritative fingerprint drift retained authorization") + } + cm, err = kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + cm.Data[securityStateDataKey] = originalState + if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := manager.Reload(t.Context()); err != nil { + t.Fatalf("restore authority: %v", err) + } + newCAPEM, newServerCert, newServerKey := syntheticPKI(t, now) + for name, contents := range map[string][]byte{"server.crt": newServerCert, "server.key": newServerKey, "clients.pem": newCAPEM} { + if err := os.WriteFile(filepath.Join(data, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + manifest.Generation = 3 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { + t.Fatalf("TLS/CA correction did not recover: ready=%v err=%v", manager.Ready(), err) + } + if string(firstCertificate) == string(manager.state.Load().snapshot.tlsConfig.Certificates[0].Certificate[0]) { + t.Fatal("TLS identity did not rotate atomically") + } + manifest.Generation = 4 + writeManifest(t, manifestPath, manifest) + peer := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + peer.now = manager.now + if err := peer.Reload(t.Context()); err != nil { + t.Fatalf("peer authority advance: %v", err) + } + if manager.CheckReady(t.Context()) || manager.Ready() { + t.Fatal("readiness retained a snapshot superseded by a peer") + } + manifest.Generation = 1 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("generation rollback accepted") + } +} + +func TestSecurityManagerStaleAuthorityFailureCannotPoisonNewerState(t *testing.T) { + now := time.Now().UTC() + window := keyValidity{activateAt: now.Add(-time.Minute), verifyUntil: now.Add(time.Hour)} + oldSnapshot := &securitySnapshot{generation: 1, digest: "old", activeWindow: window, validUntil: now.Add(time.Hour), fingerprints: map[string]string{"k1:1": "old"}} + newSnapshot := &securitySnapshot{generation: 2, digest: "new", activeWindow: window, validUntil: now.Add(time.Hour), fingerprints: map[string]string{"k1:1": "old", "k1:2": "new"}} + raw, err := json.Marshal(securityLedger{Generation: newSnapshot.generation, Digest: newSnapshot.digest, Fingerprints: newSnapshot.fingerprints}) + if err != nil { + t.Fatal(err) + } + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{securityStateDataKey: string(raw)}}) + started, release := make(chan struct{}), make(chan struct{}) + var startOnce sync.Once + kube.PrependReactor("get", "configmaps", func(_ k8stesting.Action) (bool, runtime.Object, error) { + startOnce.Do(func() { close(started) }) + <-release + return false, nil, nil + }) + manager := NewSecurityManager("", "", "ns", "authority", kube) + manager.now = func() time.Time { return now } + oldState := &securityState{snapshot: oldSnapshot} + manager.state.Store(oldState) + done := make(chan error, 1) + go func() { + _, authErr := manager.authoritative(t.Context()) + done <- authErr + }() + <-started + newState := &securityState{snapshot: newSnapshot} + manager.state.Store(newState) + close(release) + if err := <-done; !errors.Is(err, errAuthorityUnavailable) { + t.Fatalf("stale authority check error=%v", err) + } + if manager.state.Load() != newState || !manager.Ready() { + t.Fatal("stale authority check invalidated the newer published state") + } + + manager.invalidateObservedUnlessReplaced(t.Context(), oldState) + if manager.state.Load() != newState || !manager.Ready() { + t.Fatal("stale reload failure invalidated the newer authoritative state") + } +} + +func TestSecurityLedgerRejectsKeyVersionRollbackAcrossRestart(t *testing.T) { + ledger := securityLedger{Generation: 2, Fingerprints: map[string]string{"key:2": "old"}} + candidate := &securitySnapshot{generation: 3, fingerprints: map[string]string{"key:1": "new"}} + if _, err := advanceSecurityLedger(&ledger, candidate); err == nil { + t.Fatal("key version rollback was accepted") + } + candidate.fingerprints = map[string]string{"key:3": "new"} + changed, err := advanceSecurityLedger(&ledger, candidate) + if err != nil || !changed || candidate.fingerprints["key:2"] != "old" { + t.Fatalf("monotonic rotation failed: changed=%v err=%v fingerprints=%v", changed, err, candidate.fingerprints) + } +} + +func TestSecurityManagerGuardsEveryRPCOnExistingConnection(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC() + caPEM, serverPEM, serverKeyPEM, clientCert := rpcSecurityPKI(t, now, "spiffe://example/client") + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(dir, "grant.pem"), grantPriv) + for name, contents := range map[string][]byte{"server.crt": serverPEM, "server.key": serverKeyPEM, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + store := NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), "ns", testProfiles(), nil) + limiter, err := NewRPCLimiter(manager, 4, 2) + if err != nil { + t.Fatal(err) + } + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig())), grpc.UnaryInterceptor(limiter.UnaryInterceptor)) + executionv1.RegisterExecutionProviderServiceServer(server, NewHandler(HandlerConfig{Security: manager}, store)) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(caPEM) + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, ServerName: "provider.test", RootCAs: roots, Certificates: []tls.Certificate{clientCert}}))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + client := executionv1.NewExecutionProviderServiceClient(conn) + call := func() error { + _, err := client.ValidateProfile(t.Context(), &executionv1.ValidateProfileRequest{Profile: "go"}) + return err + } + if err := call(); err != nil { + t.Fatalf("initial RPC: %v", err) + } + cm, err := kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + authoritativeState := cm.Data[securityStateDataKey] + var drifted securityLedger + if err := executionenv.DecodeStrict([]byte(authoritativeState), &drifted); err != nil { + t.Fatal(err) + } + drifted.Digest = strings.Repeat("0", 64) + raw, _ := json.Marshal(drifted) + cm.Data[securityStateDataKey] = string(raw) + if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := call(); status.Code(err) != codes.Unavailable || manager.Ready() { + t.Fatalf("old snapshot survived durable authority drift: ready=%v err=%v", manager.Ready(), err) + } else if details := status.Convert(err).Details(); len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeNotReady) || !details[0].(*executionv1.ErrorDetail).Retryable { + t.Fatalf("authority lag was not a structured retryable not_ready: %v", details) + } + cm, _ = kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) + cm.Data[securityStateDataKey] = authoritativeState + if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + newCAPEM, newServerPEM, newServerKeyPEM, _ := rpcSecurityPKI(t, now, "spiffe://example/other") + for name, contents := range map[string][]byte{"server.crt": newServerPEM, "server.key": newServerKeyPEM, "clients.pem": newCAPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + manifest.Generation = 2 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if err := call(); status.Code(err) != codes.Unauthenticated { + t.Fatalf("old connection survived CA rotation: %v", err) + } + if err := os.WriteFile(manifestPath, []byte(`{"version":2}`), 0o600); err != nil { + t.Fatal(err) + } + if err := manager.Reload(t.Context()); err == nil { + t.Fatal("invalid manifest reloaded") + } + if err := call(); status.Code(err) != codes.Unavailable { + t.Fatalf("RPC remained authorized during invalid manifest: %v", err) + } + for name, contents := range map[string][]byte{"server.crt": serverPEM, "server.key": serverKeyPEM, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + manifest.Generation = 3 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil || call() != nil { + t.Fatalf("valid generation did not recover existing connection: %v", err) + } +} + +func TestRunClaimSigningAuthorityOutageIsRetryableAfterDurableMutation(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC() + caPEM, serverPEM, serverKeyPEM, clientCert := rpcSecurityPKI(t, now, "spiffe://example/client") + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(dir, "grant.pem"), grantPriv) + for name, contents := range map[string][]byte{"server.crt": serverPEM, "server.key": serverKeyPEM, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + var reads, failAt atomic.Int64 + kube.PrependReactor("get", "configmaps", func(_ k8stesting.Action) (bool, runtime.Object, error) { + read := reads.Add(1) + if target := failAt.Load(); target != 0 && read == target { + return true, nil, errors.New("injected authority read outage") + } + return false, nil, nil + }) + armSigningOutage := func() { + reads.Store(0) + failAt.Store(2) // Authorization reads first; response signing reads second. + } + recoverAuthority := func() { + failAt.Store(0) + if err := manager.Reload(t.Context()); err != nil { + t.Fatalf("reload authority: %v", err) + } + } + assertNotReady := func(err error) { + t.Helper() + st := status.Convert(err) + details := st.Details() + if st.Code() != codes.Unavailable || len(details) != 1 { + t.Fatalf("code=%v details=%v", st.Code(), details) + } + detail, ok := details[0].(*executionv1.ErrorDetail) + if !ok || detail.Code != string(executionenv.CodeNotReady) || !detail.Retryable { + t.Fatalf("detail=%v", details) + } + } + leaf, err := x509.ParseCertificate(clientCert.Certificate[0]) + if err != nil { + t.Fatal(err) + } + ctx := grpcpeer.NewContext(t.Context(), &grpcpeer.Peer{AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}, VerifiedChains: [][]*x509.Certificate{{leaf}}}}}) + backend := newDurableClaimBackend() + h := NewHandler(HandlerConfig{Security: manager}, backend) + owner := &executionv1.Owner{Issuer: "issuer", Subject: "alice"} + ref := &executionv1.EnvironmentRef{Id: "env", Revision: "rev"} + acquire := &executionv1.AcquireRunRequest{Environment: ref, Owner: owner, BindingId: "binding", RunId: "run", OperationId: "acquire-op", TtlMillis: time.Minute.Milliseconds()} + + armSigningOutage() + _, err = h.AcquireRun(ctx, acquire) + assertNotReady(err) + storedAcquire, calls, ok := backend.claim(acquire.OperationId, false) + if !ok || calls != 1 { + t.Fatalf("acquire mutation missing or retried automatically: stored=%t calls=%d", ok, calls) + } + recoverAuthority() + acquired, err := h.AcquireRun(ctx, acquire) + if err != nil { + t.Fatalf("explicit acquire retry: %v", err) + } + if callsClaim, retryCalls, ok := backend.claim(acquire.OperationId, false); !ok || retryCalls != 2 || callsClaim.ClaimID != storedAcquire.ClaimID || acquired.ClaimId != storedAcquire.ClaimID { + t.Fatalf("acquire retry did not converge: response=%q stored=%+v calls=%d", acquired.GetClaimId(), callsClaim, retryCalls) + } + + renew := &executionv1.RenewRunRequest{Environment: acquired.Environment, Owner: owner, BindingId: acquired.BindingId, RunId: acquired.RunId, ClaimId: acquired.ClaimId, Epoch: acquired.Epoch, GrantGeneration: acquired.GrantGeneration, OperationId: "renew-op", TtlMillis: time.Minute.Milliseconds()} + armSigningOutage() + _, err = h.RenewRun(ctx, renew) + assertNotReady(err) + storedRenew, calls, ok := backend.claim(renew.OperationId, true) + if !ok || calls != 1 { + t.Fatalf("renew mutation missing or retried automatically: stored=%t calls=%d", ok, calls) + } + recoverAuthority() + renewed, err := h.RenewRun(ctx, renew) + if err != nil { + t.Fatalf("explicit renew retry: %v", err) + } + if callsClaim, retryCalls, ok := backend.claim(renew.OperationId, true); !ok || retryCalls != 2 || callsClaim.ClaimID != storedRenew.ClaimID || callsClaim.GrantGeneration != storedRenew.GrantGeneration || renewed.ClaimId != storedRenew.ClaimID || renewed.GrantGeneration != storedRenew.GrantGeneration { + t.Fatalf("renew retry did not converge: response=%q/%d stored=%+v calls=%d", renewed.GetClaimId(), renewed.GetGrantGeneration(), callsClaim, retryCalls) + } +} + +func TestSecurityManagerAuthorizesPresentedIntermediateAndRevokesRootOnExistingConnection(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC() + caPEM, serverPEM, serverKeyPEM, clientCert := rpcSecurityIntermediatePKI(t, now, "spiffe://example/client") + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(dir, "grant.pem"), grantPriv) + for name, contents := range map[string][]byte{"server.crt": serverPEM, "server.key": serverKeyPEM, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + store := NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), "ns", testProfiles(), nil) + limiter, err := NewRPCLimiter(manager, 4, 2) + if err != nil { + t.Fatal(err) + } + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig())), grpc.UnaryInterceptor(limiter.UnaryInterceptor)) + executionv1.RegisterExecutionProviderServiceServer(server, NewHandler(HandlerConfig{Security: manager}, store)) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(caPEM) + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, ServerName: "provider.test", RootCAs: roots, Certificates: []tls.Certificate{clientCert}}))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + client := executionv1.NewExecutionProviderServiceClient(conn) + call := func() error { + _, err := client.ValidateProfile(t.Context(), &executionv1.ValidateProfileRequest{Profile: "go"}) + return err + } + if err := call(); err != nil { + t.Fatalf("intermediate-signed client RPC: %v", err) + } + removedRoot, _, _, _ := rpcSecurityPKI(t, now, "spiffe://example/other") + if err := os.WriteFile(filepath.Join(dir, "clients.pem"), removedRoot, 0o600); err != nil { + t.Fatal(err) + } + manifest.Generation = 2 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if err := call(); status.Code(err) != codes.Unauthenticated { + t.Fatalf("existing connection survived removal of intermediate issuer root: %v", err) + } +} + +func rpcSecurityIntermediatePKI(t *testing.T, now time.Time, clientURI string) ([]byte, []byte, []byte, tls.Certificate) { + t.Helper() + rootPub, rootKey, _ := ed25519.GenerateKey(rand.Reader) + root := &x509.Certificate{SerialNumber: big.NewInt(201), Subject: pkix.Name{CommonName: "root"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + rootDER, err := x509.CreateCertificate(rand.Reader, root, root, rootPub, rootKey) + if err != nil { + t.Fatal(err) + } + intermediatePub, intermediateKey, _ := ed25519.GenerateKey(rand.Reader) + intermediate := &x509.Certificate{SerialNumber: big.NewInt(202), Subject: pkix.Name{CommonName: "intermediate"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + intermediateDER, err := x509.CreateCertificate(rand.Reader, intermediate, root, intermediatePub, rootKey) + if err != nil { + t.Fatal(err) + } + issue := func(serial int64, parent *x509.Certificate, parentKey ed25519.PrivateKey, uriText string, usages []x509.ExtKeyUsage, dns []string) ([]byte, ed25519.PrivateKey) { + pub, key, _ := ed25519.GenerateKey(rand.Reader) + tmpl := &x509.Certificate{SerialNumber: big.NewInt(serial), NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: usages, DNSNames: dns} + if uriText != "" { + u, _ := url.Parse(uriText) + tmpl.URIs = []*url.URL{u} + } + der, issueErr := x509.CreateCertificate(rand.Reader, tmpl, parent, pub, parentKey) + if issueErr != nil { + t.Fatal(issueErr) + } + return der, key + } + serverDER, serverKey := issue(203, root, rootKey, "spiffe://example/provider", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, []string{"provider.test"}) + clientDER, clientKey := issue(204, intermediate, intermediateKey, clientURI, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, nil) + encodeKey := func(key ed25519.PrivateKey) []byte { + raw, marshalErr := x509.MarshalPKCS8PrivateKey(key) + if marshalErr != nil { + t.Fatal(marshalErr) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: raw}) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: rootDER}), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}), encodeKey(serverKey), tls.Certificate{Certificate: [][]byte{clientDER, intermediateDER}, PrivateKey: clientKey} +} + +func rpcSecurityPKI(t *testing.T, now time.Time, clientURI string) ([]byte, []byte, []byte, tls.Certificate) { + t.Helper() + caPub, caKey, _ := ed25519.GenerateKey(rand.Reader) + ca := &x509.Certificate{SerialNumber: big.NewInt(101), NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, caPub, caKey) + if err != nil { + t.Fatal(err) + } + issue := func(serial int64, uriText string, usages []x509.ExtKeyUsage, dns []string) ([]byte, ed25519.PrivateKey) { + pub, key, _ := ed25519.GenerateKey(rand.Reader) + tmpl := &x509.Certificate{SerialNumber: big.NewInt(serial), NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: usages, DNSNames: dns} + if uriText != "" { + u, _ := url.Parse(uriText) + tmpl.URIs = []*url.URL{u} + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca, pub, caKey) + if err != nil { + t.Fatal(err) + } + return der, key + } + serverDER, serverKey := issue(102, "spiffe://example/provider", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, []string{"provider.test"}) + clientDER, clientKey := issue(103, clientURI, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, nil) + encodeKey := func(key ed25519.PrivateKey) []byte { + raw, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: raw}) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}), encodeKey(serverKey), tls.Certificate{Certificate: [][]byte{clientDER}, PrivateKey: clientKey} +} + +func TestAuthorityDigestCoversSecurityMeaningfulState(t *testing.T) { + base := securityManifest{Version: 1, Generation: 7, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "key.pem", PublicSHA256: "fingerprint", ActivateAt: time.Unix(1, 0).UTC(), VerifyUntil: time.Unix(100, 0).UTC(), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "ca.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client"}}} + digest := func(m securityManifest, ca, server string) string { + t.Helper() + ttl, err := time.ParseDuration(m.GrantTTLText) + if err != nil { + t.Fatal(err) + } + skew, err := time.ParseDuration(m.ClockSkewText) + if err != nil { + t.Fatal(err) + } + got, err := authorityDigest(m, ttl, skew, map[string]string{"k1:1": "fingerprint"}, []string{ca}, server) + if err != nil { + t.Fatal(err) + } + return got + } + original := digest(base, "ca-1", "server-1") + mutations := map[string]func(*securityManifest){ + "capability": func(m *securityManifest) { m.Clients[0].Administrator = true }, + "key revoke": func(m *securityManifest) { m.Keys[0].State = "revoked" }, + "active key": func(m *securityManifest) { m.ActiveKeyID = "k2" }, + "activation time": func(m *securityManifest) { m.Keys[0].ActivateAt = m.Keys[0].ActivateAt.Add(time.Second) }, + "grant ttl": func(m *securityManifest) { m.GrantTTLText = "2m" }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + candidate := base + candidate.Keys = slices.Clone(base.Keys) + candidate.Clients = slices.Clone(base.Clients) + mutate(&candidate) + if digest(candidate, "ca-1", "server-1") == original { + t.Fatal("mutation did not change authority digest") + } + }) + } + if digest(base, "ca-2", "server-1") == original || digest(base, "ca-1", "server-2") == original { + t.Fatal("CA or server identity swap did not change authority digest") + } +} + +func TestSecurityManagerSigningWindowIsCheckedOnEveryRequest(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC() + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(dir, "grant.pem"), grantPriv) + caPEM, serverCert, serverKey := syntheticPKI(t, now) + for name, contents := range map[string][]byte{"server.crt": serverCert, "server.key": serverKey, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(10 * time.Second), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + clock := now + manager.now = func() time.Time { return clock } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + h := NewHandler(HandlerConfig{Security: manager}, &fakeBackend{}) + claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1} + if _, expiry, err := h.signClaim(t.Context(), claim, "spiffe://example/client", "owner"); err != nil || expiry.After(manifest.Keys[0].VerifyUntil) { + t.Fatalf("initial sign expiry=%v err=%v", expiry, err) + } + clock = manifest.Keys[0].VerifyUntil + if manager.Ready() { + t.Fatal("expired active key remained ready between reload ticks") + } + if _, _, err := h.signClaim(t.Context(), claim, "spiffe://example/client", "owner"); err == nil { + t.Fatal("expired active key signed between reload ticks") + } + manifest.Generation = 2 + manifest.Keys[0].VerifyUntil = clock.Add(time.Hour) + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { + t.Fatalf("corrected generation did not recover: ready=%v err=%v", manager.Ready(), err) + } +} + +func TestSecurityManagerRejectsEscapingProjectedSymlink(t *testing.T) { + dir := t.TempDir() + if err := os.Symlink("../../outside", filepath.Join(dir, "grant.pem")); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := root.Close(); err != nil { + t.Errorf("close root: %v", err) + } + }) + if _, err := readRootFile(root, "grant.pem"); err == nil { + t.Fatal("escaping symlink was read") + } +} + +func writeManifest(t *testing.T, path string, manifest securityManifest) { + t.Helper() + b, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatal(err) + } +} +func writePKCS8(t *testing.T, path string, key ed25519.PrivateKey) { + t.Helper() + b, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: b}), 0o600); err != nil { + t.Fatal(err) + } +} +func syntheticPKI(t *testing.T, now time.Time) ([]byte, []byte, []byte) { + t.Helper() + caPub, caKey, _ := ed25519.GenerateKey(rand.Reader) + ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test-ca"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(2 * time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, caPub, caKey) + if err != nil { + t.Fatal(err) + } + serverPub, serverKey, _ := ed25519.GenerateKey(rand.Reader) + uri, _ := url.Parse("spiffe://example/provider") + server := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "provider"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, URIs: []*url.URL{uri}} + serverDER, err := x509.CreateCertificate(rand.Reader, server, ca, serverPub, caKey) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(serverKey) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) +} diff --git a/internal/adapter/executioncontroller/store.go b/internal/adapter/executioncontroller/store.go new file mode 100644 index 0000000000..4ae2b2b930 --- /dev/null +++ b/internal/adapter/executioncontroller/store.go @@ -0,0 +1,604 @@ +//nolint:revive // Private store methods implement adapter-only lifecycle interfaces. +package executioncontroller + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math" + "slices" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// ExecutionEnvironmentGVR identifies the private execution-environment CRD. +var ExecutionEnvironmentGVR = schema.GroupVersionResource{Group: "execution.mecatl.dev", Version: "v1alpha1", Resource: "executionenvironments"} + +const ( + maxReferences = 64 + environmentNotFoundMessage = "environment not found" + currentSchemaVersion = int64(2) + operationLeaseTTL = 30 * time.Second + transitioningMessage = "environment is transitioning" + operationIDField = "operationID" +) + +// ExecutorTransport dispatches one request to a fixed workload helper. +type ExecutorTransport interface { + Execute(context.Context, string, executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) +} + +// Store persists authorization and fencing state in ExecutionEnvironment status. +type Store struct { + resources dynamic.ResourceInterface + profiles *Profiles + executor ExecutorTransport + kube kubernetes.Interface + namespace string + holderID string + now func() time.Time + opTTL time.Duration +} + +// NewStore constructs a namespace-scoped execution state store. +func NewStore(client dynamic.Interface, namespace string, profiles *Profiles, executor ExecutorTransport) *Store { + holder, err := randomID() + if err != nil { + holder = "unavailable" + } + return &Store{resources: client.Resource(ExecutionEnvironmentGVR).Namespace(namespace), profiles: profiles, executor: executor, namespace: namespace, holderID: holder, now: func() time.Time { return time.Now().UTC() }, opTTL: 30 * time.Second} +} + +// WithKubeClient enables provider-owned executor and retained-volume lifecycle operations. +func (s *Store) WithKubeClient(kube kubernetes.Interface) *Store { + s.kube = kube + return s +} + +// ValidateProfile returns one validated immutable profile. +func (s *Store) ValidateProfile(_ context.Context, name string) (Profile, error) { + p, ok := s.profiles.get(name) + if !ok { + return Profile{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "profile not found"} + } + return Profile{Name: name, Digest: p.Digest, MaxFileBytes: p.Spec.MaxFileBytes, MaxCommandBytes: p.Spec.MaxCommandBytes, MaxCommandDuration: p.Spec.MaxCommandDuration, Capabilities: []string{"filesystem", "foreground-command"}}, nil +} + +// Ensure idempotently allocates an environment for an immutable request fingerprint. +func (s *Store) Ensure(ctx context.Context, client, owner, binding, profile, fp string) (Allocation, error) { + return s.EnsurePending(ctx, client, owner, binding, profile, fp, "legacy-"+fp) +} + +func (s *Store) EnsurePending(ctx context.Context, client, owner, binding, profile, fp, operationID string) (Allocation, error) { + return s.ensurePending(ctx, client, owner, executionenv.Owner{}, binding, profile, fp, operationID) +} + +// EnsurePendingOwned persists the attested principal needed for owner-safe intent reconciliation. +func (s *Store) EnsurePendingOwned(ctx context.Context, client, expectedOwnerHash string, owner executionenv.Owner, binding, profile, fp, operationID string) (Allocation, error) { + if owner.Issuer == "" || owner.Subject == "" || ownerHash(owner) != expectedOwnerHash { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "owner attestation is invalid"} + } + return s.ensurePending(ctx, client, expectedOwnerHash, owner, binding, profile, fp, operationID) +} + +func (s *Store) ensurePending(ctx context.Context, client, owner string, attested executionenv.Owner, binding, profile, fp, operationID string) (Allocation, error) { + p, ok := s.profiles.get(profile) + if !ok { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "profile not found"} + } + name := allocationName(client, owner, binding) + cur, err := s.resources.Get(ctx, name, metav1.GetOptions{}) + if err == nil { + if err := s.initializeStatus(ctx, cur, binding, operationID); err != nil { + return Allocation{}, err + } + cur, err = s.resources.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return Allocation{}, err + } + if !referenceOperationMatches(cur, binding, operationID) { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference operation conflicts with existing environment"} + } + return allocationFrom(cur, client, owner, binding, profile, fp) + } + if !apierrors.IsNotFound(err) { + return Allocation{}, fmt.Errorf("get execution environment: %w", err) + } + revision, err := randomID() + if err != nil { + return Allocation{}, err + } + if s.kube != nil { + if err := s.reserveProfileSlot(ctx, profile, name, p.Spec.MaxEnvironments); err != nil { + return Allocation{}, err + } + } + spec := map[string]any{"schemaVersion": currentSchemaVersion, "allocationID": name, "revision": revision, "ownerHash": owner, "clientHash": hashText(client), "bindingID": binding, "requestFingerprint": fp, "profile": profile, "profileDigest": p.Digest, "image": p.Spec.Image, "storageClass": p.Spec.StorageClass, "storageSize": p.Spec.StorageSize, "resources": map[string]any{"cpuRequest": p.Spec.CPURequest, "memoryRequest": p.Spec.MemoryRequest, "cpuLimit": p.Spec.CPULimit, "memoryLimit": p.Spec.MemoryLimit}, "desired": "Active"} + if attested.Issuer != "" && attested.Subject != "" { + spec["ownerIssuer"] = attested.Issuer + spec["ownerSubject"] = attested.Subject + } + obj := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": name, "finalizers": []any{environmentFinalizer}}, "spec": spec}} + created, err := s.resources.Create(ctx, obj, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + created, err = s.resources.Get(ctx, name, metav1.GetOptions{}) + } + if err != nil { + if s.kube != nil && definitiveCreateRejection(err) { + if releaseErr := releaseProfileSlot(ctx, s.kube, s.namespace, profile, name); releaseErr != nil { + return Allocation{}, errors.Join(fmt.Errorf("create execution environment: %w", err), fmt.Errorf("release profile allocation: %w", releaseErr)) + } + } + return Allocation{}, fmt.Errorf("create execution environment: %w", err) + } + if err := s.initializeStatus(ctx, created, binding, operationID); err != nil { + return Allocation{}, err + } + created, err = s.resources.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return Allocation{}, err + } + return allocationFrom(created, client, owner, binding, profile, fp) +} + +func definitiveCreateRejection(err error) bool { + return apierrors.IsInvalid(err) || apierrors.IsForbidden(err) || apierrors.IsUnauthorized(err) || apierrors.IsBadRequest(err) || apierrors.IsMethodNotSupported(err) || apierrors.IsRequestEntityTooLargeError(err) +} + +const profileAllocationConfigMap = "mecatl-execution-profile-allocations" + +func profileAllocationKey(profile string) string { + keySum := sha256.Sum256([]byte(profile)) + return "profile-" + hex.EncodeToString(keySum[:16]) + ".json" +} + +func (s *Store) reserveProfileSlot(ctx context.Context, profile, allocationID string, limit int) error { //nolint:gocyclo // Durable capacity CAS and CR reconciliation are intentionally one transaction loop. + cms := s.kube.CoreV1().ConfigMaps(s.namespace) + key := profileAllocationKey(profile) + for range 8 { + cm, err := cms.Get(ctx, profileAllocationConfigMap, metav1.GetOptions{}) + exists := true + if apierrors.IsNotFound(err) { + exists = false + cm = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: s.namespace}, Data: map[string]string{}} + } else if err != nil { + return fmt.Errorf("read profile allocation authority: %w", err) + } + var slots []string + if raw := cm.Data[key]; raw != "" { + if err := executionenv.DecodeStrict([]byte(raw), &slots); err != nil || len(slots) > limit { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "profile allocation authority is invalid"} + } + } + if slices.Contains(slots, allocationID) { + return nil + } + all, err := s.resources.List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("list profile allocations: %w", err) + } + for i := range all.Items { + item := &all.Items[i] + deallocating := conditionTrue(item, "Retired") && conditionTrue(item, "ExecutorTerminated") && textNested(item.Object, "status", "lifecycleOperation", "type") == "DeleteRetiredEnvironment" && textNested(item.Object, "status", "lifecycleOperation", "phase") == "ReleasingSlot" + if textNested(item.Object, "spec", "profile") == profile && !deallocating && !slices.Contains(slots, item.GetName()) { + slots = append(slots, item.GetName()) + } + } + if len(slots) >= limit { + return &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "profile environment limit reached"} + } + slots = append(slots, allocationID) + slices.Sort(slots) + raw, err := json.Marshal(slots) + if err != nil { + return err + } + if cm.Data == nil { + cm.Data = map[string]string{} + } + cm.Data[key] = string(raw) + if exists { + _, err = cms.Update(ctx, cm, metav1.UpdateOptions{}) + } else { + _, err = cms.Create(ctx, cm, metav1.CreateOptions{}) + } + if err == nil { + return nil + } + if !apierrors.IsConflict(err) && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("reserve profile allocation: %w", err) + } + } + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "profile allocation authority changed concurrently", Retryable: true} +} + +func releaseProfileSlot(ctx context.Context, kube kubernetes.Interface, namespace, profile, allocationID string) error { + cms := kube.CoreV1().ConfigMaps(namespace) + key := profileAllocationKey(profile) + for range 8 { + cm, err := cms.Get(ctx, profileAllocationConfigMap, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + var slots []string + if err := executionenv.DecodeStrict([]byte(cm.Data[key]), &slots); err != nil { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "profile allocation authority is invalid"} + } + index := slices.Index(slots, allocationID) + if index < 0 { + return nil + } + slots = append(slots[:index], slots[index+1:]...) + raw, err := json.Marshal(slots) + if err != nil { + return err + } + cm.Data[key] = string(raw) + if _, err = cms.Update(ctx, cm, metav1.UpdateOptions{}); err == nil { + return nil + } + if !apierrors.IsConflict(err) { + return err + } + } + return errors.New("profile allocation authority changed concurrently") +} + +func (s *Store) initializeStatus(ctx context.Context, env *unstructured.Unstructured, binding, operationID string) error { + if operationID == "" { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "operation identity is required"} + } + if intNested(env.Object, "status", "epoch") > 0 { + return requireCurrentSchema(env) + } + return s.retryUpdateStatusRaw(ctx, env.GetName(), func(o *unstructured.Unstructured) error { + if intNested(o.Object, "status", "epoch") > 0 { + return requireCurrentSchema(o) + } + if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion { + return incompatibleSchemaError() + } + if err := unstructured.SetNestedField(o.Object, currentSchemaVersion, "status", "schemaVersion"); err != nil { + return err + } + if err := unstructured.SetNestedField(o.Object, int64(1), "status", "epoch"); err != nil { + return err + } + if err := unstructured.SetNestedField(o.Object, int64(1), "status", "grantGeneration"); err != nil { + return err + } + ref := map[string]any{"bindingID": binding, "state": string(executionenv.ReferencePendingCreate), operationIDField: operationID, "createdAt": time.Now().UTC().Format(time.RFC3339Nano)} + if err := unstructured.SetNestedSlice(o.Object, []any{ref}, "status", "references"); err != nil { + return err + } + return unstructured.SetNestedField(o.Object, fenceHealthy, "status", "fenceState") + }) +} +func allocationName(client, owner, binding string) string { + sum := sha256.Sum256([]byte(client + "\x00" + owner + "\x00" + binding)) + return "exec-" + hex.EncodeToString(sum[:20]) +} +func allocationFrom(o *unstructured.Unstructured, client, owner, binding, profile, fp string) (Allocation, error) { + spec, _, _ := unstructured.NestedMap(o.Object, "spec") + if text(spec, "ownerHash") != owner || text(spec, "clientHash") != hashText(client) || text(spec, "bindingID") != binding || text(spec, "profile") != profile || text(spec, "requestFingerprint") != fp { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "allocation identity conflicts with existing environment"} + } + return exactAllocation(o, client, owner, binding) +} +func exactAllocation(o *unstructured.Unstructured, client, owner, binding string) (Allocation, error) { + spec, _, _ := unstructured.NestedMap(o.Object, "spec") + if text(spec, "ownerHash") != owner || text(spec, "clientHash") != hashText(client) { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + revision := text(spec, "revision") + if revision == "" { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment identity is incomplete"} + } + epoch, ok := epochValue(o) + if !ok { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment epoch is invalid"} + } + generation := intNested(o.Object, "status", "grantGeneration") + if generation <= 0 { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "grant generation is invalid"} + } + ready := conditionTrue(o, "Ready") && textNested(o.Object, "status", "fenceState") == fenceHealthy + return Allocation{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: revision}, Epoch: epoch, GrantGeneration: uint64(generation), OwnerHash: owner, BindingID: binding, Client: client, Ready: ready}, nil +} + +// Attach adds a binding reference only to the exact healthy environment. +func (s *Store) Attach(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) (Allocation, error) { + var out Allocation + err := s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + a, err := exactAllocation(o, client, owner, binding) + if err != nil { + return err + } + if a.Environment != ref || textNested(o.Object, "spec", "desired") != "Active" { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment revision unavailable"} + } + if !a.Ready { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment is not ready at requested epoch"} + } + refs, refsErr := referenceRecords(o) + if refsErr != nil { + return refsErr + } + if !attachedReference(refs, binding) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + out = a + return nil + }) + return out, err +} + +// ReleaseReference removes one binding reference without deleting the environment. +func (s *Store) ReleaseReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + a, err := exactAllocation(o, client, owner, binding) + if err != nil { + return err + } + if a.Environment != ref { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, refsErr := referenceRecords(o) + if refsErr != nil { + return refsErr + } + for i := range refs { + if refs[i].BindingID == binding { + if refs[i].State != executionenv.ReferencePublished { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference has a pending transaction"} + } + return setReferenceRecords(o, append(refs[:i], refs[i+1:]...)) + } + } + return nil + }) +} + +// File performs one claimed and fenced filesystem operation. +func (s *Store) File(ctx context.Context, client, owner string, q executionenv.FileRequest) (executionenv.FileResponse, error) { + req := executionenv.ExecutorRequest{Operation: q.Operation, Path: q.Path, Destination: q.Destination, Pattern: q.Pattern, Data: q.Data, Version: q.Version, Limit: q.Limit} + resp, err := s.execute(ctx, client, owner, q.Context, req) + return resp.FileResponse, err +} + +// StartCommand performs one claimed foreground command through terminal receipt. +func (s *Store) StartCommand(ctx context.Context, client, owner string, q executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + id, err := randomID() + if err != nil { + return executionenv.CommandStartResponse{}, err + } + resp, err := s.execute(ctx, client, owner, q.Context, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, Command: q.Command, CommandID: id, TimeoutMillis: q.TimeoutMillis}) + if err != nil { + return executionenv.CommandStartResponse{}, err + } + if resp.Command == nil { + return executionenv.CommandStartResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "executor returned no terminal command receipt"} + } + return executionenv.CommandStartResponse{CommandID: id, State: resp.Command.State, Result: *resp.Command}, nil +} + +// CommandStatus reports that foreground-only commands have no detached status API. +func (*Store) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "foreground commands have no resumable status stream"} +} + +// CancelCommand refuses cancellation without an attached foreground control stream. +func (*Store) CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "no active foreground stream is attached; external fencing is required"} +} +func (s *Store) execute(ctx context.Context, client, owner string, rc executionenv.RequestContext, req executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { //nolint:gocyclo // Atomic claim, dispatch, and fencing remain one auditable transaction. + if rc.Epoch == 0 || rc.Epoch > math.MaxInt64 || rc.GrantGeneration == 0 || rc.GrantGeneration > math.MaxInt64 { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "invalid execution epoch or grant generation"} + } + epochStatus := int64(rc.Epoch) //nolint:gosec // bounded above by MaxInt64. + if s.executor == nil { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeNotReady, Message: "executor transport unavailable", Retryable: true} + } + opID, err := randomID() + if err != nil { + return executionenv.ExecutorResponse{}, err + } + var pod string + var maxFileBytes, maxCommandBytes int64 + err = s.retryUpdateStatus(ctx, rc.Environment.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + profile, ok := s.profiles.get(textNested(o.Object, "spec", "profile")) + if !ok || profile.Digest != textNested(o.Object, "spec", "profileDigest") { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment profile is unavailable"} + } + maxFileBytes, maxCommandBytes = profile.Spec.MaxFileBytes, profile.Spec.MaxCommandBytes + if int64(len(req.Data)) > maxFileBytes || int64(len(req.Command)) > maxCommandBytes { + return &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "operation exceeds profile bounds"} + } + if req.Operation == executionenv.OpCommandStart { + maxMillis := profile.Spec.MaxCommandDuration.Milliseconds() + if req.TimeoutMillis <= 0 { + req.TimeoutMillis = maxMillis + } else if req.TimeoutMillis > maxMillis { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "command timeout exceeds profile bound"} + } + } + refs, refsErr := referenceRecords(o) + if refsErr != nil { + return refsErr + } + claim, _, expiry, claimOK := activeRunFrom(o) + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || !publishedReference(refs, rc.BindingID) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if !claimOK || !time.Now().UTC().Before(expiry) || claim.BindingID != rc.BindingID || claim.RunID != rc.RunID || claim.ClaimID != rc.ClaimID || claim.Epoch != rc.Epoch || claim.GrantGeneration != rc.GrantGeneration || !generationMatches(o, rc.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim is not current"} + } + if textNested(o.Object, "spec", "revision") != rc.Environment.Revision || !epochMatches(o, rc.Epoch) || textNested(o.Object, "spec", "desired") != "Active" || !conditionTrue(o, "Ready") || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment is not ready at requested epoch"} + } + if textNested(o.Object, "status", "activeOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment already has an active operation", Retryable: true} + } + pod = textNested(o.Object, "status", "pod", "name") + if pod == "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "executor pod is unavailable", Retryable: true} + } + now := s.now() + return unstructured.SetNestedMap(o.Object, map[string]any{"id": opID, "operation": string(req.Operation), "startedAt": now.Format(time.RFC3339Nano), "claimID": rc.ClaimID, "runID": rc.RunID, "epoch": epochStatus, "holderID": s.holderID, "renewedAt": now.Format(time.RFC3339Nano), "expiresAt": now.Add(s.opTTL).Format(time.RFC3339Nano)}, "status", "activeOperation") + }) + if err != nil { + return executionenv.ExecutorResponse{}, err + } + current, err := s.resources.Get(ctx, rc.Environment.ID, metav1.GetOptions{}) + if err != nil || textNested(current.Object, "spec", "ownerHash") != owner || textNested(current.Object, "spec", "clientHash") != hashText(client) || textNested(current.Object, "spec", "revision") != rc.Environment.Revision || !epochMatches(current, rc.Epoch) || !operationMatches(current, opID, s.holderID, rc.ClaimID, rc.Epoch) { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "operation identity changed before executor dispatch"} + } + opCtx, stopLease := context.WithCancel(ctx) + leaseDone := make(chan error, 1) + go func() { leaseDone <- s.renewOperationLease(opCtx, rc.Environment.ID, opID, rc.ClaimID, rc.Epoch) }() + resp, dispatchErr := s.executor.Execute(opCtx, pod, req) + stopLease() + leaseErr := <-leaseDone + if leaseErr != nil && dispatchErr == nil { + dispatchErr = &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "operation lease renewal failed"} + } + if dispatchErr == nil && (int64(len(resp.Data)) > maxFileBytes || resp.Command != nil && int64(len(resp.Command.Stdout)+len(resp.Command.Stderr)) > maxCommandBytes) { + dispatchErr = &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "executor response exceeds profile bound"} + } + var controlledErr *executionenv.Error + controlledTerminal := errors.As(dispatchErr, &controlledErr) && controlledErr.Code != executionenv.CodeFenceUnknown + terminal := controlledTerminal || dispatchErr == nil && (req.Operation != executionenv.OpCommandStart || resp.Command != nil && resp.Command.TerminalReceipt != "") + finishErr := s.retryUpdateStatus(context.WithoutCancel(ctx), rc.Environment.ID, func(o *unstructured.Unstructured) error { + if !operationMatches(o, opID, s.holderID, rc.ClaimID, rc.Epoch) { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "operation ownership changed before completion"} + } + if terminal { + unstructured.RemoveNestedField(o.Object, "status", "activeOperation") + return nil + } + return unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") + }) + if finishErr != nil { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "could not persist terminal operation state"} + } + if controlledTerminal { + return executionenv.ExecutorResponse{}, controlledErr + } + if !terminal { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "executor termination is unconfirmed; environment fenced", Retryable: false} + } + return resp, nil +} +func (s *Store) retryUpdateStatus(ctx context.Context, name string, mutate func(*unstructured.Unstructured) error) error { + return s.retryUpdateStatusRaw(ctx, name, func(o *unstructured.Unstructured) error { + if err := requireCurrentSchema(o); err != nil { + return err + } + return mutate(o) + }) +} + +func (s *Store) retryUpdateStatusRaw(ctx context.Context, name string, mutate func(*unstructured.Unstructured) error) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + cur, err := s.resources.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if err != nil { + return err + } + if err := mutate(cur); err != nil { + return err + } + _, err = s.resources.UpdateStatus(ctx, cur, metav1.UpdateOptions{}) + return err + }) + if apierrors.IsConflict(err) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment changed concurrently", Retryable: true} + } + return err +} +func requireCurrentSchema(o *unstructured.Unstructured) error { + if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion || intNested(o.Object, "status", "schemaVersion") != currentSchemaVersion { + return incompatibleSchemaError() + } + return nil +} + +func incompatibleSchemaError() error { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment schema is incompatible; explicit migration is required"} +} + +func epochValue(o *unstructured.Unstructured) (uint64, bool) { + epoch, found, err := unstructured.NestedInt64(o.Object, "status", "epoch") + if err != nil || !found || epoch <= 0 { + return 0, false + } + return uint64(epoch), true +} + +func epochMatches(o *unstructured.Unstructured, expected uint64) bool { + epoch, ok := epochValue(o) + return ok && epoch == expected +} + +func conditionTrue(o *unstructured.Unstructured, name string) bool { + conds, _, _ := unstructured.NestedSlice(o.Object, "status", "conditions") + for _, raw := range conds { + m, ok := raw.(map[string]any) + if ok && text(m, "type") == name && text(m, "status") == "True" { + return true + } + } + return false +} +func text(m map[string]any, k string) string { v, _ := m[k].(string); return v } +func textNested(m map[string]any, p ...string) string { + v, _, _ := unstructured.NestedString(m, p...) + return v +} +func intNested(m map[string]any, p ...string) int64 { + v, _, _ := unstructured.NestedInt64(m, p...) + return v +} +func hashText(v string) string { sum := sha256.Sum256([]byte(v)); return hex.EncodeToString(sum[:]) } +func randomID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} +func contains(v []string, s string) bool { + for _, x := range v { + if x == s { + return true + } + } + return false +} + +var _ Backend = (*Store)(nil) diff --git a/internal/adapter/executioncontroller/store_concurrency_test.go b/internal/adapter/executioncontroller/store_concurrency_test.go new file mode 100644 index 0000000000..65c577f1bb --- /dev/null +++ b/internal/adapter/executioncontroller/store_concurrency_test.go @@ -0,0 +1,237 @@ +package executioncontroller + +import ( + "context" + "errors" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func profileSlots(t *testing.T, kube *kubefake.Clientset) []string { + t.Helper() + cm, err := kube.CoreV1().ConfigMaps("ns").Get(t.Context(), profileAllocationConfigMap, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + var slots []string + if err := executionenv.DecodeStrict([]byte(cm.Data[profileAllocationKey("go")]), &slots); err != nil { + t.Fatal(err) + } + return slots +} + +func TestDefinitiveEnvironmentCreateFailureReleasesProfileSlot(t *testing.T) { + dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + dynamicClient.PrependReactor("create", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: ExecutionEnvironmentGVR.Group, Resource: ExecutionEnvironmentGVR.Resource}, "env", errors.New("policy")) + }) + kube := kubefake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{}}) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kube) + if _, err := store.EnsurePending(t.Context(), "client", "owner", "binding", "go", "fp", "op"); err == nil { + t.Fatal("definitive create rejection succeeded") + } + if slots := profileSlots(t, kube); len(slots) != 0 { + t.Fatalf("definitive create rejection leaked slots: %v", slots) + } +} + +func TestAmbiguousEnvironmentCreateFailureRetainsProfileSlot(t *testing.T) { + dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + dynamicClient.PrependReactor("create", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("transport outcome unknown") + }) + kube := kubefake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{}}) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kube) + if _, err := store.EnsurePending(t.Context(), "client", "owner", "binding", "go", "fp", "op"); err == nil { + t.Fatal("ambiguous create failure succeeded") + } + if slots := profileSlots(t, kube); len(slots) != 1 { + t.Fatalf("ambiguous create outcome did not retain conservative capacity: %v", slots) + } +} + +func TestRetainedDeleteWaitsForSlotReleaseBeforeRemovingCRFinalizer(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "delete", "type": "DeleteRetiredEnvironment", "phase": "DeletingPVC", "expectedPVCUID": "pvc-uid", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + allocationID := env.GetName() + raw := `["` + allocationID + `"]` + kube := kubefake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{profileAllocationKey("go"): raw}}) + var fail atomic.Bool + fail.Store(true) + kube.PrependReactor("get", "configmaps", func(k8stesting.Action) (bool, runtime.Object, error) { + if fail.Load() { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Resource: "configmaps"}, profileAllocationConfigMap, errors.New("outage")) + } + return false, nil, nil + }) + r := NewReconciler(dynamicClient, kube, "ns", testProfiles()) + if err := r.Reconcile(t.Context(), env.GetName()); err != nil { + t.Fatalf("failed to persist deallocation phase: %v", err) + } + if err := r.Reconcile(t.Context(), env.GetName()); err == nil { + t.Fatal("slot authority outage did not stop deletion") + } + got, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), env.GetName(), metav1.GetOptions{}) + if err != nil || !contains(got.GetFinalizers(), environmentFinalizer) { + t.Fatalf("CR was not retained for retry: finalizers=%v err=%v", got.GetFinalizers(), err) + } + fail.Store(false) + if err := r.Reconcile(t.Context(), env.GetName()); err != nil { + t.Fatal(err) + } + if slots := profileSlots(t, kube); len(slots) != 0 { + t.Fatalf("retry did not release slot: %v", slots) + } +} + +func TestConcurrentProfileReservationsEnforceHardLimit(t *testing.T) { + dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + ledger := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns", ResourceVersion: "1"}, Data: map[string]string{}} + kubes := []*kubefake.Clientset{kubefake.NewSimpleClientset(ledger), kubefake.NewSimpleClientset(ledger)} + var mu sync.Mutex + current := ledger.DeepCopy() + barrier := make(chan struct{}) + var reads atomic.Int32 + for _, kube := range kubes { + kube.PrependReactor("get", "configmaps", func(k8stesting.Action) (bool, runtime.Object, error) { + mu.Lock() + out := current.DeepCopy() + mu.Unlock() + if reads.Add(1) == 2 { + close(barrier) + } + if reads.Load() <= 2 { + <-barrier + } + return true, out, nil + }) + kube.PrependReactor("update", "configmaps", func(action k8stesting.Action) (bool, runtime.Object, error) { + candidate := action.(k8stesting.UpdateAction).GetObject().(*corev1.ConfigMap) + mu.Lock() + defer mu.Unlock() + if candidate.ResourceVersion != current.ResourceVersion { + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "configmaps"}, candidate.Name, errors.New("resource version changed")) + } + rv, _ := strconv.Atoi(current.ResourceVersion) + current = candidate.DeepCopy() + current.ResourceVersion = strconv.Itoa(rv + 1) + return true, current.DeepCopy(), nil + }) + } + profiles := testProfiles() + profile := profiles.byName["go"] + profile.Spec.MaxEnvironments = 1 + profiles.byName["go"] = profile + stores := []*Store{NewStore(dynamicClient, "ns", profiles, nil).WithKubeClient(kubes[0]), NewStore(dynamicClient, "ns", profiles, nil).WithKubeClient(kubes[1])} + results := make(chan error, 2) + for i := range stores { + go func(i int) { + _, err := stores[i].EnsurePending(context.Background(), "client", "owner", "binding-"+strconv.Itoa(i), "go", "fp-"+strconv.Itoa(i), "op-"+strconv.Itoa(i)) + results <- err + }(i) + } + admitted, exhausted := 0, 0 + for range 2 { + err := <-results + var controlled *executionenv.Error + if err == nil { + admitted++ + } else if errors.As(err, &controlled) && controlled.Code == executionenv.CodeResourceExhausted { + exhausted++ + } else { + t.Fatalf("unexpected result: %v", err) + } + } + if admitted != 1 || exhausted != 1 { + t.Fatalf("admitted=%d exhausted=%d", admitted, exhausted) + } +} + +func TestAcquireRunAndRevokeUseResourceVersionCAS(t *testing.T) { + now := time.Date(2026, 9, 17, 5, 0, 0, 0, time.UTC) + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns", "resourceVersion": "1"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": fenceHealthy, "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} + clients := []*dynamicfake.FakeDynamicClient{ + dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env), + dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env), + } + var mu sync.Mutex + current := env.DeepCopy() + firstReads := make(chan struct{}) + var reads, conflicts atomic.Int32 + for _, client := range clients { + client.PrependReactor("get", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + mu.Lock() + out := current.DeepCopy() + mu.Unlock() + if reads.Add(1) == 2 { + close(firstReads) + } + if reads.Load() <= 2 { + <-firstReads + } + return true, out, nil + }) + client.PrependReactor("update", "executionenvironments", func(action k8stesting.Action) (bool, runtime.Object, error) { + candidate := action.(k8stesting.UpdateAction).GetObject().(*unstructured.Unstructured) + mu.Lock() + defer mu.Unlock() + if candidate.GetResourceVersion() != current.GetResourceVersion() { + conflicts.Add(1) + return true, nil, apierrors.NewConflict(schema.GroupResource{Group: ExecutionEnvironmentGVR.Group, Resource: ExecutionEnvironmentGVR.Resource}, candidate.GetName(), errors.New("resource version changed")) + } + rv, _ := strconv.Atoi(current.GetResourceVersion()) + candidate = candidate.DeepCopy() + candidate.SetResourceVersion(strconv.Itoa(rv + 1)) + current = candidate + return true, candidate.DeepCopy(), nil + }) + } + first := NewStore(clients[0], "ns", testProfiles(), nil) + second := NewStore(clients[1], "ns", testProfiles(), nil) + first.now, second.now = func() time.Time { return now }, func() time.Time { return now } + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + results := make(chan error, 2) + go func() { + _, err := first.AcquireRun(context.Background(), ref, "client", "owner", "binding", "run", "acquire", executionenv.MinRunTTL) + results <- err + }() + go func() { + _, err := second.RevokeEnvironment(context.Background(), adminLifecycleRequest{Environment: ref, Client: "client", OwnerHash: "owner", OperationID: "revoke"}, 1) + results <- err + }() + for range 2 { + if err := <-results; err != nil { + t.Fatalf("CAS retry did not converge: %v", err) + } + } + if conflicts.Load() == 0 { + t.Fatal("test did not force a resource-version conflict") + } + mu.Lock() + final := current.DeepCopy() + mu.Unlock() + if generation := intNested(final.Object, "status", "grantGeneration"); generation != 2 { + t.Fatalf("grant generation=%d, want 2", generation) + } + if claim, _, _, ok := activeRunFrom(final); ok && generationMatches(final, claim.GrantGeneration) && claim.GrantGeneration != 2 { + t.Fatalf("stale claim remained current after revoke: %+v", claim) + } +} diff --git a/internal/adapter/executioncontroller/store_lifecycle.go b/internal/adapter/executioncontroller/store_lifecycle.go new file mode 100644 index 0000000000..a985c4f2d7 --- /dev/null +++ b/internal/adapter/executioncontroller/store_lifecycle.go @@ -0,0 +1,283 @@ +//nolint:revive // Private store methods implement adapter-only lifecycle interfaces. +package executioncontroller + +import ( + "context" + "sort" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +type referenceRecord struct { + BindingID string + State executionenv.ReferenceState + OperationID string + SourceBindingID string + CreatedAt time.Time +} + +func referenceRecords(o *unstructured.Unstructured) ([]referenceRecord, error) { + raw, found, err := unstructured.NestedSlice(o.Object, "status", "references") + if err != nil || !found { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment references are invalid"} + } + out := make([]referenceRecord, 0, len(raw)) + for _, item := range raw { + m, ok := item.(map[string]any) + if !ok { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "legacy environment references require explicit migration"} + } + created, parseErr := time.Parse(time.RFC3339Nano, text(m, "createdAt")) + r := referenceRecord{BindingID: text(m, "bindingID"), State: executionenv.ReferenceState(text(m, "state")), OperationID: text(m, operationIDField), SourceBindingID: text(m, "sourceBindingID"), CreatedAt: created} + if parseErr != nil || r.BindingID == "" || r.OperationID == "" || (r.State != executionenv.ReferencePendingCreate && r.State != executionenv.ReferencePublished && r.State != executionenv.ReferencePendingDelete) { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment references are invalid"} + } + out = append(out, r) + } + return out, nil +} + +func setReferenceRecords(o *unstructured.Unstructured, refs []referenceRecord) error { + sort.Slice(refs, func(i, j int) bool { return refs[i].BindingID < refs[j].BindingID }) + raw := make([]any, len(refs)) + for i, r := range refs { + raw[i] = map[string]any{"bindingID": r.BindingID, "state": string(r.State), operationIDField: r.OperationID, "sourceBindingID": r.SourceBindingID, "createdAt": r.CreatedAt.UTC().Format(time.RFC3339Nano)} + } + return unstructured.SetNestedSlice(o.Object, raw, "status", "references") +} + +func referenceOperationMatches(o *unstructured.Unstructured, binding, operation string) bool { + refs, err := referenceRecords(o) + if err != nil { + return false + } + for _, r := range refs { + if r.BindingID == binding { + return r.OperationID == operation && (r.State == executionenv.ReferencePendingCreate || r.State == executionenv.ReferencePublished) + } + } + return false +} + +func attachedReference(refs []referenceRecord, binding string) bool { + for _, r := range refs { + if r.BindingID == binding && (r.State == executionenv.ReferencePendingCreate || r.State == executionenv.ReferencePublished) { + return true + } + } + return false +} + +func publishedReference(refs []referenceRecord, binding string) bool { + for _, r := range refs { + if r.BindingID == binding && r.State == executionenv.ReferencePublished { + return true + } + } + return false +} + +func (s *Store) mutateReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string, from, to executionenv.ReferenceState, remove bool) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, err := referenceRecords(o) + if err != nil { + return err + } + for i := range refs { + if refs[i].BindingID != binding { + continue + } + if refs[i].OperationID == operation && refs[i].State == to { + return nil + } + if refs[i].OperationID == operation && refs[i].State == from { + if remove { + refs = append(refs[:i], refs[i+1:]...) + } else { + refs[i].State = to + } + return setReferenceRecords(o, refs) + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference transaction mismatch"} + } + if remove { + return nil + } + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference not found"} + }) +} + +func (s *Store) CommitReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.mutateReference(ctx, ref, client, owner, binding, operation, executionenv.ReferencePendingCreate, executionenv.ReferencePublished, false) +} +func (s *Store) AbortReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.mutateReference(ctx, ref, client, owner, binding, operation, executionenv.ReferencePendingCreate, "", true) +} +func (s *Store) PrepareReferenceDelete(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, err := referenceRecords(o) + if err != nil { + return err + } + for i := range refs { + if refs[i].BindingID != binding { + continue + } + if refs[i].State == executionenv.ReferencePendingDelete && refs[i].OperationID == operation { + return nil + } + if refs[i].State != executionenv.ReferencePublished { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference transaction mismatch"} + } + refs[i].State, refs[i].OperationID, refs[i].CreatedAt = executionenv.ReferencePendingDelete, operation, time.Now().UTC() + return setReferenceRecords(o, refs) + } + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference not found"} + }) +} +func (s *Store) ConfirmReferenceDelete(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.mutateReference(ctx, ref, client, owner, binding, operation, executionenv.ReferencePendingDelete, "", true) +} +func (s *Store) CancelReferenceDelete(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.mutateReference(ctx, ref, client, owner, binding, operation, executionenv.ReferencePendingDelete, executionenv.ReferencePublished, false) +} + +func (s *Store) ReserveSuccessor(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, source, destination, operation string) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, err := referenceRecords(o) + if err != nil { + return err + } + if !publishedReference(refs, source) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: "source reference not found"} + } + if textNested(o.Object, "status", "activeOperation", "id") != "" || textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not quiescent"} + } + for _, r := range refs { + if r.BindingID == destination { + if r.State == executionenv.ReferencePendingCreate && r.OperationID == operation && r.SourceBindingID == source { + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "destination reference conflicts"} + } + } + if len(refs) >= maxReferences { + return &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "environment reference limit reached"} + } + refs = append(refs, referenceRecord{BindingID: destination, State: executionenv.ReferencePendingCreate, OperationID: operation, SourceBindingID: source, CreatedAt: time.Now().UTC()}) + return setReferenceRecords(o, refs) + }) +} + +func (s *Store) FindReferenceIntent(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) (executionenv.ReferenceIntent, error) { + o, err := s.resources.Get(ctx, ref.ID, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return executionenv.ReferenceIntent{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference intent not found"} + } + if err != nil { + return executionenv.ReferenceIntent{}, err + } + if textNested(o.Object, "spec", "revision") != ref.Revision || textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) { + return executionenv.ReferenceIntent{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference intent not found"} + } + refs, err := referenceRecords(o) + if err != nil { + return executionenv.ReferenceIntent{}, err + } + for _, r := range refs { + if r.BindingID == binding && r.State != executionenv.ReferencePublished { + return executionenv.ReferenceIntent{Environment: ref, BindingID: r.BindingID, State: r.State, OperationID: r.OperationID, SourceBindingID: r.SourceBindingID, CreatedAt: r.CreatedAt}, nil + } + } + return executionenv.ReferenceIntent{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference intent not found"} +} + +func (s *Store) ListReferenceIntentsForClient(ctx context.Context, client string, limit int) ([]executionenv.ReferenceIntent, error) { + if limit <= 0 || limit > maxReferences { + limit = maxReferences + } + list, err := s.resources.List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + out := make([]executionenv.ReferenceIntent, 0) + for i := range list.Items { + o := &list.Items[i] + if textNested(o.Object, "spec", "clientHash") != hashText(client) { + continue + } + owner := executionenv.Owner{Issuer: textNested(o.Object, "spec", "ownerIssuer"), Subject: textNested(o.Object, "spec", "ownerSubject")} + if owner.Issuer == "" || owner.Subject == "" || ownerHash(owner) != textNested(o.Object, "spec", "ownerHash") { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference intent owner attestation is unavailable"} + } + refs, parseErr := referenceRecords(o) + if parseErr != nil { + return nil, parseErr + } + for _, r := range refs { + if r.State == executionenv.ReferencePublished { + continue + } + out = append(out, executionenv.ReferenceIntent{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: textNested(o.Object, "spec", "revision")}, Owner: owner, BindingID: r.BindingID, State: r.State, OperationID: r.OperationID, SourceBindingID: r.SourceBindingID, CreatedAt: r.CreatedAt}) + if len(out) == limit { + return out, nil + } + } + } + return out, nil +} + +func (s *Store) ListReferenceIntents(ctx context.Context, client, owner string, limit int) ([]executionenv.ReferenceIntent, error) { + if limit <= 0 || limit > maxReferences { + limit = maxReferences + } + list, err := s.resources.List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + out := make([]executionenv.ReferenceIntent, 0) + for i := range list.Items { + o := &list.Items[i] + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) { + continue + } + refs, parseErr := referenceRecords(o) + if parseErr != nil { + return nil, parseErr + } + for _, r := range refs { + if r.State == executionenv.ReferencePublished { + continue + } + out = append(out, executionenv.ReferenceIntent{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: textNested(o.Object, "spec", "revision")}, BindingID: r.BindingID, State: r.State, OperationID: r.OperationID, SourceBindingID: r.SourceBindingID, CreatedAt: r.CreatedAt}) + if len(out) == limit { + return out, nil + } + } + } + return out, nil +} diff --git a/internal/adapter/executioncontroller/store_lifecycle_test.go b/internal/adapter/executioncontroller/store_lifecycle_test.go new file mode 100644 index 0000000000..116844259b --- /dev/null +++ b/internal/adapter/executioncontroller/store_lifecycle_test.go @@ -0,0 +1,300 @@ +package executioncontroller + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func lifecycleEnvironment(now time.Time) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", + "metadata": map[string]any{"name": "env", "namespace": "ns"}, + "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, + "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "references": []any{ + map[string]any{"bindingID": "source", "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}, + }, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}, + }} +} + +func TestEnsurePendingOwnedPersistsOwnerAttestation(t *testing.T) { + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + store := NewStore(client, "ns", testProfiles(), nil) + owner := executionenv.Owner{Issuer: "https://issuer.example", Subject: "alice"} + allocation, err := store.EnsurePendingOwned(t.Context(), "client", ownerHash(owner), owner, "binding", "go", "fingerprint", "ensure-operation") + if err != nil { + t.Fatal(err) + } + env, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), allocation.Environment.ID, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(env.Object, "spec", "ownerIssuer") != owner.Issuer || textNested(env.Object, "spec", "ownerSubject") != owner.Subject { + t.Fatalf("owner attestation was not persisted: %v", env.Object["spec"]) + } +} + +func TestPublishedBindingReattachesWithoutNewEnsureOperation(t *testing.T) { + ctx := t.Context() + client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + store := NewStore(client, "ns", testProfiles(), nil) + owner := executionenv.Owner{Issuer: "https://issuer.example", Subject: "alice"} + allocation, err := store.EnsurePendingOwned(ctx, "client", ownerHash(owner), owner, "binding", "go", "fingerprint", "create-operation") + if err != nil { + t.Fatal(err) + } + ref := allocation.Environment + if err := store.CommitReference(ctx, ref, "client", ownerHash(owner), "binding", "create-operation"); err != nil { + t.Fatal(err) + } + if err := store.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + setConditionObject(o, "Ready", true, "Ready", "fixture ready") + return nil + }); err != nil { + t.Fatal(err) + } + resources := client.Resource(ExecutionEnvironmentGVR).Namespace("ns") + before, err := resources.List(ctx, metav1.ListOptions{}) + if err != nil || len(before.Items) != 1 { + t.Fatalf("allocation list: %v, err=%v", before, err) + } + _, err = store.EnsurePendingOwned(ctx, "client", ownerHash(owner), owner, "binding", "go", "fingerprint", "qualification-reattach") + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("changed Ensure operation used as lookup: err=%v", err) + } + lookup := executionenv.EnvironmentRef{ID: before.Items[0].GetName(), Revision: textNested(before.Items[0].Object, "spec", "revision")} + attached, err := store.Attach(ctx, lookup, "client", ownerHash(owner), "binding") + if err != nil || !attached.Ready || attached.Environment != ref || attached.BindingID != "binding" || attached.OwnerHash != ownerHash(owner) { + t.Fatalf("exact published binding did not reattach: %+v, err=%v", attached, err) + } + for _, tc := range []struct { + name string + ref executionenv.EnvironmentRef + owner string + binding string + }{ + {"wrong owner", lookup, "other-owner", "binding"}, + {"wrong session", lookup, ownerHash(owner), "other-binding"}, + {"wrong revision", executionenv.EnvironmentRef{ID: lookup.ID, Revision: "other-revision"}, ownerHash(owner), "binding"}, + {"missing reference", executionenv.EnvironmentRef{}, ownerHash(owner), "binding"}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := store.Attach(ctx, tc.ref, "client", tc.owner, tc.binding); err == nil { + t.Fatal("inexact lookup was accepted") + } + }) + } + after, err := resources.List(ctx, metav1.ListOptions{}) + if err != nil || !reflect.DeepEqual(before.Items, after.Items) { + t.Fatalf("lookup changed allocations or published identity: err=%v", err) + } +} + +func TestRunClaimIsEnvironmentWideAndExact(t *testing.T) { + now := time.Now().UTC() + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), lifecycleEnvironment(now)) + store := NewStore(client, "ns", testProfiles(), nil) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + first, err := store.AcquireRun(t.Context(), ref, "client", "owner", "source", "run-a", "op-a", time.Minute) + if err != nil { + t.Fatal(err) + } + if first.Epoch != 2 || first.ClaimID == "" || first.GrantGeneration != 1 { + t.Fatalf("claim=%+v", first) + } + repeat, err := store.AcquireRun(t.Context(), ref, "client", "owner", "source", "run-a", "op-a", time.Minute) + if err != nil || repeat.ClaimID != first.ClaimID || repeat.Epoch != first.Epoch { + t.Fatalf("repeat=%+v err=%v", repeat, err) + } + if _, err := store.AcquireRun(t.Context(), ref, "client", "owner", "source", "run-b", "op-b", time.Minute); err == nil { + t.Fatal("overlapping claim admitted") + } + stale := executionenv.RunClaimRequest{Environment: ref, BindingID: "source", RunID: "run-a", ClaimID: "wrong", Epoch: first.Epoch, GrantGeneration: first.GrantGeneration, OperationID: "release", TTL: time.Minute} + if err := store.ReleaseRun(t.Context(), ref, "client", "owner", stale); err == nil { + t.Fatal("stale release admitted") + } + stale.ClaimID = first.ClaimID + if err := store.ReleaseRun(t.Context(), ref, "client", "owner", stale); err != nil { + t.Fatal(err) + } + second, err := store.AcquireRun(t.Context(), ref, "client", "owner", "source", "run-b", "op-b", time.Minute) + if err != nil || second.Epoch != first.Epoch+1 { + t.Fatalf("second=%+v err=%v", second, err) + } +} + +func TestExpiredClaimWithActiveOperationCannotBeReplaced(t *testing.T) { + now := time.Now().UTC() + env := lifecycleEnvironment(now) + statusMap, _, _ := unstructured.NestedMap(env.Object, "status") + statusMap["epoch"] = int64(4) + statusMap["activeRun"] = map[string]any{"bindingID": "source", "runID": "old", "claimID": "old-claim", "operationID": "old-acquire", "epoch": int64(4), "grantGeneration": int64(1), "expiresAt": now.Add(-time.Minute).Format(time.RFC3339Nano)} + statusMap["activeOperation"] = map[string]any{"id": "still-running", "claimID": "old-claim", "runID": "old", "epoch": int64(4)} + env.Object["status"] = statusMap + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + _, err := store.AcquireRun(t.Context(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "source", "new", "new-op", time.Minute) + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeFenceUnknown { + t.Fatalf("error=%v", err) + } +} + +func TestExactReferenceIntentLookupIsNotTruncatedByGlobalLimit(t *testing.T) { + now := time.Now().UTC() + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + objects := make([]runtime.Object, 0, 67) + for i := 0; i < 66; i++ { + env := lifecycleEnvironment(now) + env.SetName(fmt.Sprintf("env-%02d", i)) + _ = unstructured.SetNestedField(env.Object, fmt.Sprintf("rev-%02d", i), "spec", "revision") + _ = unstructured.SetNestedField(env.Object, ownerHash(owner), "spec", "ownerHash") + _ = unstructured.SetNestedField(env.Object, owner.Issuer, "spec", "ownerIssuer") + _ = unstructured.SetNestedField(env.Object, owner.Subject, "spec", "ownerSubject") + if err := setReferenceRecords(env, []referenceRecord{{BindingID: fmt.Sprintf("binding-%02d", i), State: executionenv.ReferencePendingDelete, OperationID: fmt.Sprintf("delete-%02d", i), CreatedAt: now}}); err != nil { + t.Fatal(err) + } + objects = append(objects, env) + } + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), objects...) + store := NewStore(client, "ns", testProfiles(), nil) + target := executionenv.EnvironmentRef{ID: "env-65", Revision: "rev-65"} + intent, err := store.FindReferenceIntent(t.Context(), target, "client", ownerHash(owner), "binding-65") + if err != nil || intent.OperationID != "delete-65" { + t.Fatalf("exact target beyond global list limit: intent=%+v err=%v", intent, err) + } + if _, err := store.FindReferenceIntent(t.Context(), target, "other-client", ownerHash(owner), "binding-65"); err == nil { + t.Fatal("other client enumerated exact intent") + } + if _, err := store.FindReferenceIntent(t.Context(), target, "client", ownerHash(executionenv.Owner{Issuer: "issuer", Subject: "mallory"}), "binding-65"); err == nil { + t.Fatal("other owner enumerated exact intent") + } +} + +func TestClientScopedIntentReconciliationRejectsEachOwnerMutationWithoutSideEffects(t *testing.T) { + now := time.Now().UTC() + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + mutations := map[string]func(*unstructured.Unstructured){ + "owner hash": func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, hashText("other-owner"), "spec", "ownerHash") + }, + "owner issuer": func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, "other-issuer", "spec", "ownerIssuer") + }, + "owner subject": func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, "mallory", "spec", "ownerSubject") + }, + "client binding": func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, hashText("other-client"), "spec", "clientHash") + }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + env := lifecycleEnvironment(now) + _ = unstructured.SetNestedField(env.Object, ownerHash(owner), "spec", "ownerHash") + _ = unstructured.SetNestedField(env.Object, owner.Issuer, "spec", "ownerIssuer") + _ = unstructured.SetNestedField(env.Object, owner.Subject, "spec", "ownerSubject") + refs, err := referenceRecords(env) + if err != nil { + t.Fatal(err) + } + refs = append(refs, referenceRecord{BindingID: "pending", State: executionenv.ReferencePendingDelete, OperationID: "delete", CreatedAt: now}) + if err := setReferenceRecords(env, refs); err != nil { + t.Fatal(err) + } + mutate(env) + before := env.DeepCopy() + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + intents, listErr := store.ListReferenceIntentsForClient(t.Context(), "client", 64) + if listErr == nil && len(intents) != 0 { + t.Fatalf("mutated identity exposed intents: %+v", intents) + } + after, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before.Object, after.Object) { + t.Fatal("negative reconciliation mutated durable state") + } + }) + } +} + +func TestClientScopedIntentListReturnsAttestedOwnerOnlyToOwningClient(t *testing.T) { + now := time.Now().UTC() + env := lifecycleEnvironment(now) + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + if err := unstructured.SetNestedField(env.Object, ownerHash(owner), "spec", "ownerHash"); err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(env.Object, owner.Issuer, "spec", "ownerIssuer"); err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(env.Object, owner.Subject, "spec", "ownerSubject"); err != nil { + t.Fatal(err) + } + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + if err := store.ReserveSuccessor(t.Context(), ref, "client", ownerHash(owner), "source", "destination", "fork-op"); err != nil { + t.Fatal(err) + } + intents, err := store.ListReferenceIntentsForClient(t.Context(), "client", 64) + if err != nil || len(intents) != 1 || intents[0].Owner != owner || intents[0].BindingID != "destination" { + t.Fatalf("intents=%+v err=%v", intents, err) + } + other, err := store.ListReferenceIntentsForClient(t.Context(), "other-client", 64) + if err != nil || len(other) != 0 { + t.Fatalf("other-client intents=%+v err=%v", other, err) + } +} + +func TestReferenceTransactionsRetainUnknownAndNeverChangeSource(t *testing.T) { + now := time.Now().UTC() + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), lifecycleEnvironment(now)) + store := NewStore(client, "ns", testProfiles(), nil) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + if err := store.ReserveSuccessor(t.Context(), ref, "client", "owner", "source", "destination", "fork-op"); err != nil { + t.Fatal(err) + } + if err := store.ReserveSuccessor(t.Context(), ref, "client", "owner", "source", "destination", "fork-op"); err != nil { + t.Fatal(err) + } + intents, err := store.ListReferenceIntents(t.Context(), "client", "owner", 64) + if err != nil || len(intents) != 1 || intents[0].BindingID != "destination" || intents[0].State != executionenv.ReferencePendingCreate { + t.Fatalf("intents=%+v err=%v", intents, err) + } + if err := store.CommitReference(t.Context(), ref, "client", "owner", "destination", "fork-op"); err != nil { + t.Fatal(err) + } + if err := store.PrepareReferenceDelete(t.Context(), ref, "client", "owner", "destination", "delete-op"); err != nil { + t.Fatal(err) + } + if err := store.CancelReferenceDelete(t.Context(), ref, "client", "owner", "destination", "delete-op"); err != nil { + t.Fatal(err) + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + refs, err := referenceRecords(got) + if err != nil { + t.Fatal(err) + } + if !publishedReference(refs, "source") || !publishedReference(refs, "destination") { + t.Fatalf("references=%+v", refs) + } +} diff --git a/internal/adapter/executioncontroller/store_revoke.go b/internal/adapter/executioncontroller/store_revoke.go new file mode 100644 index 0000000000..88e73db467 --- /dev/null +++ b/internal/adapter/executioncontroller/store_revoke.go @@ -0,0 +1,63 @@ +//nolint:revive // Private store methods implement adapter-only lifecycle interfaces. +package executioncontroller + +import ( + "context" + "fmt" + "math" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// RevokeEnvironment atomically invalidates every issued grant without claiming +// that an already-dispatched executor operation has stopped. +func (s *Store) RevokeEnvironment(ctx context.Context, q adminLifecycleRequest, expected uint64) (uint64, error) { + ref, client, owner, operationID := q.Environment, q.Client, q.OwnerHash, q.OperationID + if expected == 0 || expected >= math.MaxInt64 || operationID == "" || len(operationID) > executionenv.MaxIdentityBytes { + return 0, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "grant generation or operation identity is invalid"} + } + requestFingerprint := fingerprint(ref.ID, ref.Revision, client, owner, fmt.Sprint(expected)) + var next uint64 + err := s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { + receipts, found, err := unstructured.NestedSlice(o.Object, "status", "revocationReceipts") + if err != nil { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "revocation receipt ledger is invalid"} + } + if found { + for _, raw := range receipts { + receipt, ok := raw.(map[string]any) + if !ok { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "revocation receipt ledger is invalid"} + } + if text(receipt, operationIDField) != operationID { + continue + } + if text(receipt, "fingerprint") != requestFingerprint { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "operation identity was reused for a different revocation"} + } + generation, ok := receipt["grantGeneration"].(int64) + if !ok || generation <= 0 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "revocation receipt ledger is invalid"} + } + next = uint64(generation) + return nil + } + } + current := intNested(o.Object, "status", "grantGeneration") + if current <= 0 || uint64(current) != expected { //nolint:gosec // positivity is checked before conversion. + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "grant generation changed concurrently"} + } + next = expected + 1 + receipts = append(receipts, map[string]any{operationIDField: operationID, "fingerprint": requestFingerprint, "expectedGrantGeneration": int64(expected), "grantGeneration": int64(next)}) //nolint:gosec // values are bounded above. + if len(receipts) > 32 { + receipts = receipts[len(receipts)-32:] + } + if err := unstructured.SetNestedSlice(o.Object, receipts, "status", "revocationReceipts"); err != nil { + return err + } + return unstructured.SetNestedField(o.Object, int64(next), "status", "grantGeneration") //nolint:gosec // expected is bounded below MaxInt64. + }) + return next, err +} diff --git a/internal/adapter/executioncontroller/store_revoke_test.go b/internal/adapter/executioncontroller/store_revoke_test.go new file mode 100644 index 0000000000..b83500af38 --- /dev/null +++ b/internal/adapter/executioncontroller/store_revoke_test.go @@ -0,0 +1,81 @@ +package executioncontroller + +import ( + "context" + "errors" + "math" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestRevokeEnvironmentFencesOldClaimWithoutChangingExecutionEpoch(t *testing.T) { + env := runFixtureEnvironment(7, 3) + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), &terminalErrorExecutor{}) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + next, err := store.RevokeEnvironment(t.Context(), adminLifecycleRequest{Environment: ref, Client: "client", OwnerHash: "owner", OperationID: "revoke-1"}, 3) + if err != nil || next != 4 { + t.Fatalf("revoke=(%d,%v)", next, err) + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if intNested(got.Object, "status", "epoch") != 7 || intNested(got.Object, "status", "grantGeneration") != 4 { + t.Fatalf("status=%v", got.Object["status"]) + } + otherReplica := NewStore(client, "ns", testProfiles(), &terminalErrorExecutor{}) + _, err = otherReplica.File(t.Context(), "client", "owner", executionenv.FileRequest{Context: executionenv.RequestContext{Environment: ref, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3}, Operation: executionenv.OpFileRead, Path: "x"}) + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("old grant error=%v", err) + } + if _, err := store.RenewRun(t.Context(), ref, "client", "owner", executionenv.RunClaimRequest{Environment: ref, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3, OperationID: "renew", TTL: time.Minute}); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("old renewal error=%v", err) + } + replayed, err := store.RevokeEnvironment(t.Context(), adminLifecycleRequest{Environment: ref, Client: "client", OwnerHash: "owner", OperationID: "revoke-1"}, 3) + if err != nil || replayed != 4 { + t.Fatalf("replayed revoke=(%d,%v)", replayed, err) + } + if _, err := store.RevokeEnvironment(t.Context(), adminLifecycleRequest{Environment: ref, Client: "client", OwnerHash: "owner", OperationID: "revoke-2"}, 3); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("stale CAS error=%v", err) + } +} + +func TestRevokeEnvironmentOverflowFailsClosed(t *testing.T) { + env := runFixtureEnvironment(1, math.MaxInt64) + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + if _, err := store.RevokeEnvironment(context.Background(), adminLifecycleRequest{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, Client: "client", OwnerHash: "owner", OperationID: "overflow"}, math.MaxInt64); err == nil { + t.Fatal("overflow accepted") + } +} + +func TestProfileEnvironmentAdmissionLimit(t *testing.T) { + env := lifecycleEnvironment(time.Now().UTC()) + client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}, env) + profiles := testProfiles() + profile := profiles.byName["go"] + profile.Spec.MaxEnvironments = 1 + profiles.byName["go"] = profile + store := NewStore(client, "ns", profiles, nil).WithKubeClient(kubernetesfake.NewSimpleClientset()) + _, err := store.Ensure(t.Context(), "client", "owner", "new-binding", "go", "new-fingerprint") + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeResourceExhausted { + t.Fatalf("limit error=%v", err) + } +} + +func runFixtureEnvironment(epoch, generation uint64) *unstructured.Unstructured { + now := time.Now().UTC() + return &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(epoch), "grantGeneration": int64(generation), "fenceState": "Healthy", "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "epoch": int64(epoch), "grantGeneration": int64(generation), "expiresAt": now.Add(time.Minute).Format(time.RFC3339Nano)}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} +} diff --git a/internal/adapter/executioncontroller/store_run.go b/internal/adapter/executioncontroller/store_run.go new file mode 100644 index 0000000000..beb2cee30d --- /dev/null +++ b/internal/adapter/executioncontroller/store_run.go @@ -0,0 +1,226 @@ +//nolint:revive // Private store methods implement adapter-only lifecycle interfaces. +package executioncontroller + +import ( + "context" + "fmt" + "math" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func boundedRunTTL(ttl time.Duration) (time.Duration, error) { + if ttl < executionenv.MinRunTTL || ttl > executionenv.MaxRunTTL { + return 0, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "run claim ttl is outside the operator bound"} + } + return ttl, nil +} + +func generationMatches(o *unstructured.Unstructured, generation uint64) bool { + return generation > 0 && generation <= math.MaxInt64 && intNested(o.Object, "status", "grantGeneration") == int64(generation) +} + +// ValidateRunClaim rechecks durable generation and ownership before any grant-authorized operation. +func (s *Store) ValidateRunClaim(ctx context.Context, client, owner string, rc executionenv.RequestContext) error { + if rc.Epoch == 0 || rc.Epoch > math.MaxInt64 || rc.GrantGeneration == 0 || rc.GrantGeneration > math.MaxInt64 { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "run claim identity is invalid"} + } + o, err := s.resources.Get(ctx, rc.Environment.ID, metav1.GetOptions{}) + if err != nil { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if err := requireCurrentSchema(o); err != nil { + return err + } + claim, _, expiry, ok := activeRunFrom(o) + if !ok || !s.now().Before(expiry) || claim.Environment != rc.Environment || claim.BindingID != rc.BindingID || claim.RunID != rc.RunID || claim.ClaimID != rc.ClaimID || claim.Epoch != rc.Epoch || claim.GrantGeneration != rc.GrantGeneration || !generationMatches(o, rc.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim is not current"} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + return nil +} + +func activeRunFrom(o *unstructured.Unstructured) (executionenv.RunClaim, string, time.Time, bool) { + m, found, _ := unstructured.NestedMap(o.Object, "status", "activeRun") + if !found { + return executionenv.RunClaim{}, "", time.Time{}, false + } + expires, err := time.Parse(time.RFC3339Nano, text(m, "expiresAt")) + epoch, epochOK := epochValue(o) + generationValue := intNested(m, "grantGeneration") + if generationValue <= 0 { + return executionenv.RunClaim{}, "", time.Time{}, false + } + generation := uint64(generationValue) //nolint:gosec // positivity is checked immediately above. + claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: textNested(o.Object, "spec", "revision")}, BindingID: text(m, "bindingID"), RunID: text(m, "runID"), ClaimID: text(m, "claimID"), Epoch: epoch, GrantGeneration: generation, ExpiresAt: expires} + return claim, text(m, operationIDField), expires, err == nil && epochOK && generation > 0 && claim.BindingID != "" && claim.RunID != "" && claim.ClaimID != "" +} + +//nolint:gocyclo // Run ownership admission keeps every CAS precondition in one auditable transition. +func (s *Store) AcquireRun(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, runID, operationID string, ttl time.Duration) (executionenv.RunClaim, error) { + ttl, err := boundedRunTTL(ttl) + if err != nil { + return executionenv.RunClaim{}, err + } + var out executionenv.RunClaim + err = s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, parseErr := referenceRecords(o) + if parseErr != nil { + return parseErr + } + if !publishedReference(refs, binding) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: "published reference not found"} + } + if textNested(o.Object, "spec", "desired") != "Active" || !conditionTrue(o, "Ready") || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment is not ready"} + } + if current, currentOp, expires, found := activeRunFrom(o); found { + if current.BindingID == binding && current.RunID == runID && currentOp == operationID { + if !generationMatches(o, current.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim was revoked"} + } + out = current + return nil + } + if s.now().Before(expires) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment has an active run"} + } + if textNested(o.Object, "status", "activeOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "expired run still has an active operation"} + } + } + if textNested(o.Object, "status", "activeOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment has an active operation"} + } + epoch, ok := epochValue(o) + if !ok || epoch >= math.MaxInt64 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment epoch is invalid"} + } + epoch++ + epochStatus := int64(epoch) //nolint:gosec // the persisted epoch was checked below MaxInt64 before increment. + claimID, idErr := randomID() + if idErr != nil { + return idErr + } + expires := s.now().Add(ttl) + generation := intNested(o.Object, "status", "grantGeneration") + if generation <= 0 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "grant generation is invalid"} + } + if setErr := unstructured.SetNestedField(o.Object, epochStatus, "status", "epoch"); setErr != nil { + return setErr + } + if setErr := unstructured.SetNestedMap(o.Object, map[string]any{"bindingID": binding, "runID": runID, "claimID": claimID, operationIDField: operationID, "ownerHash": owner, "clientHash": hashText(client), "epoch": epochStatus, "grantGeneration": generation, "expiresAt": expires.Format(time.RFC3339Nano)}, "status", "activeRun"); setErr != nil { + return setErr + } + out = executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: runID, ClaimID: claimID, Epoch: epoch, GrantGeneration: uint64(generation), ExpiresAt: expires} + return nil + }) + return out, err +} + +func renewFingerprint(ref executionenv.EnvironmentRef, client, owner string, req executionenv.RunClaimRequest) string { + return fingerprint(ref.ID, ref.Revision, client, owner, req.BindingID, req.RunID, req.ClaimID, fmt.Sprint(req.Epoch), fmt.Sprint(req.GrantGeneration), req.TTL.String()) +} + +func replayRenewReceipt(o *unstructured.Unstructured, operationID, requestFingerprint string, now time.Time, claim executionenv.RunClaim) (executionenv.RunClaim, bool, error) { + receipts, _, err := unstructured.NestedSlice(o.Object, "status", "renewReceipts") + if err != nil { + return executionenv.RunClaim{}, false, &executionenv.Error{Code: executionenv.CodeConflict, Message: "run renewal receipts are invalid"} + } + for _, raw := range receipts { + receipt, ok := raw.(map[string]any) + if !ok || text(receipt, operationIDField) != operationID { + continue + } + if text(receipt, "fingerprint") != requestFingerprint { + return executionenv.RunClaim{}, true, &executionenv.Error{Code: executionenv.CodeConflict, Message: "run renewal operation conflicts"} + } + expires, parseErr := time.Parse(time.RFC3339Nano, text(receipt, "expiresAt")) + if parseErr != nil || !now.Before(expires) { + return executionenv.RunClaim{}, true, &executionenv.Error{Code: executionenv.CodeConflict, Message: "run renewal receipt is expired"} + } + claim.ExpiresAt = expires + return claim, true, nil + } + return executionenv.RunClaim{}, false, nil +} + +func appendRenewReceipt(o *unstructured.Unstructured, operationID, requestFingerprint string, expires time.Time) error { + receipts, _, err := unstructured.NestedSlice(o.Object, "status", "renewReceipts") + if err != nil { + return err + } + receipts = append(receipts, map[string]any{operationIDField: operationID, "fingerprint": requestFingerprint, "expiresAt": expires.Format(time.RFC3339Nano)}) + if len(receipts) > 32 { + receipts = receipts[len(receipts)-32:] + } + return unstructured.SetNestedSlice(o.Object, receipts, "status", "renewReceipts") +} + +func (s *Store) RenewRun(ctx context.Context, ref executionenv.EnvironmentRef, client, owner string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + ttl, err := boundedRunTTL(req.TTL) + if err != nil { + return executionenv.RunClaim{}, err + } + requestFingerprint := renewFingerprint(ref, client, owner, req) + var out executionenv.RunClaim + err = s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + cur, _, expiry, ok := activeRunFrom(o) + if !ok || textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || cur.Environment != ref || cur.BindingID != req.BindingID || cur.RunID != req.RunID || cur.ClaimID != req.ClaimID || cur.Epoch != req.Epoch || cur.GrantGeneration != req.GrantGeneration || !generationMatches(o, req.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim mismatch"} + } + if replay, found, replayErr := replayRenewReceipt(o, req.OperationID, requestFingerprint, s.now(), cur); found { + out = replay + return replayErr + } + if !s.now().Before(expiry) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim mismatch"} + } + expires := s.now().Add(ttl) + m, _, _ := unstructured.NestedMap(o.Object, "status", "activeRun") + m["expiresAt"] = expires.Format(time.RFC3339Nano) + if setErr := unstructured.SetNestedMap(o.Object, m, "status", "activeRun"); setErr != nil { + return setErr + } + if setErr := appendRenewReceipt(o, req.OperationID, requestFingerprint, expires); setErr != nil { + return setErr + } + cur.ExpiresAt = expires + out = cur + return nil + }) + return out, err +} + +func (s *Store) ReleaseRun(ctx context.Context, ref executionenv.EnvironmentRef, client, owner string, req executionenv.RunClaimRequest) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + cur, _, _, ok := activeRunFrom(o) + if !ok { + return nil + } + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || cur.Environment != ref || cur.BindingID != req.BindingID || cur.RunID != req.RunID || cur.ClaimID != req.ClaimID || cur.Epoch != req.Epoch || cur.GrantGeneration != req.GrantGeneration || !generationMatches(o, req.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim mismatch"} + } + if textNested(o.Object, "status", "activeOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "run has an active operation"} + } + unstructured.RemoveNestedField(o.Object, "status", "activeRun") + return nil + }) +} diff --git a/internal/adapter/executioncontroller/store_test.go b/internal/adapter/executioncontroller/store_test.go new file mode 100644 index 0000000000..c7f227db55 --- /dev/null +++ b/internal/adapter/executioncontroller/store_test.go @@ -0,0 +1,165 @@ +package executioncontroller + +import ( + "context" + "errors" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestRenewRunRejectsExpiredClaimAtBoundary(t *testing.T) { + now := time.Date(2026, 9, 17, 4, 0, 0, 0, time.UTC) + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(7), "grantGeneration": int64(3), "fenceState": fenceHealthy, "references": []any{}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "epoch": int64(7), "grantGeneration": int64(3), "expiresAt": now.Format(time.RFC3339Nano)}}}} + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + store.now = func() time.Time { return now } + req := executionenv.RunClaimRequest{BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3, OperationID: "renew", TTL: executionenv.MinRunTTL} + if _, err := store.RenewRun(t.Context(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", req); err == nil { + t.Fatal("renewal at the expiry boundary resurrected the claim") + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if expiry := textNested(got.Object, "status", "activeRun", "expiresAt"); expiry != now.Format(time.RFC3339Nano) { + t.Fatalf("expired claim changed to %q", expiry) + } +} + +func TestRenewRunReplaysExactOperationWithoutExtendingLease(t *testing.T) { + now := time.Date(2026, 9, 17, 4, 0, 0, 0, time.UTC) + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(7), "grantGeneration": int64(3), "fenceState": fenceHealthy, "references": []any{}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "epoch": int64(7), "grantGeneration": int64(3), "expiresAt": now.Add(time.Minute).Format(time.RFC3339Nano)}}}} + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + clock := now + store.now = func() time.Time { return clock } + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + req := executionenv.RunClaimRequest{BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3, OperationID: "renew", TTL: executionenv.MinRunTTL} + first, err := store.RenewRun(t.Context(), ref, "client", "owner", req) + if err != nil { + t.Fatal(err) + } + clock = clock.Add(10 * time.Second) + replay, err := store.RenewRun(t.Context(), ref, "client", "owner", req) + if err != nil || !replay.ExpiresAt.Equal(first.ExpiresAt) { + t.Fatalf("replay extended or rejected lease: first=%v replay=%v err=%v", first.ExpiresAt, replay.ExpiresAt, err) + } + req.TTL = time.Minute + if _, err := store.RenewRun(t.Context(), ref, "client", "owner", req); err == nil { + t.Fatal("same operation accepted conflicting inputs") + } + clock = first.ExpiresAt + req.TTL = executionenv.MinRunTTL + if _, err := store.RenewRun(t.Context(), ref, "client", "owner", req); err == nil { + t.Fatal("expired renewal receipt resurrected claim") + } +} + +func TestStoreEnsureUsesStableLookupAndRejectsFingerprintDrift(t *testing.T) { + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + profiles := &Profiles{byName: map[string]resolvedProfile{"go": {Digest: "sha256:profile", Spec: ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100}}, "other": {Digest: "sha256:other", Spec: ProfileSpec{Image: "example@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100}}}} + s := NewStore(client, "ns", profiles, nil) + ctx := context.Background() + a, err := s.Ensure(ctx, "spiffe://client", "owner", "binding", "go", "fp1") + if err != nil { + t.Fatal(err) + } + b, err := s.Ensure(ctx, "spiffe://client", "owner", "binding", "go", "fp1") + if err != nil { + t.Fatal(err) + } + if a.Environment != b.Environment || a.Epoch != b.Epoch { + t.Fatalf("identity changed: %+v %+v", a, b) + } + if _, err := s.Ensure(ctx, "spiffe://client", "owner", "binding", "other", "fp2"); err == nil { + t.Fatal("profile drift adopted existing allocation") + } +} +func TestAllocationNameExcludesMutableProfile(t *testing.T) { + a := allocationName("client", "owner", "binding") + b := allocationName("client", "owner", "binding") + if a != b { + t.Fatal("unstable allocation name") + } +} + +type terminalErrorExecutor struct{ called int } + +type blockingExecutor struct { + started chan struct{} + release chan struct{} +} + +func (e *blockingExecutor) Execute(context.Context, string, executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + close(e.started) + <-e.release + return executionenv.ExecutorResponse{}, nil +} + +func (e *terminalErrorExecutor) Execute(context.Context, string, executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + e.called++ + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "path not found"} +} +func TestCancelledStoreOperationStaysActiveUntilBackendStopsThenFences(t *testing.T) { + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "epoch": int64(1), "grantGeneration": int64(1), "expiresAt": time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + exec := &blockingExecutor{started: make(chan struct{}), release: make(chan struct{})} + s := NewStore(client, "ns", testProfiles(), exec) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := s.StartCommand(ctx, "client", "owner", executionenv.CommandStartRequest{Context: executionenv.RequestContext{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1}, Command: "sleep 10"}) + done <- err + }() + <-exec.started + cancel() + active, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(active.Object, "status", "activeOperation", "id") == "" { + t.Fatal("active operation was released while executor was still live") + } + close(exec.release) + var pe *executionenv.Error + if err := <-done; !errors.As(err, &pe) || pe.Code != executionenv.CodeFenceUnknown { + t.Fatalf("completion error=%v", err) + } + fenced, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(fenced.Object, "status", "activeOperation", "id") == "" || textNested(fenced.Object, "status", "fenceState") != "FenceUnknown" { + t.Fatalf("uncertain operation identity was not retained: status=%v", fenced.Object["status"]) + } + _, err = s.Attach(context.Background(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "binding") + if !errors.As(err, &pe) || pe.Code != executionenv.CodeNotReady { + t.Fatalf("attach after fencing error=%v", err) + } +} + +func TestControlledExecutorErrorClearsOperationWithoutFencing(t *testing.T) { + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "epoch": int64(1), "grantGeneration": int64(1), "expiresAt": time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + exec := &terminalErrorExecutor{} + s := NewStore(client, "ns", testProfiles(), exec) + _, err := s.File(context.Background(), "client", "owner", executionenv.FileRequest{Context: executionenv.RequestContext{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1}, Operation: executionenv.OpFileRead, Path: "missing"}) + var pe *executionenv.Error + if !errors.As(err, &pe) || pe.Code != executionenv.CodeNotFound { + t.Fatalf("error=%v", err) + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if exec.called != 1 || textNested(got.Object, "status", "activeOperation", "id") != "" || textNested(got.Object, "status", "fenceState") != "Healthy" { + t.Fatalf("status=%v calls=%d", got.Object["status"], exec.called) + } +} diff --git a/internal/adapter/server/acp_placement.go b/internal/adapter/server/acp_placement.go index 5c95b60bac..7ae85573f3 100644 --- a/internal/adapter/server/acp_placement.go +++ b/internal/adapter/server/acp_placement.go @@ -59,7 +59,7 @@ func (s *Service) LoadACPSession(ctx context.Context, id session.SessionID, cwd if !persisted.EnvironmentRef.Valid() { return nil, fmt.Errorf("%w: session has no exact placement", ErrFailedPrecondition) } - binding, err := s.ReattachPlacement(ctx, persisted.EnvironmentRef) + binding, err := s.ReattachPlacementForBinding(ctx, persisted.EnvironmentRef, persisted.ID) if err != nil { return nil, err } diff --git a/internal/adapter/server/classification.go b/internal/adapter/server/classification.go index 91f30b0287..ba03490ab3 100644 --- a/internal/adapter/server/classification.go +++ b/internal/adapter/server/classification.go @@ -250,44 +250,45 @@ func requireCallerOwnedContext(surface string, t reflect.Type, table map[string] // TestInvariant_owned_access_is_classified fails, naming the method. var serviceAccessTable = map[string]ClassificationEntry{ // --- caller-owned: sessions --- - "CreateSession": {KindCallerOwned, "binds the verified context principal as owner atomically with visibility (reserveSessionID)"}, - "CreateSessionWithProvider": {KindCallerOwned, "delegates to CreateSessionWithProfile's atomic owner bind"}, - "CreateSessionWithProfile": {KindCallerOwned, "atomic owner bind at creation (reserveSessionID); ForkSession/carryover sources are authorized via authorizeSession before copying history"}, - "CreateSessionWithMCP": {KindCallerOwned, "delegates to CreateSessionWithProfile's atomic owner bind"}, - "CreateACPSession": {KindCallerOwned, "binds the trusted composition default before treating ACP cwd solely as an equality assertion"}, - "LoadACPSession": {KindCallerOwned, "owner-authorizes and reattaches the exact persisted placement before checking ACP cwd or mounting client MCP"}, - "BindPlacement": {KindCallerOwned, "passes the verified context principal and trusted composition scope to the provider's single atomic authorization-and-resolution operation"}, - "ReattachPlacement": {KindCallerOwned, "passes the exact persisted ref, verified context principal, and trusted scope to the provider without falling back to Bind"}, - "ReattachPlacementInScope": {KindCallerOwned, "requires the durable schedule scope to equal trusted composition scope, then reauthorizes the exact persisted ref as the verified owner without default fallback"}, - "ListCommandsForSession": {KindCallerOwned, "loads and owner-authorizes the source session before exact placement reattachment and command discovery"}, - "ListWorktreesForSession": {KindCallerOwned, "loads and owner-authorizes the source session before exact placement reattachment and scoped selector issuance"}, - "ClearSessionSuccessor": {KindCallerOwned, "serializes and leases an owner-authorized source before atomically publishing an empty-history placed successor"}, - "ForkSessionSuccessor": {KindCallerOwned, "serializes and leases an owner-authorized source before atomically publishing a history-carrying placed successor"}, - "GetSession": {KindCallerOwned, "authorizeSession: owner mismatch or absence both return ErrNotFound"}, - "WithAuthorizedSession": {KindCallerOwned, "ownership preflight excludes foreign lock contention; authoritative reload under runEntryMu precedes the caller-owned effect"}, - "GetTranscript": {KindCallerOwned, "one SessionStore.Load followed by authorizeSession; no run-entry side effects"}, - "LoadSession": {KindCallerOwned, "authorizeSession before rehydration"}, - "LoadSessionWithMCP": {KindCallerOwned, "delegates to LoadSession's authorizeSession before mounting client MCP"}, - "SetMode": {KindCallerOwned, "authorizes via GetSession before changing the session's permission mode"}, - "RenameSession": {KindCallerOwned, "authorizes via GetSession, then revalidates ownership, kind, state, liveness, and lease under runEntryMu before persisting"}, - "CompactSession": {KindCallerOwned, "binds the verified caller, then revalidates ownership, chat purpose, state, liveness, and lease under runEntryMu before saving and appending events"}, - "DeleteSession": {KindCallerOwned, "authorizes via GetSession, then revalidates ownership, kind, state, liveness, and lease under runEntryMu before physical deletion"}, - "EndSession": {KindCallerOwned, "authorizes via GetSession before CloseSession"}, - "ListSessions": {KindCallerOwned, "filters to the caller's own rows before any pagination/count is computed"}, - "ListSessionPage": {KindCallerOwned, "passes caller ownership into the store query before keyset page formation and counting"}, - "StorageHealth": {KindCallerOwned, "gates on the trusted-context management authorizer, NOT caller ownership — the aggregate it reads is store-wide (every session), never scoped to the caller's own rows; see the AccessKind doc comment note on management-authority boundaries"}, - "PlanSessionMigration": {KindCallerOwned, "gates on management authorization over the ENTIRE store, not the caller's own sessions; only the returned generation handle is bound to the verified caller for later Apply/Resume/Cancel binding"}, - "ApplySessionMigration": {KindCallerOwned, "gates on management authorization over the entire store; only the created job is caller-bound, so a foreign-caller job lookup is denied identically to a missing job"}, - "ResumeSessionMigration": {KindCallerOwned, "gates on management authorization; the same caller-bound job-handle check applies before processing another bounded batch — the underlying migration data remains store-wide, never caller-owned"}, - "CancelSessionMigration": {KindCallerOwned, "gates on management authorization; the same caller-bound job-handle check applies before stopping future items — the underlying migration data remains store-wide, never caller-owned"}, - "SessionMigrationJob": {KindCallerOwned, "gates on management authorization and conceals missing and cross-caller job handles identically; the underlying migration data is store-wide, not the caller's own sessions"}, - "PlanSessionCleanup": {KindCallerOwned, "gates on management authority, NOT caller ownership — the metadata pager plans over the ENTIRE store (owner scope is nil); the decision resolved here is 'is this caller a management principal', never per-session ownership"}, - "ApplySessionCleanup": {KindCallerOwned, "gates on management authority over the entire store; only the confirmation token/plan is bound to the verified caller, so a stolen or foreign token is rejected before any deletion"}, - "CancelSessionCleanup": {KindCallerOwned, "gates on management authority; matches the verified principal against the bounded job registry — the underlying cleanup scope is store-wide, never caller-owned"}, - "SessionCleanupJob": {KindCallerOwned, "gates on management authority and returns only a caller-bound sanitized job projection; the underlying cleanup data is store-wide, not the caller's own sessions"}, - "StreamSessionEvents": {KindCallerOwned, "event log/live stream resolves through the owning session's authorizeSession check"}, - "WatchSessionEvents": {KindCallerOwned, "durable replay-then-follow watch (ADR 0250); watchLog resolves ownership through the same GetSession check StreamSessionEvents uses, EAGERLY — before any envelope is yielded — because the durable log holds the whole transcript"}, - "Subscribe": {KindCallerOwned, "authorizes via GetSession before registering a live subscriber (issue #368)"}, + "CreateSession": {KindCallerOwned, "binds the verified context principal as owner atomically with visibility (reserveSessionID)"}, + "CreateSessionWithProvider": {KindCallerOwned, "delegates to CreateSessionWithProfile's atomic owner bind"}, + "CreateSessionWithProfile": {KindCallerOwned, "atomic owner bind at creation (reserveSessionID); ForkSession/carryover sources are authorized via authorizeSession before copying history"}, + "CreateSessionWithMCP": {KindCallerOwned, "delegates to CreateSessionWithProfile's atomic owner bind"}, + "CreateACPSession": {KindCallerOwned, "binds the trusted composition default before treating ACP cwd solely as an equality assertion"}, + "LoadACPSession": {KindCallerOwned, "owner-authorizes and reattaches the exact persisted placement before checking ACP cwd or mounting client MCP"}, + "BindPlacement": {KindCallerOwned, "passes the verified context principal and trusted composition scope to the provider's single atomic authorization-and-resolution operation"}, + "ReattachPlacement": {KindCallerOwned, "passes the exact persisted ref, verified context principal, and trusted scope to the provider without falling back to Bind"}, + "ReattachPlacementForBinding": {KindCallerOwned, "passes the exact persisted ref and session binding with the verified context principal to the provider"}, + "ReattachPlacementInScope": {KindCallerOwned, "requires the durable schedule scope to equal trusted composition scope, then reauthorizes the exact persisted ref as the verified owner without default fallback"}, + "ListCommandsForSession": {KindCallerOwned, "loads and owner-authorizes the source session before exact placement reattachment and command discovery"}, + "ListWorktreesForSession": {KindCallerOwned, "loads and owner-authorizes the source session before exact placement reattachment and scoped selector issuance"}, + "ClearSessionSuccessor": {KindCallerOwned, "serializes and leases an owner-authorized source before atomically publishing an empty-history placed successor"}, + "ForkSessionSuccessor": {KindCallerOwned, "serializes and leases an owner-authorized source before atomically publishing a history-carrying placed successor"}, + "GetSession": {KindCallerOwned, "authorizeSession: owner mismatch or absence both return ErrNotFound"}, + "WithAuthorizedSession": {KindCallerOwned, "ownership preflight excludes foreign lock contention; authoritative reload under runEntryMu precedes the caller-owned effect"}, + "GetTranscript": {KindCallerOwned, "one SessionStore.Load followed by authorizeSession; no run-entry side effects"}, + "LoadSession": {KindCallerOwned, "authorizeSession before rehydration"}, + "LoadSessionWithMCP": {KindCallerOwned, "delegates to LoadSession's authorizeSession before mounting client MCP"}, + "SetMode": {KindCallerOwned, "authorizes via GetSession before changing the session's permission mode"}, + "RenameSession": {KindCallerOwned, "authorizes via GetSession, then revalidates ownership, kind, state, liveness, and lease under runEntryMu before persisting"}, + "CompactSession": {KindCallerOwned, "binds the verified caller, then revalidates ownership, chat purpose, state, liveness, and lease under runEntryMu before saving and appending events"}, + "DeleteSession": {KindCallerOwned, "authorizes via GetSession, then revalidates ownership, kind, state, liveness, and lease under runEntryMu before physical deletion"}, + "EndSession": {KindCallerOwned, "authorizes via GetSession before CloseSession"}, + "ListSessions": {KindCallerOwned, "filters to the caller's own rows before any pagination/count is computed"}, + "ListSessionPage": {KindCallerOwned, "passes caller ownership into the store query before keyset page formation and counting"}, + "StorageHealth": {KindCallerOwned, "gates on the trusted-context management authorizer, NOT caller ownership — the aggregate it reads is store-wide (every session), never scoped to the caller's own rows; see the AccessKind doc comment note on management-authority boundaries"}, + "PlanSessionMigration": {KindCallerOwned, "gates on management authorization over the ENTIRE store, not the caller's own sessions; only the returned generation handle is bound to the verified caller for later Apply/Resume/Cancel binding"}, + "ApplySessionMigration": {KindCallerOwned, "gates on management authorization over the entire store; only the created job is caller-bound, so a foreign-caller job lookup is denied identically to a missing job"}, + "ResumeSessionMigration": {KindCallerOwned, "gates on management authorization; the same caller-bound job-handle check applies before processing another bounded batch — the underlying migration data remains store-wide, never caller-owned"}, + "CancelSessionMigration": {KindCallerOwned, "gates on management authorization; the same caller-bound job-handle check applies before stopping future items — the underlying migration data remains store-wide, never caller-owned"}, + "SessionMigrationJob": {KindCallerOwned, "gates on management authorization and conceals missing and cross-caller job handles identically; the underlying migration data is store-wide, not the caller's own sessions"}, + "PlanSessionCleanup": {KindCallerOwned, "gates on management authority, NOT caller ownership — the metadata pager plans over the ENTIRE store (owner scope is nil); the decision resolved here is 'is this caller a management principal', never per-session ownership"}, + "ApplySessionCleanup": {KindCallerOwned, "gates on management authority over the entire store; only the confirmation token/plan is bound to the verified caller, so a stolen or foreign token is rejected before any deletion"}, + "CancelSessionCleanup": {KindCallerOwned, "gates on management authority; matches the verified principal against the bounded job registry — the underlying cleanup scope is store-wide, never caller-owned"}, + "SessionCleanupJob": {KindCallerOwned, "gates on management authority and returns only a caller-bound sanitized job projection; the underlying cleanup data is store-wide, not the caller's own sessions"}, + "StreamSessionEvents": {KindCallerOwned, "event log/live stream resolves through the owning session's authorizeSession check"}, + "WatchSessionEvents": {KindCallerOwned, "durable replay-then-follow watch (ADR 0250); watchLog resolves ownership through the same GetSession check StreamSessionEvents uses, EAGERLY — before any envelope is yielded — because the durable log holds the whole transcript"}, + "Subscribe": {KindCallerOwned, "authorizes via GetSession before registering a live subscriber (issue #368)"}, // --- caller-owned: live run verbs --- "RetryFailedRun": {KindCallerOwned, "authorizes and reloads under runEntryMu before failed-step retry eligibility and launch"}, @@ -343,6 +344,7 @@ var serviceAccessTable = map[string]ClassificationEntry{ "LookupRun": {KindDerived, "in-memory run registry read; every caller-facing entry point (Cancel, Persist, Approve*, MaybeAutoApprovePlan) authorizes the session FIRST and only then consults this"}, "IsLive": {KindDerived, "combined Service-run and engine-child process-local registry; consumed by destructive maintenance, not a caller-facing verb"}, "MaintenanceMutationAvailable": {KindDerived, "read-only capability truth consumed by composition before scheduling automatic retention"}, + "ReconcileReferenceIntents": {KindDerived, "composition-owned maintenance joins provider-owned exact intents to durable sessions and invokes only exact retained callbacks"}, "FinishRun": {KindDerived, "deregisters an id the wire adapter already finished draining from its own authorized run"}, "PublishSessionEvent": {KindDerived, "publishes to subscribers already registered via the (caller-owned) Subscribe for this id; PublishSessionEvent itself takes no ctx and makes no independent decision"}, "RecoverNotice": {KindDerived, "pops a notice keyed by id that only the relay's own immediately-preceding, already-authorized StartRunContent call could have set"}, diff --git a/internal/adapter/server/execution_renewal_test.go b/internal/adapter/server/execution_renewal_test.go new file mode 100644 index 0000000000..06cc3d5ce1 --- /dev/null +++ b/internal/adapter/server/execution_renewal_test.go @@ -0,0 +1,33 @@ +package server + +import ( + "context" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/tool" +) + +type deadlineExecutionHandle struct { + deadline time.Time +} + +func (*deadlineExecutionHandle) Environment() tool.Environment { return tool.Environment{} } +func (*deadlineExecutionHandle) Renew(context.Context) error { return nil } +func (*deadlineExecutionHandle) Release(context.Context) error { return nil } +func (h *deadlineExecutionHandle) RenewalDeadline() time.Time { return h.deadline } + +var _ ExecutionRunHandle = (*deadlineExecutionHandle)(nil) + +func TestExecutionRenewDelayTracksShortGrantExpiry(t *testing.T) { + now := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC) + handle := &deadlineExecutionHandle{deadline: now.Add(5 * time.Second)} + delay, usable := executionRenewDelay(handle, now) + if !usable || delay <= 0 || delay >= 5*time.Second || delay < 100*time.Millisecond { + t.Fatalf("five-second grant schedule delay=%v usable=%t", delay, usable) + } + handle.deadline = now.Add(250 * time.Millisecond) + if _, usable := executionRenewDelay(handle, now); usable { + t.Fatal("unusable grant remainder would start execution renewal") + } +} diff --git a/internal/adapter/server/http.go b/internal/adapter/server/http.go index 701ac3056a..6928718d90 100644 --- a/internal/adapter/server/http.go +++ b/internal/adapter/server/http.go @@ -20,6 +20,7 @@ import ( "github.com/stacklok/mecatl/engine/port" "github.com/stacklok/mecatl/engine/session" "github.com/stacklok/mecatl/internal/adapter/mcp" + "github.com/stacklok/mecatl/internal/creatediag" ) // HTTPHandler is the HTTP/SSE adapter over the shared Service. It serves the @@ -563,6 +564,11 @@ type approveBody struct { // createSession handles POST /v1/sessions. func (h *HTTPHandler) createSession(w http.ResponseWriter, r *http.Request) { + if h.svc.cfg.ExecutionAccess != nil { + r = r.WithContext(creatediag.Start(r.Context(), h.svc.Diagnostics())) + } + creatediag.Note(r.Context(), "http_handler", "begin", 0) + defer func() { creatediag.Note(r.Context(), "http_handler", "returned", 1) }() var body createSessionBody // STRICT decode. An unknown field is a 400, not a silent drop. // @@ -633,6 +639,7 @@ func (h *HTTPHandler) createSession(w http.ResponseWriter, r *http.Request) { writeServiceError(w, err) return } + responseDone := creatediag.Begin(r.Context(), "http_response") scaps := h.svc.sessionCapabilitiesFor(sess) writeJSON(w, http.StatusCreated, createSessionResp{ SessionID: string(sess.ID), @@ -641,6 +648,7 @@ func (h *HTTPHandler) createSession(w http.ResponseWriter, r *http.Request) { ResolvedModel: resolvedModelToJSON(h.svc.ResolvedModel(sess.ID)), Placement: placementMetadataToJSON(sess.Placement), }) + responseDone(r.Context().Err()) } // getSession handles GET /v1/sessions/{id}. diff --git a/internal/adapter/server/learning.go b/internal/adapter/server/learning.go index 0118f42746..bc9fa4f2a5 100644 --- a/internal/adapter/server/learning.go +++ b/internal/adapter/server/learning.go @@ -82,7 +82,7 @@ func (s *Service) ReflectSession(ctx context.Context, id session.SessionID) (*me return nil, fmt.Errorf("%w: reflection requires a completed session", ErrFailedPrecondition) } if s.placementBinder != nil { - binding, bindErr := s.ReattachPlacement(ctx, sess.EnvironmentRef) + binding, bindErr := s.ReattachPlacementForBinding(ctx, sess.EnvironmentRef, sess.ID) if bindErr != nil { return nil, bindErr } @@ -570,7 +570,7 @@ func (s *Service) learningWorkspace(ctx context.Context, sess *session.Session) if s.placementBinder == nil { return "", true } - binding, err := s.ReattachPlacement(ctx, sess.EnvironmentRef) + binding, err := s.ReattachPlacementForBinding(ctx, sess.EnvironmentRef, sess.ID) if err != nil { return "", false } diff --git a/internal/adapter/server/local_session_context.go b/internal/adapter/server/local_session_context.go index 37b98e24ba..6ef0b8ecf0 100644 --- a/internal/adapter/server/local_session_context.go +++ b/internal/adapter/server/local_session_context.go @@ -47,7 +47,7 @@ func (s *Service) localSessionContextRoot(ctx context.Context, id session.Sessio if persisted.EnvironmentRef.Kind != session.EnvKindLocal { return "", ErrFailedPrecondition } - binding, err := s.ReattachPlacement(ctx, persisted.EnvironmentRef) + binding, err := s.ReattachPlacementForBinding(ctx, persisted.EnvironmentRef, persisted.ID) if err != nil { return "", err } diff --git a/internal/adapter/server/native_execution_lifecycle_test.go b/internal/adapter/server/native_execution_lifecycle_test.go new file mode 100644 index 0000000000..3dc5999b03 --- /dev/null +++ b/internal/adapter/server/native_execution_lifecycle_test.go @@ -0,0 +1,96 @@ +package server_test + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/adapter/memfs" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/adapter/permpolicy" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" +) + +type nativeRunProvider struct { + ref session.EnvironmentRef + acquires atomic.Int64 + releases atomic.Int64 +} + +func (p *nativeRunProvider) Bind(context.Context, server.PlacementBindRequest) (server.PlacementBinding, error) { + return p.binding(), nil +} +func (p *nativeRunProvider) Reattach(context.Context, server.PlacementReattachRequest) (server.PlacementBinding, error) { + return p.binding(), nil +} +func (p *nativeRunProvider) Applies(ref session.EnvironmentRef) bool { return ref == p.ref } +func (p *nativeRunProvider) AcquireRun(context.Context, server.ExecutionRunRequest) (server.ExecutionRunHandle, error) { + p.acquires.Add(1) + return &nativeRunHandle{provider: p, env: p.binding().Environment}, nil +} +func (p *nativeRunProvider) binding() server.PlacementBinding { + return server.PlacementBinding{Ref: p.ref, Environment: tool.MustEnvironment(p.ref, memfs.NewWorkspace("/native"), memledger.New(), nil)} +} + +type nativeRunHandle struct { + provider *nativeRunProvider + env tool.Environment +} + +func (h *nativeRunHandle) Environment() tool.Environment { return h.env } +func (*nativeRunHandle) Renew(context.Context) error { return nil } +func (h *nativeRunHandle) Release(context.Context) error { h.provider.releases.Add(1); return nil } + +func TestNativeRunClaimHeldUntilActualDrainRelease(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + provider := &nativeRunProvider{ref: ref} + llm := mockllm.New(mockllm.TextTurn("first"), mockllm.TextTurn("second")) + engine := agent.NewEngine(agent.Deps{LLM: llm, Catalog: tool.NewCatalog(), Policy: permpolicy.NewPolicy(nil, nil), Model: "test"}) + store := memstore.New() + svc, err := newPlacementTestService(server.Config{Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "test", ExecutionAccess: provider, SharedEngineRoot: "/native", DefaultLimits: session.Limits{MaxTurns: 2}, Now: func() time.Time { return time.Unix(1, 0) }}) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + sess := session.New("native-session", session.ModeDefault, ref, session.Limits{MaxTurns: 2}, time.Unix(1, 0)) + if err := store.Save(t.Context(), sess); err != nil { + t.Fatal(err) + } + + first, err := svc.StartRunContent(t.Context(), sess.ID, "one", nil) + if err != nil { + t.Fatal(err) + } + for range first.Events() { + } + if _, err := svc.StartRunContent(t.Context(), sess.ID, "too early", nil); err == nil { + t.Fatal("native continuation started before the prior relay released its claim") + } + if provider.acquires.Load() != 1 || provider.releases.Load() != 0 { + t.Fatalf("before FinishRun acquires=%d releases=%d", provider.acquires.Load(), provider.releases.Load()) + } + + svc.FinishRun(sess.ID, first) + if provider.releases.Load() != 1 { + t.Fatalf("release count=%d, want 1", provider.releases.Load()) + } + second, err := svc.StartRunContent(t.Context(), sess.ID, "two", nil) + if err != nil { + t.Fatal(err) + } + for range second.Events() { + } + svc.FinishRun(sess.ID, second) + svc.FinishRun(sess.ID, second) + svc.Close() + svc.Close() + if provider.acquires.Load() != 2 || provider.releases.Load() != 2 { + t.Fatalf("after continuation acquires=%d releases=%d", provider.acquires.Load(), provider.releases.Load()) + } +} diff --git a/internal/adapter/server/placement.go b/internal/adapter/server/placement.go index cc8ccb81ba..58245c0fbb 100644 --- a/internal/adapter/server/placement.go +++ b/internal/adapter/server/placement.go @@ -1,3 +1,4 @@ +//nolint:revive // Host-only lifecycle seams are exported solely for internal adapter composition. package server import ( @@ -5,6 +6,7 @@ import ( "errors" "fmt" "strings" + "time" "unicode" "unicode/utf8" @@ -103,6 +105,9 @@ type PlacementBindRequest struct { Principal *session.Principal Scope PlacementScope Operation PlacementOperation + // BindingID is the final server-minted session identity. Providers that + // allocate durable environments use it as their idempotency/reference key. + BindingID session.SessionID } // PlacementMetadata is the bounded, display-safe provider projection returned @@ -119,9 +124,29 @@ type PlacementBinding struct { Environment tool.Environment Ref session.EnvironmentRef Metadata PlacementMetadata - // Close releases provisional provider resources. It is called after creation - // because ordinary bindings are reattached fresh at run entry. - Close func() error + // Commit publishes a provisional durable reference after the session store + // definitively publishes the matching binding. Close aborts it only while Commit + // has not succeeded; an ambiguous commit must remain recoverable. + Commit func(context.Context) error + Close func() error +} + +type ExecutionRunRequest struct { + Ref session.EnvironmentRef + Principal *session.Principal + BindingID session.SessionID + RunID string +} + +type ExecutionRunHandle interface { + Environment() tool.Environment + Renew(context.Context) error + Release(context.Context) error +} + +type ExecutionAccess interface { + Applies(session.EnvironmentRef) bool + AcquireRun(context.Context, ExecutionRunRequest) (ExecutionRunHandle, error) } // PlacementProvider owns placement authorization, atomic binding resolution, @@ -130,6 +155,46 @@ type PlacementProvider interface { Bind(context.Context, PlacementBindRequest) (PlacementBinding, error) } +type PlacementSuccessorRequest struct { + Ref session.EnvironmentRef + Principal *session.Principal + SourceBindingID session.SessionID + DestinationBindingID session.SessionID +} + +type ReferenceDeleteHandle interface { + Confirm(context.Context) error + Cancel(context.Context) error +} +type ReferenceLifecycle interface { + Applies(session.EnvironmentRef) bool + PrepareReferenceDelete(context.Context, PlacementSuccessorRequest) (ReferenceDeleteHandle, error) +} + +// ReferenceIntent is the owner-attested, exact provider operation retained after +// an ambiguous publication or deletion outcome. +type ReferenceIntent struct { + Ref session.EnvironmentRef + Principal *session.Principal + BindingID session.SessionID + SourceBindingID session.SessionID + OperationID string + PendingDelete bool +} + +// ReferenceIntentLifecycle is the optional host reconciliation seam. Its list is +// scoped by the provider to the authenticated mTLS client and bounded by limit. +type ReferenceIntentLifecycle interface { + ListReferenceIntents(context.Context, int) ([]ReferenceIntent, error) + CommitReferenceIntent(context.Context, ReferenceIntent) error + ConfirmReferenceIntentDelete(context.Context, ReferenceIntent) error + CancelReferenceIntentDelete(context.Context, ReferenceIntent) error +} + +type PlacementSuccessorReservoir interface { + ReserveSuccessor(context.Context, PlacementSuccessorRequest) (PlacementBinding, error) +} + // PlacementDiscoveryRequest scopes alternate-worktree discovery to an owned // source and its exact current placement. type PlacementDiscoveryRequest struct { @@ -159,6 +224,14 @@ type PlacementReattachRequest struct { Ref session.EnvironmentRef Principal *session.Principal Scope PlacementScope + // BindingID identifies the durable reference being reattached. + BindingID session.SessionID +} + +// PlacementValidator performs side-effect-free startup validation. Providers +// whose Bind allocates resources implement this seam so preflight never binds. +type PlacementValidator interface { + ValidatePlacement(context.Context) error } // PlacementBinder is the server-owned choke point around one deployment @@ -241,10 +314,14 @@ func configuredPlacementBinder(ctx context.Context, cfg Config) (*PlacementBinde if err != nil { return nil, err } - // NewService runs before a listener can serve. Binding the configured - // default proves that its current record is authorized, available, - // revision-stable, and capable of constructing a complete environment. The - // result is deliberately not cached. + if validator, ok := cfg.PlacementProvider.(PlacementValidator); ok { + if err := validator.ValidatePlacement(ctx); err != nil { + return nil, fmt.Errorf("server: validate default placement: %w", sanitizePlacementProviderError(err)) + } + return binder, nil + } + // Legacy providers validate by binding the configured default. Providers whose + // Bind allocates must implement PlacementValidator above. validation, err := binder.Bind(ctx, PlacementBindRequest{ Selector: DefaultPlacement(), Scope: cfg.PlacementScope, Operation: PlacementOperationCreate, @@ -258,14 +335,14 @@ func configuredPlacementBinder(ctx context.Context, cfg Config) (*PlacementBinde return binder, nil } -func (s *Service) bindPlacementForCreate(ctx context.Context, profile SessionProfile, owner *session.Principal) (string, *PlacementBinding, error) { +func (s *Service) bindPlacementForCreate(ctx context.Context, profile SessionProfile, owner *session.Principal, bindingID session.SessionID) (string, *PlacementBinding, error) { selector := DefaultPlacement() if profile == ProfileNoFS { selector = NoFSPlacement() } binding, err := s.placementBinder.Bind(ctx, PlacementBindRequest{ Selector: selector, Principal: owner, Scope: s.cfg.PlacementScope, - Operation: PlacementOperationCreate, + Operation: PlacementOperationCreate, BindingID: bindingID, }) if err != nil { return "", nil, err @@ -284,7 +361,17 @@ func (s *Service) persistPlacedCreatedSession(ctx context.Context, sess *session if !sess.EnvironmentRef.Valid() { return nil, fmt.Errorf("%w: placement did not provide an exact environment ref", ErrInvalidPlacementBinding) } - return s.persistCreatedSession(ctx, sess, owner, request) + persisted, err := s.persistCreatedSession(ctx, sess, owner, request) + if err != nil || persisted != sess || placement == nil || placement.Commit == nil { + return persisted, err + } + commitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + if err := placement.Commit(commitCtx); err != nil { + s.logPlacementProviderError(ctx, "commit reference", err) + return sess, fmt.Errorf("%w: session %q persisted but reference publication must be retried", ErrInternal, sess.ID) + } + return sess, nil } func (s *Service) resolveSchedulePlacement(ctx context.Context, ref session.EnvironmentRef, profile SessionProfile) (session.EnvironmentRef, string, SessionProfile, error) { @@ -318,7 +405,7 @@ func (s *Service) privateWorkspace(ctx context.Context, sess *session.Session) ( if ok && env.Ref() == sess.EnvironmentRef && env.Workspace() != nil { return env.Workspace().Root(), nil } - binding, err := s.ReattachPlacement(ctx, sess.EnvironmentRef) + binding, err := s.ReattachPlacementForBinding(ctx, sess.EnvironmentRef, sess.ID) if err != nil { return "", err } diff --git a/internal/adapter/server/placement_discovery.go b/internal/adapter/server/placement_discovery.go index 3dbc4cb752..b9949cdfec 100644 --- a/internal/adapter/server/placement_discovery.go +++ b/internal/adapter/server/placement_discovery.go @@ -37,7 +37,7 @@ func (s *Service) ownedSessionEnvironment(ctx context.Context, id session.Sessio if sess.EnvironmentRef.Kind == session.EnvKindNoFS { return sess, tool.Environment{}, nil } - binding, err := s.ReattachPlacement(ctx, sess.EnvironmentRef) + binding, err := s.ReattachPlacementForBinding(ctx, sess.EnvironmentRef, sess.ID) if err != nil { return nil, tool.Environment{}, err } diff --git a/internal/adapter/server/placement_successor.go b/internal/adapter/server/placement_successor.go index 5aca1d5883..0f85e8c0c2 100644 --- a/internal/adapter/server/placement_successor.go +++ b/internal/adapter/server/placement_successor.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/stacklok/mecatl/engine/port" "github.com/stacklok/mecatl/engine/session" @@ -57,7 +58,7 @@ func (s *Service) ClearSessionSuccessor(ctx context.Context, source session.Sess // non-destructive gate. Exact inherited placement stays on the lease-protected // path below because reattachment may itself wait for lease loss/cancellation. if placement.Selector != "" { - preflight, placementErr := s.successorPlacement(ctx, lockedSource, placement) + preflight, placementErr := s.successorPlacement(ctx, lockedSource, placement, "") if placementErr != nil { return "", placementErr } @@ -153,11 +154,17 @@ func (s *Service) createPlacedSuccessorLocked(ctx context.Context, req ForkSucce if err != nil || absent { return "", err } + destinationID := s.cfg.NewID() + releaseDestination, reserveErr := s.reserveGeneratedSessionID(mutationCtx, destinationID) + if reserveErr != nil { + return "", reserveErr + } + defer releaseDestination() selector, err := successorProviderSelector(source, req) if err != nil { return "", err } - binding, err := s.successorPlacement(mutationCtx, source, req.Placement) + binding, err := s.successorPlacement(mutationCtx, source, req.Placement, destinationID) if err != nil { return "", err } @@ -176,7 +183,7 @@ func (s *Service) createPlacedSuccessorLocked(ctx context.Context, req ForkSucce } } - created := session.New(s.cfg.NewID(), source.Mode, binding.Ref, source.Limits, s.cfg.Now()) + created := session.New(destinationID, source.Mode, binding.Ref, source.Limits, s.cfg.Now()) created.Placement = canonicalPlacementMetadata(binding) authority, bound := source.BoundAuthority() if !bound { @@ -261,6 +268,15 @@ func (s *Service) createPlacedSuccessorLocked(ctx context.Context, req ForkSucce s.logDiscoveryError(ctx, "persist successor placement", err) return "", fmt.Errorf("%w: placement storage failed", ErrInternal) } + if binding.Commit != nil { + commitCtx, cancelCommit := context.WithTimeout(context.WithoutCancel(mutationCtx), 10*time.Second) + commitErr := binding.Commit(commitCtx) + cancelCommit() + if commitErr != nil { + s.logPlacementProviderError(ctx, "commit successor reference", commitErr) + return created.ID, fmt.Errorf("%w: session %q persisted but reference publication must be retried", ErrInternal, created.ID) + } + } if broker != nil { commitCtx, cancelCommit := context.WithTimeout(context.WithoutCancel(mutationCtx), engineCloseTimeout) commitErr := s.commitBrokerAttachment(commitCtx, created.ID, broker) @@ -291,9 +307,16 @@ func successorProviderSelector(source *session.Session, req ForkSuccessorRequest return selector, nil } -func (s *Service) successorPlacement(ctx context.Context, source *session.Session, requested SuccessorPlacement) (PlacementBinding, error) { +func (s *Service) successorPlacement(ctx context.Context, source *session.Session, requested SuccessorPlacement, destination session.SessionID) (PlacementBinding, error) { if requested.Selector == "" { - return s.ReattachPlacement(ctx, source.EnvironmentRef) + if destination != "" && s.cfg.ExecutionAccess != nil && s.cfg.ExecutionAccess.Applies(source.EnvironmentRef) { + reservoir, ok := s.cfg.PlacementProvider.(PlacementSuccessorReservoir) + if !ok { + return PlacementBinding{}, ErrPlacementUnavailable + } + return reservoir.ReserveSuccessor(ctx, PlacementSuccessorRequest{Ref: source.EnvironmentRef, Principal: source.Owner, SourceBindingID: source.ID, DestinationBindingID: destination}) + } + return s.ReattachPlacementForBinding(ctx, source.EnvironmentRef, source.ID) } if source.EnvironmentRef.Kind == session.EnvKindNoFS { return PlacementBinding{}, ErrPlacementNotFound diff --git a/internal/adapter/server/reference_intent_reconcile_test.go b/internal/adapter/server/reference_intent_reconcile_test.go new file mode 100644 index 0000000000..fd0d694113 --- /dev/null +++ b/internal/adapter/server/reference_intent_reconcile_test.go @@ -0,0 +1,86 @@ +package server_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/adapter/permpolicy" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" +) + +type intentLifecycle struct { + intents []server.ReferenceIntent + mu sync.Mutex + committed, confirmed, reset []session.SessionID +} + +func (l *intentLifecycle) ListReferenceIntents(context.Context, int) ([]server.ReferenceIntent, error) { + return append([]server.ReferenceIntent(nil), l.intents...), nil +} +func (l *intentLifecycle) CommitReferenceIntent(_ context.Context, i server.ReferenceIntent) error { + l.mu.Lock() + defer l.mu.Unlock() + l.committed = append(l.committed, i.BindingID) + return nil +} +func (l *intentLifecycle) ConfirmReferenceIntentDelete(_ context.Context, i server.ReferenceIntent) error { + l.mu.Lock() + defer l.mu.Unlock() + l.confirmed = append(l.confirmed, i.BindingID) + return nil +} +func (l *intentLifecycle) CancelReferenceIntentDelete(_ context.Context, i server.ReferenceIntent) error { + l.mu.Lock() + defer l.mu.Unlock() + l.reset = append(l.reset, i.BindingID) + return nil +} + +func TestReferenceIntentReconciliationIsOwnerAndRefExact(t *testing.T) { + owner := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + otherRef := session.EnvironmentRef{Kind: ref.Kind, ID: ref.ID, Revision: "new-revision"} + store := memstore.New() + persist := func(id session.SessionID, placement session.EnvironmentRef) { + sess := session.New(id, session.ModeDefault, placement, session.Limits{MaxTurns: 1}, time.Unix(1, 0)) + if err := sess.RestoreLabels(owner, session.Authority{}); err != nil { + t.Fatal(err) + } + if err := store.Save(t.Context(), sess); err != nil { + t.Fatal(err) + } + } + persist("create-match", ref) + persist("delete-restored", ref) + persist("reused-id", otherRef) + lifecycle := &intentLifecycle{intents: []server.ReferenceIntent{ + {Ref: ref, Principal: owner, BindingID: "create-match", OperationID: "create-op"}, + {Ref: ref, Principal: owner, BindingID: "delete-gone", OperationID: "delete-op", PendingDelete: true}, + {Ref: ref, Principal: owner, BindingID: "delete-restored", OperationID: "delete-restore-op", PendingDelete: true}, + {Ref: ref, Principal: owner, BindingID: "reused-id", OperationID: "old-delete-op", PendingDelete: true}, + }} + engine := agent.NewEngine(agent.Deps{LLM: mockllm.New(), Catalog: tool.NewCatalog(), Policy: permpolicy.NewPolicy(nil, nil)}) + svc, err := newPlacementTestService(server.Config{Engine: engine, Store: store, PlacementProvider: resolverPlacementProvider{}, PlacementScope: "test", ReferenceIntents: lifecycle}) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + + svc.ReconcileReferenceIntents(t.Context()) + if len(lifecycle.committed) != 1 || lifecycle.committed[0] != "create-match" { + t.Fatalf("committed=%v", lifecycle.committed) + } + if len(lifecycle.confirmed) != 1 || lifecycle.confirmed[0] != "delete-gone" { + t.Fatalf("confirmed=%v", lifecycle.confirmed) + } + if len(lifecycle.reset) != 0 { + t.Fatalf("ambiguous present delete was cancelled=%v", lifecycle.reset) + } +} diff --git a/internal/adapter/server/sdk_typescript_release_test.go b/internal/adapter/server/sdk_typescript_release_test.go index 2aa455a359..3c1ad07d9c 100644 --- a/internal/adapter/server/sdk_typescript_release_test.go +++ b/internal/adapter/server/sdk_typescript_release_test.go @@ -229,6 +229,7 @@ func TestSDKTypescriptRelease_Scenario1_PublicServiceProjectionParity(t *testing "LostOwnershipCandidates", "MaintenanceMutationAvailable", "ManualDreamCapabilities", + "ReconcileReferenceIntents", "MaybeAutoApprovePlan", "OwnershipEnforced", "Persist", @@ -236,6 +237,7 @@ func TestSDKTypescriptRelease_Scenario1_PublicServiceProjectionParity(t *testing "ProviderStatuses", "PublishSessionEvent", "ReattachPlacement", + "ReattachPlacementForBinding", "ReattachPlacementInScope", "ReconcileLeaseLossTombstone", "RecoverNotice", diff --git a/internal/adapter/server/service.go b/internal/adapter/server/service.go index 0b62f31cec..4e1e3fecc3 100644 --- a/internal/adapter/server/service.go +++ b/internal/adapter/server/service.go @@ -33,6 +33,7 @@ import ( "github.com/stacklok/mecatl/internal/adapter/scheduler" "github.com/stacklok/mecatl/internal/adapter/skills" "github.com/stacklok/mecatl/internal/adapter/tools" + "github.com/stacklok/mecatl/internal/creatediag" brokercontract "github.com/stacklok/mecatl/internal/mcpbroker" ) @@ -343,6 +344,12 @@ type Config struct { // PlacementScope is the trusted deployment scope supplied to every provider // Bind. It must be non-empty when PlacementProvider is configured. PlacementScope PlacementScope + // ExecutionAccess is an optional host-only run-ownership seam. Local and no-FS + // placements leave it nil and perform no lifecycle RPCs. + ExecutionAccess ExecutionAccess + // ReferenceIntents enables owner-safe reconciliation of provider-retained + // publication/deletion outcomes. Nil is fully inert. + ReferenceIntents ReferenceIntentLifecycle // SessionReadLedger optionally selects a durable read-before-write ledger for // each session independently of its placement's content backend. The returned // handle must be non-nil; nil fails the run closed. @@ -902,6 +909,8 @@ type Service struct { // refresh — background or on-demand — re-projects it). Never nil after // NewService. providerStatus atomic.Pointer[[]*mecatlv1.ProviderStatus] + // referenceIntentWarned bounds diagnostics for retained ambiguous intents. + referenceIntentWarned atomic.Bool // modelsRefresher is the OPTIONAL composition-supplied closure ListModels // calls before returning its snapshot (issue #262, R1.4: a proxy started @@ -1293,8 +1302,14 @@ type runState struct { admissionCancel context.CancelFunc // runContextStop releases the lease-linked launch context after the promoted // run has settled. It must outlive the run-entry call itself. - runContextStop context.CancelFunc - awaiting atomic.Bool + runContextStop context.CancelFunc + execution ExecutionRunHandle + executionMu sync.Mutex + executionClosed bool + executionRenewCancel context.CancelFunc + executionRenewDone chan struct{} + executionTeardown sync.Once + awaiting atomic.Bool // titleRevision is the last title metadata revision successfully persisted and // published for this run. It starts from the admitted durable snapshot so // prompt-ingress changes publish only after their save succeeds. @@ -1553,11 +1568,21 @@ func (s *Service) BindPlacement(ctx context.Context, selector PlacementSelector, // ReattachPlacement authorizes and resolves the exact persisted environment // identity. It never invokes Bind and therefore cannot follow a changed default. func (s *Service) ReattachPlacement(ctx context.Context, ref session.EnvironmentRef) (PlacementBinding, error) { + return s.reattachPlacement(ctx, ref, "") +} + +// ReattachPlacementForBinding supplies the durable session reference identity +// required by allocating remote providers. +func (s *Service) ReattachPlacementForBinding(ctx context.Context, ref session.EnvironmentRef, bindingID session.SessionID) (PlacementBinding, error) { + return s.reattachPlacement(ctx, ref, bindingID) +} + +func (s *Service) reattachPlacement(ctx context.Context, ref session.EnvironmentRef, bindingID session.SessionID) (PlacementBinding, error) { if s == nil || s.placementBinder == nil { return PlacementBinding{}, fmt.Errorf("%w: no PlacementProvider is configured", ErrFailedPrecondition) } binding, err := s.placementBinder.Reattach(ctx, PlacementReattachRequest{ - Ref: ref, Principal: session.PrincipalFromContext(ctx), Scope: s.cfg.PlacementScope, + Ref: ref, Principal: session.PrincipalFromContext(ctx), Scope: s.cfg.PlacementScope, BindingID: bindingID, }) if err != nil { s.logPlacementProviderError(ctx, "reattach", err) @@ -2095,53 +2120,61 @@ func (s *Service) classifyCreateWinner(existing *session.Session, owner *session return existing, nil } -// reserveSessionID validates a caller-chosen session id (WithSessionID, ADR 0059 -// decision #7 Phase-2) against THREE collision sources and reserves it for the -// duration of the create, returning a release func the caller MUST defer: -// -// 1. a LIVE per-session engine (sessionEngines — a collision would shadow an -// in-flight session); -// 2. an in-flight create holding the id (reservedIDs — closes the old TOCTOU: -// the prior check released s.mu before the slow factory call and the separate -// registration, so two concurrent creates on the same id both passed); -// 3. a PERSISTED session already in the store (a completed prior create is NOT -// in sessionEngines — e.g. the shared-engine fast path never registers there). -// -// The in-memory reservation (1)+(2) is taken under a single s.mu hold; the store -// probe (3) runs after (no I/O under the mutex). On a collision or an infra probe -// fault the reservation is released before returning the error. Once the session -// is registered (per-session) or persisted (shared) the durable collision sources -// take over, so the reservation only needs to live for the create. -func (s *Service) reserveSessionID(ctx context.Context, id session.SessionID, owner *session.Principal, request createRequest) (existing *session.Session, release func(), err error) { +func (s *Service) reserveCreateID(ctx context.Context, id session.SessionID, owner *session.Principal) (*session.Session, func(), error) { s.mu.Lock() _, liveEngine := s.sessionEngines[id] _, reserved := s.reservedIDs[id] if liveEngine || reserved { s.mu.Unlock() - // Accurate for BOTH cases: a live per-session engine (liveEngine) OR a - // concurrent in-flight create holding the id (reserved). return nil, nil, fmt.Errorf("%w: session id %q is already in use", ErrInvalidArgument, id) } s.reservedIDs[id] = struct{}{} s.mu.Unlock() - release = func() { + release := func() { s.mu.Lock() delete(s.reservedIDs, id) s.mu.Unlock() } - // Probe the store for a persisted session under this id. A not-found error - // means the id is clear; any other infrastructure fault fails closed and is - // exposed only through a content-free public category. - if existing, lerr := s.cfg.Store.Load(ctx, id); lerr == nil && existing != nil { - release() - winner, classifyErr := s.classifyCreateWinner(existing, owner, request) - return winner, nil, classifyErr - } else if lerr != nil && !errors.Is(lerr, port.ErrSessionNotFound) { + existing, err := s.cfg.Store.Load(ctx, id) + if errors.Is(err, port.ErrSessionNotFound) { + return nil, release, nil + } + if err != nil { release() - s.logDiscoveryError(ctx, "probe session placement", lerr) return nil, nil, fmt.Errorf("%w: placement storage failed", ErrInternal) } - return nil, release, nil + sameOwner := existing != nil && (existing.Owner == nil && owner == nil || existing.Owner.SameIdentity(owner)) + if !sameOwner { + release() + return nil, nil, fmt.Errorf("%w: %q", ErrNotFound, id) + } + return existing, release, nil +} + +func (s *Service) reserveGeneratedSessionID(ctx context.Context, id session.SessionID) (func(), error) { + s.mu.Lock() + _, liveEngine := s.sessionEngines[id] + _, reserved := s.reservedIDs[id] + if liveEngine || reserved { + s.mu.Unlock() + return nil, fmt.Errorf("%w: session id %q is already in use", ErrInvalidArgument, id) + } + s.reservedIDs[id] = struct{}{} + s.mu.Unlock() + release := func() { + s.mu.Lock() + delete(s.reservedIDs, id) + s.mu.Unlock() + } + if existing, err := s.cfg.Store.Load(ctx, id); err == nil && existing != nil { + release() + return nil, port.ErrSessionAlreadyExists + } else if err != nil && !errors.Is(err, port.ErrSessionNotFound) { + release() + s.logDiscoveryError(ctx, "probe generated session placement", err) + return nil, fmt.Errorf("%w: placement storage failed", ErrInternal) + } + return release, nil } func (s *Service) persistNewSession(ctx context.Context, sess *session.Session) error { @@ -2155,7 +2188,10 @@ func (s *Service) persistNewSession(ctx context.Context, sess *session.Session) } func (s *Service) persistCreatedSession(ctx context.Context, sess *session.Session, owner *session.Principal, request *createRequest) (*session.Session, error) { - if err := s.persistNewSession(ctx, sess); err != nil { + persistDone := creatediag.Begin(ctx, "session_persist") + persistErr := s.persistNewSession(ctx, sess) + persistDone(persistErr) + if err := persistErr; err != nil { if existing, ok, collisionErr := s.resolveCreateCollision(ctx, sess.ID, owner, request, err); ok || collisionErr != nil { return existing, collisionErr } @@ -2260,6 +2296,30 @@ func (s *Service) createSession(ctx context.Context, mode session.PermissionMode // The owner stamped on the new session: the explicit WithOwner injection, else // the verified principal on the context, else nil (the ownerless no-auth path). owner := resolveOwner(ctx, opts) + // Durable placement allocation is keyed by the final session id. Mint it once + // before Bind; the id is never replaced after a remote allocation succeeds. + finalID := opts.id + generatedID := !opts.idSet + if opts.idSet && finalID == "" { + return nil, fmt.Errorf("%w: session id must not be empty", ErrInvalidArgument) + } + if !opts.idSet { + finalID = s.cfg.NewID() + } + creatediag.Session(ctx, string(finalID)) + probeDone := creatediag.Begin(ctx, "session_id_probe") + var existingCreate *session.Session + var releaseCreate func() + if generatedID { + releaseCreate, err = s.reserveGeneratedSessionID(ctx, finalID) + } else { + existingCreate, releaseCreate, err = s.reserveCreateID(ctx, finalID, owner) + } + probeDone(err) + if err != nil { + return nil, err + } + defer releaseCreate() var placement *PlacementBinding if opts.placement != nil { if err := validatePlacementBinding(*opts.placement); err != nil { @@ -2271,7 +2331,7 @@ func (s *Service) createSession(ctx context.Context, mode session.PermissionMode return nil, ErrInvalidPlacementBinding } } else { - workspace, placement, err = s.bindPlacementForCreate(ctx, profile, owner) + workspace, placement, err = s.bindPlacementForCreate(ctx, profile, owner, finalID) if err != nil { return nil, err } @@ -2286,57 +2346,17 @@ func (s *Service) createSession(ctx context.Context, mode session.PermissionMode return nil, err } - // Resolve the session id: the caller's override (WithSessionID, ADR 0059 - // decision #7 Phase-2) wins; otherwise the Service's NewID generator mints a - // fresh one (the byte-identical pre-Phase-2 path). WithSessionID with an EMPTY - // id is rejected (the doc promises it), distinguished from "never called" by - // idSet. A caller-chosen id is validated + reserved by reserveSessionID (see - // its doc for the three collision sources); the reservation is released on - // EVERY exit path. + // Reserve the final id for the rest of creation. Placement allocation above is + // idempotent on this same key, so a retry cannot allocate a second environment. + request := newCreateRequest(placement.Ref, mode, limits, sel, profile, opts.sourceSessionID, opts) var retryRequest *createRequest - mintID := s.cfg.NewID - if opts.idSet { - if opts.id == "" { - return nil, fmt.Errorf("%w: session id must not be empty", ErrInvalidArgument) - } - request := newCreateRequest(placement.Ref, mode, limits, sel, profile, opts.sourceSessionID, opts) - retryRequest = &request - existing, release, err := s.reserveSessionID(ctx, opts.id, owner, request) - if err != nil { - return nil, err - } - if existing != nil { - return existing, nil - } - defer release() - mintID = func() session.SessionID { return opts.id } - } - // A broker attachment is keyed by the canonical persisted identity. Mint and - // reserve generated IDs before any attachment or catalogue construction. - if s.cfg.MCPBroker != nil && !opts.idSet { - id := mintID() - request := newCreateRequest(placement.Ref, mode, limits, sel, profile, opts.sourceSessionID, opts) - // Populate the outer retryRequest too (not just the local var used for - // reserveSessionID above): persistCreatedSession's collision-retry path - // (resolveCreateCollision) needs a non-nil *createRequest to classify an - // idempotent-retry winner on this generated-id branch, exactly as the - // opts.idSet branch above already does. Before this fix, retryRequest - // stayed nil here (the "request" identifier above is a fresh local, not - // the outer var), so resolveCreateCollision's request==nil guard always - // short-circuited and a genuine ErrSessionAlreadyExists from persistNewSession - // always hard-failed instead of resolving to the existing winner. + if !generatedID { retryRequest = &request - existing, release, reserveErr := s.reserveSessionID(ctx, id, owner, request) - if reserveErr != nil { - return nil, reserveErr + if existingCreate != nil { + return s.classifyCreateWinner(existingCreate, owner, request) } - if existing != nil { - release() - return nil, fmt.Errorf("%w: generated session id %q already exists", ErrInvalidArgument, id) - } - defer release() - mintID = func() session.SessionID { return id } } + mintID := func() session.SessionID { return finalID } // Issue #20 (model-switch context carryover): when a source session is // named, validate it (turn-boundary) and snapshot its conversation ONCE @@ -2446,7 +2466,9 @@ func (s *Service) createPerSessionEngine(ctx context.Context, mintID func() sess } defer s.finalizeBrokerAttachment(broker, &committed) } + factoryDone := creatediag.Begin(ctx, "engine_factory") res, err = s.callSessionEngine(ctx, sel, specs, profile, workspace, mode, brokerTools(broker)) + factoryDone(err) } if err != nil { // Factory maps an unknown/unavailable provider to ErrInvalidArgument; any @@ -2546,6 +2568,15 @@ func (s *Service) createPerSessionEngine(ctx context.Context, mintID func() sess } return persisted, perr } + if placement != nil && placement.Commit != nil { + commitCtx, cancelCommit := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + commitErr := placement.Commit(commitCtx) + cancelCommit() + if commitErr != nil { + s.logPlacementProviderError(ctx, "commit reference", commitErr) + return sess, fmt.Errorf("%w: session %q persisted but reference publication must be retried", ErrInternal, sess.ID) + } + } if broker != nil { commitCtx, cancelCommit := context.WithTimeout(context.WithoutCancel(ctx), engineCloseTimeout) commitErr := s.commitBrokerAttachment(commitCtx, id, broker) @@ -3009,6 +3040,7 @@ func (s *Service) Close() { } else if admissionCancel != nil { admissionCancel() } + s.teardownExecution(rs) } s.waitDetachedControlRelays() @@ -3337,10 +3369,101 @@ func (s *Service) StorageReady(ctx context.Context) bool { return p.Ping(ctx) == nil } +// ReconcileReferenceIntents resolves the provider's bounded, client-scoped +// ambiguous publication and deletion outcomes against exact durable sessions. +func (s *Service) ReconcileReferenceIntents(ctx context.Context) { + lifecycle := s.cfg.ReferenceIntents + if lifecycle == nil { + return + } + intents, err := lifecycle.ListReferenceIntents(ctx, 64) + if err != nil { + s.warnReferenceIntentRetention(ctx) + return + } + retained := false + for _, intent := range intents { + if intent.BindingID == "" || intent.OperationID == "" || !intent.Ref.Valid() || intent.Principal == nil { + retained = true + continue + } + sess, loadErr := s.cfg.Store.Load(ctx, intent.BindingID) + if loadErr != nil { + if errors.Is(loadErr, port.ErrSessionNotFound) && intent.PendingDelete { + if err := lifecycle.ConfirmReferenceIntentDelete(ctx, intent); err != nil { + retained = true + } + } else { + // A missing pending create may have been published by another replica + // after this read. It is never safe to abort or TTL-collect it here. + retained = true + } + continue + } + matching := sess != nil && sess.ID == intent.BindingID && sess.EnvironmentRef == intent.Ref && sess.Owner != nil && sess.Owner.SameIdentity(intent.Principal) + if !matching { + retained = true + continue + } + if intent.PendingDelete { + // Presence after an earlier ambiguous delete attempt is not proof that the + // attempt had no effect: a delayed commit may still remove this incarnation. + // Keep the exact pending intent until absence proves deletion or an explicit + // retry obtains a deterministic outcome. + retained = true + continue + } + err = lifecycle.CommitReferenceIntent(ctx, intent) + if err != nil { + retained = true + } + } + if retained { + s.warnReferenceIntentRetention(ctx) + } else { + s.referenceIntentWarned.Store(false) + } +} + +func (s *Service) warnReferenceIntentRetention(ctx context.Context) { + if s.referenceIntentWarned.CompareAndSwap(false, true) { + s.cfg.Diagnostics.Log(ctx, port.LevelWarn, "execution reference reconciliation retained ambiguous intents") + } +} + func (s *Service) saveSession(ctx context.Context, sess *session.Session) error { return s.cfg.MutationCapability.GuardStore(s.cfg.Store).Save(ctx, sess) } +func (s *Service) prepareReferenceDelete(ctx context.Context, sess *session.Session) (ReferenceDeleteHandle, error) { + if sess == nil || s.cfg.ExecutionAccess == nil || !s.cfg.ExecutionAccess.Applies(sess.EnvironmentRef) { + return nil, nil + } + lifecycle, ok := s.cfg.PlacementProvider.(ReferenceLifecycle) + if !ok { + return nil, ErrPlacementUnavailable + } + return lifecycle.PrepareReferenceDelete(ctx, PlacementSuccessorRequest{Ref: sess.EnvironmentRef, Principal: sess.Owner, SourceBindingID: sess.ID}) +} +func (s *Service) cancelReferenceDelete(ctx context.Context, handle ReferenceDeleteHandle) { + if handle == nil { + return + } + cancelCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + if err := handle.Cancel(cancelCtx); err != nil { + s.logPlacementProviderError(ctx, "cancel reference delete", err) + } +} +func (*Service) confirmReferenceDelete(ctx context.Context, handle ReferenceDeleteHandle) error { + if handle == nil { + return nil + } + confirmCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + return handle.Confirm(confirmCtx) +} + func (s *Service) deleteSessionFamily(ctx context.Context, id session.SessionID, store port.PrunableStore) error { if !s.mutationLeaseHeld(id) { return fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) @@ -3563,10 +3686,17 @@ func (s *Service) DeleteSession(ctx context.Context, id session.SessionID) error if err != nil || absent { return err } + referenceDelete, err := s.prepareReferenceDelete(ctx, sess) + if err != nil { + return err + } unlockBroker := s.brokerMu.lock(id) defer unlockBroker() if err := s.deleteSessionFamily(ctx, sess.ID, prunable); err != nil { if errors.Is(err, port.ErrSessionNotFound) { + if confirmErr := s.confirmReferenceDelete(ctx, referenceDelete); confirmErr != nil { + return fmt.Errorf("%w: confirm deleted session reference: %v", ErrInternal, confirmErr) + } // The durable record is already gone: still attempt broker cleanup // (best-effort) before reporting success, so a locally-retained // broker handle is never orphaned by an already-completed delete. @@ -3582,6 +3712,9 @@ func (s *Service) DeleteSession(ctx context.Context, id session.SessionID) error } return fmt.Errorf("%w: delete session: %v", ErrInternal, err) } + if err := s.confirmReferenceDelete(ctx, referenceDelete); err != nil { + return fmt.Errorf("%w: session deleted but reference confirmation must be retried", ErrInternal) + } // The durable record is gone; broker cleanup is now best-effort. Reordered // deliberately (I-8): deleting broker state FIRST left an unrecoverable // partial-deletion window if the durable delete then failed — the snapshot @@ -3633,6 +3766,10 @@ func (s *Service) DeleteSessionForRetentionCandidate(ctx context.Context, candid if !retentionCandidateMatches(sess, candidate) { return errRetentionCandidateChanged } + referenceDelete, err := s.prepareReferenceDelete(ctx, sess) + if err != nil { + return err + } unlockBroker := s.brokerMu.lock(candidate.ID) defer unlockBroker() if !s.mutationLeaseHeld(candidate.ID) { @@ -3646,8 +3783,12 @@ func (s *Service) DeleteSessionForRetentionCandidate(ctx context.Context, candid return fmt.Errorf("%w: delete retention candidate: %v", ErrInternal, err) } if !deleted { + s.cancelReferenceDelete(ctx, referenceDelete) return errRetentionCandidateChanged } + if err := s.confirmReferenceDelete(ctx, referenceDelete); err != nil { + return fmt.Errorf("%w: retention deleted session but reference confirmation must be retried", ErrInternal) + } // The durable record is gone; broker cleanup is now best-effort (I-8: see // deleteBrokerSessionLocked's doc comment for the ordering rationale). if err := s.deleteBrokerSessionLocked(ctx, candidate.ID); err != nil { @@ -3673,6 +3814,8 @@ func retentionCandidateMatches(sess *session.Session, candidate port.SessionDisc // chats, but it still serializes against run entry, acquires the cross-process // mutation lease, and revalidates durable taxonomy and lifecycle after acquiring // that lease. It is an internal composition callback, never a wire operation. +// +//nolint:gocyclo // Retention keeps lease, taxonomy, reference intent, and deletion in one transaction. func (s *Service) DeleteSessionForRetention(ctx context.Context, id session.SessionID) error { unlock := s.runEntryMu.lock(id) defer unlock() @@ -3703,10 +3846,17 @@ func (s *Service) DeleteSessionForRetention(ctx context.Context, id session.Sess if sess.State == session.StateRunning || sess.State == session.StateAwaiting || sess.State == session.StateAuthorizing || s.IsLive(id) { return fmt.Errorf("%w: retention candidate is active or awaiting approval", ErrFailedPrecondition) } + referenceDelete, err := s.prepareReferenceDelete(ctx, sess) + if err != nil { + return err + } unlockBroker := s.brokerMu.lock(id) defer unlockBroker() if err := s.deleteSessionFamily(ctx, id, prunable); err != nil { if errors.Is(err, port.ErrSessionNotFound) { + if confirmErr := s.confirmReferenceDelete(ctx, referenceDelete); confirmErr != nil { + return fmt.Errorf("%w: confirm deleted retention reference: %v", ErrInternal, confirmErr) + } if brokerErr := s.deleteBrokerSessionLocked(ctx, id); brokerErr != nil { s.cfg.Diagnostics.Log(context.WithoutCancel(ctx), port.LevelWarn, "broker cleanup after already-deleted retention candidate failed", "session", string(id), "err", brokerErr.Error()) @@ -3719,6 +3869,9 @@ func (s *Service) DeleteSessionForRetention(ctx context.Context, id session.Sess } return fmt.Errorf("%w: delete retention candidate: %v", ErrInternal, err) } + if err := s.confirmReferenceDelete(ctx, referenceDelete); err != nil { + return fmt.Errorf("%w: retention deleted session but reference confirmation must be retried", ErrInternal) + } // The durable record is gone; broker cleanup is now best-effort (I-8: see // deleteBrokerSessionLocked's doc comment for the ordering rationale). if err := s.deleteBrokerSessionLocked(ctx, id); err != nil { @@ -4396,11 +4549,8 @@ func (s *Service) RetryFailedRun(ctx context.Context, id session.SessionID) (*ag if err := admitRunPurpose(sess, runPurposeChat); err != nil { return nil, fmt.Errorf("%w: session %q is not eligible for chat retry", ErrFailedStepRetryIneligible, id) } - if registered, ok := s.LookupRun(id); ok { - if !sess.State.IsTerminal() { - return nil, fmt.Errorf("%w: session %q already has an active run", ErrFailedStepRetryIneligible, id) - } - s.deregister(id, registered) + if _, ok := s.LookupRun(id); ok { + return nil, fmt.Errorf("%w: session %q still has an undrained run", ErrFailedStepRetryIneligible, id) } eligibleFailure, err := failedStepRetryEligibility(sess) if err != nil { @@ -4440,6 +4590,10 @@ func (s *Service) RetryFailedRun(ctx context.Context, id session.SessionID) (*ag if !leaseHeld() { return nil, fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) } + env, err = s.acquireExecution(ctx, st, sess, sess.RunID(), env) + if err != nil { + return nil, err + } ctx = memory.WithWorkspace(ctx, env.Workspace().Root()) run, err := s.promoteRunAdmission(id, st, stopAdmission, func() *agent.Run { return engine.RetryFailedStep(ctx, sess, env) @@ -4448,6 +4602,7 @@ func (s *Service) RetryFailedRun(ctx context.Context, id session.SessionID) (*ag return nil, err } promoted = true + s.startExecutionRenewal(id, st, run) return run, nil } @@ -4561,12 +4716,19 @@ func (s *Service) startRunContent(ctx context.Context, id session.SessionID, tex } // The run registry is the authoritative same-process single-run gate while the // loaded aggregate is non-terminal. A terminal snapshot means the registered - // run has finished driving but its relay has not called FinishRun yet. Remove - // that exact run before reopening so the finished relay's later FinishRun - // cannot deregister the continuation that replaces it. + // run has finished driving but its relay has not called FinishRun yet. Local + // and no-FS runs retain the historical fast handoff: remove that exact run + // before reopening so its later FinishRun cannot deregister the continuation. + // Native execution is different: its runState owns the provider claim until + // FinishRun after relay drain, so an undrained native run must remain registered + // and block a replacement claim. if registered, ok := s.LookupRun(id); ok { - if !sess.State.IsTerminal() { - return nil, fmt.Errorf("%w: session %q already has an active run", ErrFailedPrecondition, id) + s.mu.Lock() + st := s.runs[id] + nativeUndrained := st != nil && st.run == registered && s.cfg.ExecutionAccess != nil && s.cfg.ExecutionAccess.Applies(sess.EnvironmentRef) + s.mu.Unlock() + if !sess.State.IsTerminal() || nativeUndrained { + return nil, fmt.Errorf("%w: session %q still has an active or undrained run", ErrFailedPrecondition, id) } s.deregister(id, registered) } @@ -4681,6 +4843,10 @@ func (s *Service) startRunContent(ctx context.Context, id session.SessionID, tex // one mint site for a new run. runID := newRunID() sess.BeginRun(runID) + env, err = s.acquireExecution(ctx, st, sess, runID, env) + if err != nil { + return nil, err + } if !leaseHeld() { return nil, fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) } @@ -4697,6 +4863,7 @@ func (s *Service) startRunContent(ctx context.Context, id session.SessionID, tex *interruptedContinuationOwned = true } promoted = true + s.startExecutionRenewal(id, st, run) return run, nil } @@ -5446,7 +5613,7 @@ func (s *Service) engineAndEnvironmentFor(ctx context.Context, sess *session.Ses if !sess.EnvironmentRef.Valid() { return nil, tool.Environment{}, ErrInvalidPlacementSelection } - verified, err := s.ReattachPlacement(ctx, sess.EnvironmentRef) + verified, err := s.ReattachPlacementForBinding(ctx, sess.EnvironmentRef, sess.ID) if err != nil { return nil, tool.Environment{}, err } @@ -6246,6 +6413,10 @@ func (s *Service) resumeFromAwaiting(ctx context.Context, id session.SessionID, if !leaseHeld() { return nil, fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) } + env, err = s.acquireExecution(ctx, st, sess, sess.RunID(), env) + if err != nil { + return nil, err + } ctx = memory.WithWorkspace(ctx, env.Workspace().Root()) run, err := s.promoteRunAdmission(id, st, stopAdmission, func() *agent.Run { return engine.ResumeApproval(ctx, sess, env, askID, verdict) @@ -6254,6 +6425,7 @@ func (s *Service) resumeFromAwaiting(ctx context.Context, id session.SessionID, return nil, err } promoted = true + s.startExecutionRenewal(id, st, run) return run, nil } @@ -6353,7 +6525,7 @@ func (s *Service) approvePlan(ctx context.Context, id session.SessionID, targetM var resumedStop session.StopReason if resumed != nil { resumedStop = s.forwardRunEvents(ctx, id, resumed, out) - s.deregister(id, resumed) + s.FinishRun(id, resumed) } // (5) Atomic continuation (allow paths only). A deny leaves the session // in plan mode with no continuation run — the model re-plans on the next @@ -6389,7 +6561,7 @@ func (s *Service) approvePlan(ctx context.Context, id session.SessionID, targetM return } s.forwardRunEvents(ctx, id, cont, out) - s.deregister(id, cont) + s.FinishRun(id, cont) }() return out, nil @@ -6710,7 +6882,7 @@ func (s *Service) completeRelay(ctx context.Context, id session.SessionID, run * // disconnected client cannot lose the terminal snapshot, then release the run. func (s *Service) finishRelayRun(ctx context.Context, id session.SessionID, run *agent.Run) { s.completeRelay(ctx, id, run) - s.deregister(id, run) + s.FinishRun(id, run) } // appendEvent durably records one projected relay event to the configured @@ -7003,7 +7175,7 @@ func (s *Service) autoApproveContinuation(ctx context.Context, id session.Sessio recorder.Observe(ev) } recorder.Close() - s.deregister(id, cont) + s.FinishRun(id, cont) } // autoApproveWaitTimeout bounds how long autoApproveContinuation waits for the @@ -7824,6 +7996,7 @@ func (s *Service) cleanupRunAdmission(id session.SessionID, st *runState, promot return } s.cancelRegisteredRunState(id, st, nil, true) + s.teardownExecution(st) s.removeRunState(id, st) } @@ -7904,6 +8077,131 @@ func (s *Service) deregister(id session.SessionID, run *agent.Run) { } } +func (s *Service) acquireExecution(ctx context.Context, st *runState, sess *session.Session, runID string, fallback tool.Environment) (tool.Environment, error) { + access := s.cfg.ExecutionAccess + if access == nil || !access.Applies(sess.EnvironmentRef) { + return fallback, nil + } + handle, err := access.AcquireRun(ctx, ExecutionRunRequest{Ref: sess.EnvironmentRef, Principal: sess.Owner, BindingID: sess.ID, RunID: runID}) + if err != nil { + return tool.Environment{}, err + } + env := handle.Environment() + if env.Ref() != sess.EnvironmentRef || env.Workspace() == nil { + _ = handle.Release(context.WithoutCancel(ctx)) + return tool.Environment{}, ErrInvalidPlacementBinding + } + st.executionMu.Lock() + if st.executionClosed { + st.executionMu.Unlock() + _ = handle.Release(context.WithoutCancel(ctx)) + return tool.Environment{}, ErrPlacementUnavailable + } + st.execution = handle + st.executionMu.Unlock() + return env, nil +} + +type executionRenewalDeadline interface { + RenewalDeadline() time.Time +} + +func executionRenewDelay(handle ExecutionRunHandle, now time.Time) (time.Duration, bool) { + deadlineHandle, ok := handle.(executionRenewalDeadline) + if !ok { + return 20 * time.Second, true + } + remaining := deadlineHandle.RenewalDeadline().Sub(now) + if remaining <= 300*time.Millisecond { + return 0, false + } + delay := remaining / 3 + if delay > 20*time.Second { + delay = 20 * time.Second + } + if delay < 100*time.Millisecond { + delay = 100 * time.Millisecond + } + return delay, delay < remaining +} + +func (s *Service) startExecutionRenewal(_ session.SessionID, st *runState, run *agent.Run) { + st.executionMu.Lock() + if st.execution == nil || st.executionClosed { + st.executionMu.Unlock() + return + } + ctx, cancel := context.WithCancel(context.Background()) + st.executionRenewCancel = cancel + st.executionRenewDone = make(chan struct{}) + done := st.executionRenewDone + handle := st.execution + st.executionMu.Unlock() + go func() { + defer close(done) + for { + delay, usable := executionRenewDelay(handle, time.Now()) + if !usable { + s.logDiscoveryError(context.Background(), "execution ownership renewal failed", errors.New("execution grant has insufficient time remaining")) + run.Cancel() + return + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return + case <-timer.C: + remaining := 10 * time.Second + if deadlineHandle, ok := handle.(executionRenewalDeadline); ok { + remaining = time.Until(deadlineHandle.RenewalDeadline()) / 2 + if remaining <= 0 { + run.Cancel() + return + } + } + renewCtx, stop := context.WithTimeout(ctx, remaining) + err := handle.Renew(renewCtx) + stop() + if err != nil { + s.logDiscoveryError(context.Background(), "execution ownership renewal failed", err) + run.Cancel() + return + } + } + } + }() +} + +func (s *Service) teardownExecution(st *runState) { + if st == nil { + return + } + st.executionTeardown.Do(func() { + st.executionMu.Lock() + st.executionClosed = true + cancelRenew := st.executionRenewCancel + done := st.executionRenewDone + handle := st.execution + st.executionMu.Unlock() + if cancelRenew != nil { + cancelRenew() + } + if done != nil { + <-done + } + if handle != nil { + releaseCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if err := handle.Release(releaseCtx); err != nil { + s.logDiscoveryError(context.Background(), "execution ownership release failed", err) + } + cancel() + } + }) +} + // FinishRun removes run from the in-flight registry for id. It is the EXPORTED // counterpart of register that every wire adapter must call (typically via // `defer`) once it has finished draining run.Events(), so a completed run does @@ -7923,6 +8221,7 @@ func (s *Service) FinishRun(id session.SessionID, run *agent.Run) { if parked && st.sess != nil { pending, pendingOK = st.sess.PendingAuthorization() } + s.teardownExecution(st) s.removeRunState(id, st) if parked { s.scheduleAuthorizationExpiry(id, pending, pendingOK) diff --git a/internal/adapter/server/team_test.go b/internal/adapter/server/team_test.go index 32121cd3b7..448e465f4b 100644 --- a/internal/adapter/server/team_test.go +++ b/internal/adapter/server/team_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "iter" "strings" "sync" "testing" @@ -54,9 +55,9 @@ func (r *teamLivenessTracker) IsLive(id session.SessionID) bool { } // teamService builds a team-enabled Service whose per-member Engine uses the -// supplied mockllm provider (shared by all members) and carries that member's +// supplied provider (shared by all members) and carries that member's // coordination tools. -func teamService(t *testing.T, llm *mockllm.Provider) *server.Service { +func teamService(t *testing.T, llm port.LLMProvider) *server.Service { t.Helper() allow := permpolicy.NewPolicy(permpolicy.AllowAllFloorRules(), nil) memberEngine := func(tm *team.Team, spec agent.MemberSpec, _ string) agent.MemberBuild { @@ -723,13 +724,45 @@ func TestCreateTeamMaxTeams(t *testing.T) { } } +// cleanupTeamBlockingProvider holds a member's actual stream after it has entered +// the runner, making the running-phase probe deterministic. +type cleanupTeamBlockingProvider struct { + entered chan<- struct{} + release <-chan struct{} +} + +func (cleanupTeamBlockingProvider) Capabilities() port.ProviderCapabilities { + return port.ProviderCapabilities{} +} + +func (p cleanupTeamBlockingProvider) Stream(ctx context.Context, _ port.LLMRequest) (iter.Seq2[port.Chunk, error], error) { + return func(yield func(port.Chunk, error) bool) { + if !yield(port.Chunk{Kind: port.ChunkText, Text: "thinking"}, nil) { + return + } + select { + case p.entered <- struct{}{}: + case <-ctx.Done(): + return + } + select { + case <-p.release: + case <-ctx.Done(): + } + }, nil +} + // TestCleanupTeamRejectsRunning asserts Fix D: CleanupTeam on a running team is // rejected with FailedPrecondition (deleting it would orphan the live supervisor), // while a created or done team can be cleaned up and frees its slot. func TestCleanupTeamRejectsRunning(t *testing.T) { - // A member whose single turn blocks until ctx is cancelled keeps the team in the - // running phase for the duration of the probe. - llm := mockllm.New(mockllm.ChunksTurn(blockingChunks()...)) + // The provider signals only after the real member stream has started, then + // remains blocked until the probe has observed CleanupTeam's running guard. + entered := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + defer releaseOnce.Do(func() { close(release) }) + llm := cleanupTeamBlockingProvider{entered: entered, release: release} svc := teamService(t, llm) h := server.NewHarnessServer(svc) ctx := context.Background() @@ -755,22 +788,16 @@ func TestCleanupTeamRejectsRunning(t *testing.T) { runCtx, cancelRun := context.WithCancel(context.Background()) defer cancelRun() - firstEvent := make(chan struct{}, 1) runDone := make(chan struct{}) go func() { defer close(runDone) - _, _ = svc.RunTeam(runCtx, teamID, func(agent.TeamEvent) { - select { - case firstEvent <- struct{}{}: - default: - } - }) + _, _ = svc.RunTeam(runCtx, teamID, func(agent.TeamEvent) {}) }() select { - case <-firstEvent: + case <-entered: case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for the team run to start") + t.Fatal("timed out waiting for the member stream to start") } // CleanupTeam on the running team is rejected (FailedPrecondition). @@ -781,6 +808,7 @@ func TestCleanupTeamRejectsRunning(t *testing.T) { t.Fatalf("Service.CleanupTeam(running): err = %v, want ErrTeamRunning", serr) } + releaseOnce.Do(func() { close(release) }) cancelRun() select { case <-runDone: diff --git a/internal/app/build.go b/internal/app/build.go index ae170d437d..96dba602a9 100644 --- a/internal/app/build.go +++ b/internal/app/build.go @@ -144,6 +144,9 @@ type Config struct { // PlacementScope is the trusted authorization scope passed to the provider. // Empty defaults to the process deployment scope. PlacementScope server.PlacementScope + // RemoteExecution selects the remote-environment catalog attenuation and + // model-visible posture. It is set only by an explicit composition root. + RemoteExecution bool // ClientMCPOnCreate permits client-provided MCP servers on a session-creating // API request (issue #821, ADR 0237 applied to outbound MCP). It is a // deployment policy the cmd/ main decides from its listener topology and Build passes through verbatim; the zero value fails @@ -2155,18 +2158,21 @@ func Build(ctx context.Context, cfg Config) (*Built, error) { sessionReadLedger = redisBackend.ReadLedger } worktreeLister := buildWorktreeLister(cfg) + selectorIssuer, err := server.NewWorktreeSelectorIssuer(placementSelectorKey[:]) + if err != nil { + return nil, fmt.Errorf("initialize placement selector issuer: %w", err) + } + localProvider := &localPlacementProvider{ + scope: placementScope, root: cfg.Workspace, workspace: workspaceFactory, + runnerForRoot: func(root string) tool.CommandRunner { + return buildCommandRunnerForRoot(cfg, root) + }, + worktrees: worktreeLister, selectors: selectorIssuer, + } if placementProvider == nil { - selectorIssuer, err := server.NewWorktreeSelectorIssuer(placementSelectorKey[:]) - if err != nil { - return nil, fmt.Errorf("initialize placement selector issuer: %w", err) - } - placementProvider = &localPlacementProvider{ - scope: placementScope, root: cfg.Workspace, workspace: workspaceFactory, - runnerForRoot: func(root string) tool.CommandRunner { - return buildCommandRunnerForRoot(cfg, root) - }, - worktrees: worktreeLister, selectors: selectorIssuer, - } + placementProvider = localProvider + } else if cfg.RemoteExecution { + placementProvider = &profilePlacementProvider{remote: placementProvider, local: localProvider} } attempts := startAttemptRecovery(ctx, cfg, reg, store, eventLog, assets.attemptRepository, assets.reflectionRepository, assets, placementProvider, placementScope) if attempts != nil { @@ -2207,6 +2213,8 @@ func Build(ctx context.Context, cfg Config) (*Built, error) { PlacementProvider: placementProvider, PlacementScope: placementScope, + ExecutionAccess: executionAccess(placementProvider), + ReferenceIntents: referenceIntentLifecycle(placementProvider), SessionReadLedger: sessionReadLedger, RootAuthority: func(kind session.SessionKind) session.Authority { return mintRootAuthority(assets.rootCatalog, mcpResourceCapabilities(assets.globalMgr), kind) @@ -2714,6 +2722,7 @@ func Build(ctx context.Context, cfg Config) (*Built, error) { // subagent-*/parallel-*/team-* children the run-entry funnel's own repair // (Step 3) never sees. See internal/app/session_reconcile.go. staleSessionReconcileClose := startStaleSessionReconcile(cfg, svc) + referenceIntentReconcileClose := startReferenceIntentReconcile(ctx, svcCfg.ReferenceIntents, svc) // Close tears down the main MCP manager AND any per-session client-MCP engines // still registered (svc.Close), so a process exit leaks neither. It also cancels @@ -2724,6 +2733,7 @@ func Build(ctx context.Context, cfg Config) (*Built, error) { if assets.reflectionLifecycle != nil { assets.reflectionLifecycle.close() } + referenceIntentReconcileClose() staleSessionReconcileClose() childGCClose() managedTempWorkerClose() @@ -3010,6 +3020,19 @@ func sessionEngineFactory( } } +func remoteSessionConfiguration(cfg Config, profile server.SessionProfile, workspace string, instructions prompt.InstructionAssembler, policy port.PermissionPolicy) (string, bool, prompt.InstructionAssembler, port.PermissionPolicy) { + if !cfg.RemoteExecution || profile == server.ProfileNoFS { + return workspace, false, instructions, policy + } + if variants, ok := instructions.(sessionInstructionSet); ok { + instructions = variants.remote + } + if variants, ok := policy.(sessionPermissionPolicies); ok { + policy = variants.remote + } + return "", true, instructions, policy +} + func sessionEngineFactoryWithTools( cfg Config, reg *providerRegistry, @@ -3034,7 +3057,8 @@ func sessionEngineFactoryWithTools( // is scoped to this one session's assembly; the SHARED engine keeps the // build-time pin over cfg.Workspace (the server's own root — the one // the shared engine was assembled for). - cfg.childPermResolver = childPermResolverFor(cfg, workspace) + projectWorkspace, remote, sessionInstructions, sessionPolicy := remoteSessionConfiguration(cfg, profile, workspace, instructions, policy) + cfg.childPermResolver = childPermResolverFor(cfg, projectWorkspace) // The NO-FS profile (issue #55): the service routes every no-fs session // through this factory unconditionally (the shared engine has the FS tools // baked in), and the profile selects the no-FS catalog assembly + the @@ -3203,7 +3227,7 @@ func sessionEngineFactoryWithTools( // Caller-scoped lazy hydration: rebuild this authenticated session's global // and admitted project generations from durable state before binding the // Skill tool. A failed authoritative read clears only these partitions. - skillPartitions := hydrateLearnedSkillPartitions(ctx, cfg, assets, workspace) + skillPartitions := hydrateLearnedSkillPartitions(ctx, cfg, assets, projectWorkspace) cat, closeFn, clientToolNames := assembleCatalog(ctx, cfg, reg, store, hooks, &assets, catalogSession{ provider: resolvedProvider, @@ -3212,6 +3236,7 @@ func sessionEngineFactoryWithTools( clientMgr: mgr, narrate: false, noFS: noFS, + remote: remote, mode: mode, skillPartitions: skillPartitions, sessionTools: sessionTools, @@ -3223,13 +3248,14 @@ func sessionEngineFactoryWithTools( // the shared wiring, so no collaborator is silently dropped and a non-default // provider never contaminates compaction/counting. learningCfg := cfg - learningCfg.Workspace = workspace - learningCfg.LearningMode, learningCfg.LearningSensitivity, learningCfg.SkillActivationPolicy = learningPolicyForWorkspace(cfg, workspace) + learningCfg.Workspace = projectWorkspace + learningCfg.LearningMode, learningCfg.LearningSensitivity, learningCfg.SkillActivationPolicy = learningPolicyForWorkspace(cfg, projectWorkspace) learningCfg.Model = resolvedModel learningCfg.attemptRepository = assets.attemptRepository learningCfg.automaticAdmissionLedger = assets.automaticAdmissionLedger learningCfg.learningSourceStore = store - deps := engineDepsForProvider(cfg, resolvedProvider, resolvedModel, windowFn, store, policy, hooks, mcpProvider, instructions) + deps := engineDepsForProvider(cfg, resolvedProvider, resolvedModel, windowFn, store, sessionPolicy, hooks, mcpProvider, sessionInstructions) + deps = applyRemoteExecutionPosture(deps, remote) attachOperatorProfile(&deps, assets.userModelStore) deps.LearningMode = learningCfg.LearningMode deps.LearningObserver = bindMaterializationLifecycle(buildReflectionObserver(learningCfg, resolvedProvider, learningCfg.Model, assets.userModelStore, assets.memStore, assets.reflectionRepository, assets.reflectionCoordinator, assets.learningAdmission, buildProcedureProcessor(learningCfg, assets)), assets.reflectionLifecycle) @@ -4106,7 +4132,15 @@ func buildEngine(ctx context.Context, cfg Config, reg *providerRegistry, provide // the tier-0 project MemoryIndexAssembler. Operator facts no longer ride a // turn-0 user fragment; the same user store is loaded per request into the // volatile system-prompt suffix below. - instructions := buildInstructionAssembler(rulesSrc, soulSrc, memStore, userModelStore, !projectIngestionAdmitted(cfg)) + remoteRules := prompt.RulesSource(nil) + if cfg.RemoteExecution { + // Project admission is off for this deployment; retain user-tier rules. + remoteRules = rulesSrc + } + instructions := sessionInstructionSet{ + standard: buildInstructionAssembler(rulesSrc, soulSrc, memStore, userModelStore, !projectIngestionAdmitted(cfg)), + remote: buildInstructionAssembler(remoteRules, soulSrc, memStore, userModelStore, true), + } // Guardrails decorate the ordinary hook chain. Completion learning has its own // synchronous engine seam and no longer shares Stop-hook ownership. @@ -4140,7 +4174,12 @@ func buildEngine(ctx context.Context, cfg Config, reg *providerRegistry, provide // recursion guard and operator-tier-only config carry over); the option is a // no-op at any non-auto posture or with no checker, so yolo/strict/trusted // and the un-knobbed auto stay byte-identical. - sharedPolicy := escapePolicyForConfig(cfg, reg, provider, policy) + sharedPolicy := sessionPermissionPolicies{ + standard: escapePolicyForConfig(cfg, reg, provider, policy), + // Pin no workspace: retain operator permissions without project discovery. + remote: escapePolicyForConfig(cfg, reg, provider, + permpolicy.NewPolicyWithResolver(mainRules(cfg), learned, childPermResolverFor(cfg, ""), mainEvaluatorOptions(cfg)...)), + } var learningAdmission *learningAdmission if cfg.operatorLearningMode != learning.Off { learningAdmission = newLearningAdmission(cfg.UserModelReviewInterval) @@ -4219,6 +4258,27 @@ func attachOperatorProfile(deps *agent.Deps, store tool.MemoryStore) { } } +type sessionPermissionPolicies struct { + standard port.PermissionPolicy + remote port.PermissionPolicy +} + +func (p sessionPermissionPolicies) Evaluate(ctx context.Context, id session.SessionID, mode session.PermissionMode, call session.ToolCall, ws tool.WorkspaceReader) governance.PermissionDecision { + return p.standard.Evaluate(ctx, id, mode, call, ws) +} +func (p sessionPermissionPolicies) Learn(id session.SessionID, call session.ToolCall) { + p.standard.Learn(id, call) +} + +type sessionInstructionSet struct { + standard prompt.InstructionAssembler + remote prompt.InstructionAssembler +} + +func (s sessionInstructionSet) Assemble(ctx context.Context, ws tool.Workspace) ([]session.Message, error) { + return s.standard.Assemble(ctx, ws) +} + // buildInstructionAssembler composes the ephemeral turn-0 instruction fragments: // RootAssembler (project instructions), rules, soul, then the project memory // index. userModelStore remains in the internal signature to keep existing @@ -4635,6 +4695,9 @@ func buildCommandExpander(cfg Config, mcpProvider mcp.Provider) prompt.CommandEx // and exactly reattaches the owned session, this lister opens a fresh osfs Workspace // at that provider-verified private root so discovery reflects current command files. func buildCommandLister(cfg Config, mcpProvider mcp.Provider) server.CommandLister { + if cfg.RemoteExecution { + return nil + } exp := buildCommandExpander(cfg, mcpProvider) lister, ok := exp.(prompt.CommandLister) if !ok { @@ -5355,7 +5418,7 @@ func compactionDecision(cfg Config) diagFact { // here with the composition-resolved provider (or the not-configured sentinel), // NOT as a zero-value tools.All()/NoFS() entry — an only-when-configured tool that // vanishes would be the silent-disable this harness avoids. -func registerCoreTools(cfg Config, cat *tool.Catalog, log, noFS bool, searchProvider tool.SearchProvider) { +func registerCoreTools(cfg Config, cat *tool.Catalog, log, noFS, remote bool, searchProvider tool.SearchProvider) { if noFS { for _, t := range tools.NoFS() { cat.MustRegister(t) @@ -5367,7 +5430,8 @@ func registerCoreTools(cfg Config, cat *tool.Catalog, log, noFS bool, searchProv cat.MustRegister(t) } cat.MustRegister(tools.NewWebSearchTool(searchProvider)) - if runner := buildCommandRunner(cfg); runner != nil { + runner := buildCommandRunner(cfg) + if runner != nil || remote { // The AGENT-loop Shell tool (not the fstools one): foreground byte-identical, // plus the `background: true` detach over the run's child registry. Its // companion ShellStatus — the SOLE status/collect/cancel channel for those @@ -5377,7 +5441,11 @@ func registerCoreTools(cfg Config, cat *tool.Catalog, log, noFS bool, searchProv cat.MustRegister(agent.NewShellTool()) cat.MustRegister(agent.NewShellStatusTool()) if log { - cfg.diag().Log(context.Background(), port.LevelInfo, "Shell tool ENABLED", "shell", cfg.Shell, "cwd", cfg.Workspace) + if remote { + cfg.diag().Log(context.Background(), port.LevelInfo, "Shell tool ENABLED", "shell", "remote execution") + } else { + cfg.diag().Log(context.Background(), port.LevelInfo, "Shell tool ENABLED", "shell", cfg.Shell, "cwd", cfg.Workspace) + } } } else if log { cfg.diag().Log(context.Background(), port.LevelInfo, "Shell tool DISABLED (shell-less mode): the agent has no command execution", @@ -5687,6 +5755,7 @@ func buildCatalog(ctx context.Context, cfg Config, reg *providerRegistry, provid provider: provider, providerID: reg.Default(), model: cfg.Model, + remote: cfg.RemoteExecution, narrate: true, }) // Aggregate the build-time Subagent per-def INLINE MCP managers' teardown into @@ -8176,6 +8245,22 @@ func applyRedisWorkspacePosture(pc prompt.Config, enabled bool) prompt.Config { const workspaceRootForPrompt = "/workspace" +const remoteExecutionPostureNote = "This session uses a persistent remote Kubernetes workspace. Use the filesystem tools and foreground Shell for work in /workspace. Local project instructions, rules, project-scoped skills, commands, schedules, SkillDraft, Parallel, Team, and Subagent delegation are unavailable; operator-global skills, MCP, memory, and web tools remain available. Never assume harness-local files are part of this workspace." + +func applyRemoteExecutionPosture(deps agent.Deps, enabled bool) agent.Deps { + if !enabled { + return deps + } + deps.CommandExpander = prompt.NoopExpander{} + pc := &deps.PromptConfig + pc.Env.Cwd, pc.Env.Shell, pc.Env.GitStatus = workspaceRootForPrompt, "remote foreground shell", "" + if pc.Role == "" { + pc.Role = prompt.DefaultRole() + } + pc.Role += "\n\n" + remoteExecutionPostureNote + return deps +} + func applyDebugSessionPosture(pc prompt.Config, target session.SessionID, selectedServers []string) prompt.Config { if pc.Role == "" { pc.Role = prompt.DefaultRole() @@ -8460,7 +8545,9 @@ const ( "the terminal client, which starts an embedded server by default or attaches to a remote one via " + "`mecatui connect ADDRESS`; mecated the general-purpose gRPC and HTTP/SSE server; mecak8s the " + "Kubernetes-native server keeping session state in Redis; mecatequi a one-shot CI task returning " + - "a patch." + "a patch; mecatl-execution-provider the authenticated Kubernetes control plane that owns persistent " + + "execution-environment lifecycle; mecatl-executor the credential-free workload helper that performs " + + "one confined filesystem or command operation inside an executor Pod." // selfKnowledgePostureAxes is the load-bearing content clause: the three safety // axes, kept distinct. Conflating the per-session permission mode with the diff --git a/internal/app/catalog.go b/internal/app/catalog.go index 3d407ff7dd..f11b5833ae 100644 --- a/internal/app/catalog.go +++ b/internal/app/catalog.go @@ -167,7 +167,8 @@ type catalogSession struct { // EXACTLY as in the default profile — guarded by TestNoFSCatalogProfile, // which pins the EXACT name-set delta. Always false for the build-time shared // catalog (a process always has a default-profile shared engine). - noFS bool + noFS bool + remote bool // mode is the session's permission mode (the per-session factory passes the // session's resolved mode; the build-time shared catalog leaves it // ModeDefault). The Schedule registration reads it to register the @@ -214,7 +215,7 @@ func assembleCatalog(ctx context.Context, cfg Config, reg *providerRegistry, sto classified := newClassifiedCatalog() cat := classified.catalog classified.captureEach(coreToolClassification, func() { - registerCoreTools(cfg, cat, s.narrate, s.noFS, a.searchProvider) + registerCoreTools(cfg, cat, s.narrate, s.noFS, s.remote, a.searchProvider) }) for _, sessionTool := range s.sessionTools { // A direct global manager retains ownership of its existing query wrapper. @@ -285,27 +286,31 @@ func assembleCatalog(ctx context.Context, cfg Config, reg *providerRegistry, sto refMgr = s.clientMgr } var subagentClose func() error - classified.captureEach(delegationToolClassification, func() { - subagentClose = registerSubagentTrio(ctx, cfg, cat, reg, store, hooks, *a, s, refMgr) - }) - // Parallel is ABSENT under the no-FS profile (not merely disarmed): every - // branch is a force-copy filesystem fork and the deliverable is a preserved - // fork PATH — both meaningless without a filesystem. - if !s.noFS { + if !s.remote { + classified.captureEach(delegationToolClassification, func() { + subagentClose = registerSubagentTrio(ctx, cfg, cat, reg, store, hooks, *a, s, refMgr) + }) + } + // Parallel is ABSENT under no-FS and remote execution profiles. + if !s.noFS && !s.remote { classified.captureEach(delegationToolClassification, func() { registerParallelTool(ctx, cfg, cat, reg, store, hooks, *a, s) }) } - classified.captureEach(delegationToolClassification, func() { - registerTeamTools(ctx, cfg, cat, reg, store, *a, s, refMgr) - }) + if !s.remote { + classified.captureEach(delegationToolClassification, func() { + registerTeamTools(ctx, cfg, cat, reg, store, *a, s, refMgr) + }) + } classified.capture(server.ClassificationEntry{Kind: server.KindCallerOwned, Rationale: "memory tools resolve the verified caller through the caller-partitioned store"}, func() { registerMemoryFamilies(ctx, cfg, cat, *a) }) - classified.captureEach(scheduleToolClassification, func() { - registerScheduleTool(ctx, cfg, cat, a, s) - }) + if !s.remote { + classified.captureEach(scheduleToolClassification, func() { + registerScheduleTool(ctx, cfg, cat, a, s) + }) + } classified.captureEach(func(t tool.Tool) (server.ClassificationEntry, bool) { return skillToolClassification(t, *a, s) }, func() { @@ -646,7 +651,7 @@ func registerSkillFamily(ctx context.Context, cfg Config, cat *tool.Catalog, a c cfg.diag().Log(ctx, port.LevelWarn, "registering skills failed; Skill tool disabled", "err", err) } } - if !s.noFS { + if !s.noFS && !s.remote { if a.learnedSkills != nil && cfg.SkillsDraftDir != "" { inventory := make([]learning.SkillInventoryItem, 0, len(a.skills)) for _, meta := range a.skills { diff --git a/internal/app/placement.go b/internal/app/placement.go index fbecac9187..2a6cf94380 100644 --- a/internal/app/placement.go +++ b/internal/app/placement.go @@ -18,6 +18,105 @@ const ( defaultPlacementScope = server.PlacementScope("deployment") ) +func referenceIntentLifecycle(provider server.PlacementProvider) server.ReferenceIntentLifecycle { + if profiled, ok := provider.(*profilePlacementProvider); ok { + lifecycle, _ := profiled.remote.(server.ReferenceIntentLifecycle) + return lifecycle + } + lifecycle, _ := provider.(server.ReferenceIntentLifecycle) + return lifecycle +} + +func executionAccess(provider server.PlacementProvider) server.ExecutionAccess { + access, _ := provider.(server.ExecutionAccess) + return access +} + +type profilePlacementProvider struct { + remote server.PlacementProvider + local *localPlacementProvider +} + +func (p *profilePlacementProvider) ValidatePlacement(ctx context.Context) error { + if validator, ok := p.remote.(server.PlacementValidator); ok { + return validator.ValidatePlacement(ctx) + } + return nil +} +func (p *profilePlacementProvider) Bind(ctx context.Context, req server.PlacementBindRequest) (server.PlacementBinding, error) { + if req.Selector.IsNoFS() { + return p.local.Bind(ctx, req) + } + return p.remote.Bind(ctx, req) +} +func (p *profilePlacementProvider) Reattach(ctx context.Context, req server.PlacementReattachRequest) (server.PlacementBinding, error) { + if req.Ref.Kind == session.EnvKindNoFS { + return p.local.Reattach(ctx, req) + } + remote, ok := p.remote.(server.PlacementReattacher) + if !ok { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + return remote.Reattach(ctx, req) +} + +func (p *profilePlacementProvider) PrepareReferenceDelete(ctx context.Context, req server.PlacementSuccessorRequest) (server.ReferenceDeleteHandle, error) { + lifecycle, ok := p.remote.(server.ReferenceLifecycle) + if !ok || !lifecycle.Applies(req.Ref) { + return nil, server.ErrPlacementUnavailable + } + return lifecycle.PrepareReferenceDelete(ctx, req) +} + +func (p *profilePlacementProvider) ListReferenceIntents(ctx context.Context, limit int) ([]server.ReferenceIntent, error) { + lifecycle, ok := p.remote.(server.ReferenceIntentLifecycle) + if !ok { + return nil, server.ErrPlacementUnavailable + } + return lifecycle.ListReferenceIntents(ctx, limit) +} +func (p *profilePlacementProvider) CommitReferenceIntent(ctx context.Context, intent server.ReferenceIntent) error { + lifecycle, ok := p.remote.(server.ReferenceIntentLifecycle) + if !ok { + return server.ErrPlacementUnavailable + } + return lifecycle.CommitReferenceIntent(ctx, intent) +} +func (p *profilePlacementProvider) ConfirmReferenceIntentDelete(ctx context.Context, intent server.ReferenceIntent) error { + lifecycle, ok := p.remote.(server.ReferenceIntentLifecycle) + if !ok { + return server.ErrPlacementUnavailable + } + return lifecycle.ConfirmReferenceIntentDelete(ctx, intent) +} +func (p *profilePlacementProvider) CancelReferenceIntentDelete(ctx context.Context, intent server.ReferenceIntent) error { + lifecycle, ok := p.remote.(server.ReferenceIntentLifecycle) + if !ok { + return server.ErrPlacementUnavailable + } + return lifecycle.CancelReferenceIntentDelete(ctx, intent) +} + +func (p *profilePlacementProvider) ReserveSuccessor(ctx context.Context, req server.PlacementSuccessorRequest) (server.PlacementBinding, error) { + reservoir, ok := p.remote.(server.PlacementSuccessorReservoir) + if !ok { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + return reservoir.ReserveSuccessor(ctx, req) +} + +func (p *profilePlacementProvider) Applies(ref session.EnvironmentRef) bool { + access, ok := p.remote.(server.ExecutionAccess) + return ok && access.Applies(ref) +} +func (p *profilePlacementProvider) AcquireRun(ctx context.Context, req server.ExecutionRunRequest) (server.ExecutionRunHandle, error) { + access, ok := p.remote.(server.ExecutionAccess) + if !ok || !access.Applies(req.Ref) { + return nil, server.ErrPlacementUnavailable + } + return access.AcquireRun(ctx, req) +} + // localPlacementProvider is the trusted composition default. It owns exactly // one configured local record plus the no-FS attenuation; it has no inventory // registry or path-derived public identifier. diff --git a/internal/app/project_ingestion.go b/internal/app/project_ingestion.go index 19ca62f0ee..ef365fc40a 100644 --- a/internal/app/project_ingestion.go +++ b/internal/app/project_ingestion.go @@ -14,7 +14,9 @@ package app // shell also reads cfg.TrustProject directly because both decisions intentionally // express the same operator vouch for the workspace and its .git. func projectIngestionAdmitted(cfg Config) bool { - return cfg.TrustProject + // Local project trust cannot authorize source ingestion for a remote + // deployment; its explicit no-FS attenuation has no project source either. + return cfg.TrustProject && !cfg.RemoteExecution } // projectIngestionAdmittedForRoot binds the single project-ingestion decision to diff --git a/internal/app/reference_intent_reconcile.go b/internal/app/reference_intent_reconcile.go new file mode 100644 index 0000000000..03fd722a8a --- /dev/null +++ b/internal/app/reference_intent_reconcile.go @@ -0,0 +1,36 @@ +package app + +import ( + "context" + "time" + + "github.com/stacklok/mecatl/internal/adapter/server" +) + +const referenceIntentReconcileInterval = time.Minute + +func startReferenceIntentReconcile(ctx context.Context, lifecycle server.ReferenceIntentLifecycle, svc *server.Service) func() { + if lifecycle == nil || svc == nil { + return func() {} + } + workerCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + defer close(done) + svc.ReconcileReferenceIntents(workerCtx) + ticker := time.NewTicker(referenceIntentReconcileInterval) + defer ticker.Stop() + for { + select { + case <-workerCtx.Done(): + return + case <-ticker.C: + svc.ReconcileReferenceIntents(workerCtx) + } + } + }() + return func() { + cancel() + <-done + } +} diff --git a/internal/app/remote_create_test.go b/internal/app/remote_create_test.go new file mode 100644 index 0000000000..b9fd6352b7 --- /dev/null +++ b/internal/app/remote_create_test.go @@ -0,0 +1,116 @@ +package app + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/adapter/memfs" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/permconfig" + "github.com/stacklok/mecatl/internal/adapter/server" +) + +func TestRemoteHTTPCreateRealOpenRouterDoesNotWaitForDiscoveryOrInfer(t *testing.T) { + fakeRulesEnv(t, t.TempDir(), t.TempDir()) + t.Setenv("XDG_DATA_HOME", t.TempDir()) + t.Setenv("XDG_STATE_HOME", t.TempDir()) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + started, release := make(chan struct{}), make(chan struct{}) + var releaseOnce, startedOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer unblock() + live := &http.Client{Transport: nativeRoundTripFunc(func(r *http.Request) (*http.Response, error) { + startedOnce.Do(func() { close(started) }) + select { + case <-release: + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"data":[]}`))}, nil + case <-r.Context().Done(): + return nil, r.Context().Err() + } + })} + var inference atomic.Int64 + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + inference.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer endpoint.Close() + ref := session.EnvironmentRef{Kind: "kubernetes", ID: "private-ref-sentinel", Revision: "private-revision-sentinel"} + env := tool.MustEnvironment(ref, memfs.NewWorkspace("/workspace"), memledger.New(), nil) + diag := &attrCapturingDiag{} + binds := 0 + built, err := buildIsolated(t, ctx, Config{ + // No selector, no launch root and no mock: /workspace must use the real + // per-session factory, exactly like the native live create request. + RemoteExecution: true, PlacementScope: "remote", PlacementProvider: remoteFactoryPlacement{env: env, binds: &binds}, + DefaultProvider: "openrouter", Model: "anthropic/claude-haiku-4.5", NoSoul: true, NoUserModel: true, + OpenRouterKey: "synthetic-offline-key", envDetector: fakeEnv(nil), Diagnostics: diag, + ProviderOverrides: permconfig.ProviderOverrides{providerOpenRouter: {BaseURL: endpoint.URL + "/v1"}}, + liveModelHTTPClient: live, + }) + if err != nil { + t.Fatal(err) + } + defer built.Close() + // Release discovery before closing Build, including the failure path. + defer unblock() + select { + case <-started: + case <-ctx.Done(): + t.Fatal("actual registry did not start live discovery") + } + principal := &session.Principal{Issuer: "private-issuer-sentinel", Subject: "private-owner-sentinel", GrantType: session.GrantTypeUser} + req := httptest.NewRequest(http.MethodPost, "/v1/sessions", strings.NewReader(`{"mode":"default","limits":{"max_turns":8,"max_tool_calls":20,"max_consecutive_failures":3}}`)) + req = req.WithContext(session.WithPrincipal(ctx, principal)) + response := httptest.NewRecorder() + done := make(chan struct{}) + go func() { server.NewHTTPHandler(built.Service).ServeHTTP(response, req); close(done) }() + select { + case <-done: + case <-ctx.Done(): + unblock() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("create did not unwind after discovery release and cancellation") + } + t.Fatalf("create blocked behind discovery; stages: %s", diag.dump()) + } + if response.Code != http.StatusCreated || binds != 1 { + t.Fatalf("create status=%d binds=%d", response.Code, binds) + } + var result struct { + ID string `json:"session_id"` + Model struct { + Provider string `json:"provider_id"` + } `json:"resolved_model"` + } + if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil || result.Model.Provider != "openrouter" || result.ID == "" { + t.Fatal("not a real OpenRouter session response") + } + persisted, err := built.Service.GetSession(req.Context(), session.SessionID(result.ID)) + if err != nil || persisted.EnvironmentRef != ref || len(persisted.Conversation.Messages) != 0 || inference.Load() != 0 { + t.Fatal("create inferred, recorded a prompt, or lost exact placement") + } + stages := diag.dump() + for _, contract := range []string{"stage http_handler reason begin", "stage session_id_probe reason ok", "stage engine_factory reason ok", "stage session_persist reason ok", "stage http_response reason ok"} { + if !strings.Contains(stages, contract) { + t.Errorf("actual built factory missing diagnostic %q", contract) + } + } + for _, line := range strings.Split(stages, "\n") { + if strings.Contains(line, "remote create stage") && (strings.Contains(line, "sentinel") || strings.Contains(line, "/workspace") || strings.Contains(line, "synthetic-offline-key")) { + t.Fatal("private create data reached diagnostics") + } + } +} diff --git a/internal/app/remote_execution_test.go b/internal/app/remote_execution_test.go new file mode 100644 index 0000000000..b4a85d4ce1 --- /dev/null +++ b/internal/app/remote_execution_test.go @@ -0,0 +1,325 @@ +package app + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/stacklok/mecatl/engine/adapter/memfs" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" +) + +type remoteFactoryPlacement struct { + env tool.Environment + binds *int + validateErr error +} + +func (p remoteFactoryPlacement) ValidatePlacement(context.Context) error { return p.validateErr } +func (p remoteFactoryPlacement) Bind(_ context.Context, req server.PlacementBindRequest) (server.PlacementBinding, error) { + if p.binds != nil { + *p.binds++ + } + if req.BindingID == "" || req.Principal == nil { + return server.PlacementBinding{}, server.ErrInvalidPlacementSelection + } + return server.PlacementBinding{Environment: p.env, Ref: p.env.Ref(), Metadata: server.PlacementMetadata{Label: "Remote Kubernetes workspace"}}, nil +} +func (p remoteFactoryPlacement) Reattach(_ context.Context, req server.PlacementReattachRequest) (server.PlacementBinding, error) { + if req.BindingID == "" || req.Ref != p.env.Ref() { + return server.PlacementBinding{}, server.ErrPlacementNotFound + } + return server.PlacementBinding{Environment: p.env, Ref: p.env.Ref()}, nil +} + +func TestRemoteExecutionPreflightNeverAllocates(t *testing.T) { + fakeRulesEnv(t, t.TempDir(), t.TempDir()) + for _, validation := range []error{nil, server.ErrInvalidPlacementBinding, server.ErrPlacementUnavailable} { + binds := 0 + built, err := buildIsolated(t, t.Context(), Config{UseMock: true, NoSoul: true, RemoteExecution: true, PlacementScope: "remote", PlacementProvider: remoteFactoryPlacement{binds: &binds, validateErr: validation}}) + if built != nil { + built.Close() + } + if validation == nil && err != nil || validation != nil && !errors.Is(err, validation) { + t.Fatalf("validation=%v startup error=%v", validation, err) + } + if binds != 0 { + t.Fatalf("preflight allocated %d environments", binds) + } + } +} + +func TestRemoteDeploymentNoFSUsesLocalAttenuationWithoutProviderCall(t *testing.T) { + fakeRulesEnv(t, t.TempDir(), t.TempDir()) + binds := 0 + remoteWS := memfs.NewWorkspace("/workspace") + remoteRef := session.EnvironmentRef{Kind: "kubernetes", ID: "env-1", Revision: "rev-1"} + remoteEnv := tool.MustEnvironment(remoteRef, remoteWS, memledger.New(), nil) + var captured port.LLMRequest + provider := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { captured = req })}, + mockllm.ToolCallTurn(session.ToolCall{ID: "read", Name: "Read", Args: json.RawMessage(`{"path":"secret"}`)}), + mockllm.TextTurn("done"), + ) + built, err := buildIsolated(t, context.Background(), Config{MockProvider: provider, PlacementProvider: remoteFactoryPlacement{env: remoteEnv, binds: &binds}, PlacementScope: "remote", RemoteExecution: true, NoSoul: true, SchedulerEnabled: false}) + if err != nil { + t.Fatal(err) + } + defer built.Close() + ctx := session.WithPrincipal(context.Background(), &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser}) + sess, err := built.Service.CreateSessionWithProfile(ctx, session.ModeAccept, session.Limits{}, server.ProviderSelector{}, server.ProfileNoFS) + if err != nil { + t.Fatal(err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "no files") + if err != nil { + t.Fatal(err) + } + for range run.Events() { + } + built.Service.FinishRun(sess.ID, run) + if binds != 0 { + t.Fatalf("remote provider Bind calls = %d, want zero", binds) + } + if !strings.Contains(captured.System.StablePrefix, noFSPostureNote) || strings.Contains(captured.System.StablePrefix, remoteExecutionPostureNote) { + t.Fatalf("no-fs posture was replaced by remote posture: %q", captured.System.StablePrefix) + } + for _, spec := range captured.Tools { + if spec.Name == "Read" || spec.Name == "Shell" { + t.Fatalf("no-fs catalog exposed %q", spec.Name) + } + } +} + +type remotePermissionRunner struct { + *memfs.CommandRunner + root string +} + +func (r remotePermissionRunner) BoundWorkspaceRoot() string { return r.root } + +func TestRemoteExecutionPreservesOperatorPermissionsUnderAuto(t *testing.T) { + for _, source := range []string{"global", "explicit"} { + t.Run(source, func(t *testing.T) { + xdg := t.TempDir() + fakeRulesEnv(t, xdg, t.TempDir()) + t.Setenv("XDG_STATE_HOME", t.TempDir()) + operatorPath := filepath.Join(xdg, "mecatl", "settings.yaml") + if source == "explicit" { + operatorPath = filepath.Join(t.TempDir(), "operator.yaml") + } + writeProjectFile(t, operatorPath, "permissions:\n deny:\n - 'Shell(echo denied)'\n ask:\n - 'Shell(echo asked)'\n") + project := t.TempDir() + projectRules := "permissions:\n deny:\n - 'Shell(echo allowed)'\n ask:\n - 'Shell(echo project-ask)'\n allow:\n - 'Shell(echo denied)'\n - 'Shell(echo asked)'\n" + mkdirProjectSettings(t, project, projectRules) + ws := memfs.NewWorkspace(project) + if _, err := ws.CreateFile(t.Context(), ".mecatl/settings.yaml", []byte(projectRules)); err != nil { + t.Fatal(err) + } + runner := memfs.NewCommandRunner() + runner.SetResult(&tool.CommandResult{Stdout: "EXECUTED"}, nil) + ref := session.EnvironmentRef{Kind: "kubernetes", ID: "permissions", Revision: "v1"} + env := tool.MustEnvironment(ref, ws, memledger.New(), remotePermissionRunner{runner, project}) + llm := mockllm.New( + mockllm.ToolCallTurn( + session.NewToolCall("denied", "Shell", json.RawMessage(`{"command":"echo denied"}`)), + session.NewToolCall("asked", "Shell", json.RawMessage(`{"command":"echo asked"}`)), + session.NewToolCall("allowed", "Shell", json.RawMessage(`{"command":"echo allowed"}`)), + session.NewToolCall("project-ask", "Shell", json.RawMessage(`{"command":"echo project-ask"}`)), + ), mockllm.TextTurn("done")) + cfg := Config{MockProvider: llm, Workspace: project, Posture: PostureAuto, PermissionsConventional: true, NoSoul: true, RemoteExecution: true, PlacementScope: "remote", PlacementProvider: remoteFactoryPlacement{env: env}} + if source == "explicit" { + cfg.PermissionConfigs = []string{operatorPath} + } + built, err := buildIsolated(t, t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + defer built.Close() + ctx := session.WithPrincipal(t.Context(), &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser}) + sess, err := built.Service.CreateSession(ctx, session.ModeDefault, session.Limits{}) + if err != nil { + t.Fatal(err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "check permissions") + if err != nil { + t.Fatal(err) + } + asks := 0 + results := map[session.ToolCallID]session.ToolResult{} + for ev := range run.Events() { + if ev.Ask != nil { + asks++ + if ev.Ask.Call != "asked" || !ev.Ask.ConfiguredAsk { + t.Errorf("unexpected permission ask: %+v", ev.Ask) + } + if _, err := built.Service.ApproveRun(ctx, sess.ID, ev.Ask.AskID, session.VerdictDeny, ""); err != nil { + t.Fatal(err) + } + } + if ev.ToolResult != nil { + results[ev.ToolResult.CallID] = *ev.ToolResult + } + } + built.Service.FinishRun(sess.ID, run) + if asks != 1 { + t.Errorf("configured asks = %d, want 1", asks) + } + for _, id := range []session.ToolCallID{"denied", "asked", "allowed", "project-ask"} { + result, ok := results[id] + blocked := id == "denied" || id == "asked" + if !ok || result.IsError != blocked || strings.Contains(result.Content, "EXECUTED") == blocked { + t.Errorf("%s result = %+v, present=%t, want blocked=%t", id, result, ok, blocked) + } + } + }) + } +} + +func TestRemoteExecutionRealFactoryCarriesPostureAndAttenuatedCatalog(t *testing.T) { + xdg := t.TempDir() + fakeRulesEnv(t, xdg, t.TempDir()) + t.Setenv("XDG_STATE_HOME", t.TempDir()) + writeUserRule(t, xdg, "operator-rule", "OPERATOR_RULE_MARKER\n") + writeSkill(t, filepath.Join(xdg, "mecatl/skills"), "operator-skill", "Operator skill", "OPERATOR_SKILL_BODY") + localProject := t.TempDir() + initTestRepo(t, localProject) + localSnapshotSeen := false + localLLM := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { + localSnapshotSeen = strings.Contains(req.System.Render(), "initial commit") && strings.Contains(req.System.Render(), "") + })}, mockllm.TextTurn("local done")) + local, err := buildIsolated(t, t.Context(), Config{MockProvider: localLLM, Workspace: localProject, TrustProject: true, NoSoul: true, Shell: "/bin/sh"}) + if err != nil { + t.Fatal(err) + } + defer local.Close() + localSession, err := local.Service.CreateSession(t.Context(), session.ModeDefault, session.Limits{}) + if err != nil { + t.Fatal(err) + } + localRun, err := local.Service.StartRun(t.Context(), localSession.ID, "inspect local") + if err != nil { + t.Fatal(err) + } + for range localRun.Events() { + } + local.Service.FinishRun(localSession.ID, localRun) + if !localSnapshotSeen { + t.Fatal("local factory omitted the test repository's Git snapshot") + } + ws := memfs.NewWorkspace(localProject) + if _, err := ws.CreateFile(context.Background(), "AGENTS.md", []byte("REMOTE_PROJECT_MARKER_DO_NOT_LOAD")); err != nil { + t.Fatal(err) + } + if _, err := ws.CreateFile(context.Background(), "main.go", []byte("package main\n")); err != nil { + t.Fatal(err) + } + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env-1", Revision: "rev-1"} + env := tool.MustEnvironment(ref, ws, memledger.New(), nil) + var mu sync.Mutex + var captured port.LLMRequest + provider := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { mu.Lock(); captured = req; mu.Unlock() })}, + mockllm.ToolCallTurn( + session.ToolCall{ID: "Subagent", Name: "Subagent", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "Parallel", Name: "Parallel", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "Team", Name: "Team", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "SkillDraft", Name: "SkillDraft", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "Schedule", Name: "Schedule", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "project-skill", Name: "Skill", Args: json.RawMessage(`{"name":"project-only"}`)}, + session.ToolCall{ID: "operator-skill", Name: "Skill", Args: json.RawMessage(`{"name":"operator-skill"}`)}, + ), + mockllm.ToolCallTurn(session.ToolCall{ID: "read", Name: "Read", Args: json.RawMessage(`{"path":"main.go"}`)}), + mockllm.TextTurn(""), + mockllm.TextTurn("done"), + ) + if err := os.WriteFile(filepath.Join(localProject, "AGENTS.md"), []byte("LOCAL_PROJECT_MARKER_DO_NOT_LOAD\n"), 0o600); err != nil { + t.Fatal(err) + } + writeProjectRule(t, localProject, "project-rule", "LOCAL_RULE_MARKER_DO_NOT_LOAD\n") + writeSkill(t, filepath.Join(localProject, ".mecatl/skills"), "project-only", "LOCAL_SKILL_MARKER_DO_NOT_LOAD", "LOCAL_SKILL_BODY_DO_NOT_LOAD") + commandDir := filepath.Join(localProject, ".claude", "commands") + if err := os.MkdirAll(commandDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(commandDir, "project-only.md"), []byte("LOCAL_COMMAND_MARKER_DO_NOT_LOAD"), 0o600); err != nil { + t.Fatal(err) + } + built, err := buildIsolated(t, context.Background(), Config{MockProvider: provider, PlacementProvider: remoteFactoryPlacement{env: env}, PlacementScope: "remote", RemoteExecution: true, Workspace: localProject, TrustProject: true, AllowAllTools: true, EnableCommands: true, SkillsConventional: true, NoSoul: true, SchedulerEnabled: false}) + if err != nil { + t.Fatal(err) + } + defer built.Close() + principal := &session.Principal{Issuer: "https://issuer.example", Subject: "alice", GrantType: session.GrantTypeUser} + ctx := session.WithPrincipal(context.Background(), principal) + sess, err := built.Service.CreateSession(ctx, session.ModeAccept, session.Limits{}) + if err != nil { + t.Fatal(err) + } + if commands, err := built.Service.ListCommandsForSession(ctx, sess.ID); err != nil || len(commands) != 0 { + t.Fatalf("remote command discovery fell back locally: %v, %v", commands, err) + } + if _, err := built.Service.ListWorktreesForSession(ctx, sess.ID); !errors.Is(err, server.ErrPlacementUnavailable) { + t.Fatalf("remote worktree discovery did not reject unsupported placement: %v", err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "/project-only") + if err != nil { + t.Fatal(err) + } + for range run.Events() { + } + built.Service.FinishRun(sess.ID, run) + mu.Lock() + defer mu.Unlock() + if !strings.Contains(captured.System.StablePrefix, remoteExecutionPostureNote) { + t.Fatal("real per-session factory omitted remote execution posture") + } + if strings.Contains(captured.System.Render(), "") { + t.Fatal("remote factory ingested the local Git snapshot") + } + requestJSON, err := json.Marshal(captured) + if err != nil { + t.Fatal(err) + } + for _, marker := range []string{"LOCAL_PROJECT_MARKER", "REMOTE_PROJECT_MARKER", "LOCAL_RULE_MARKER", "LOCAL_SKILL_MARKER", "LOCAL_SKILL_BODY", "LOCAL_COMMAND_MARKER", "initial commit"} { + if strings.Contains(string(requestJSON), marker) { + t.Errorf("remote request ingested forbidden project source %s", marker) + } + } + if !strings.Contains(string(requestJSON), "OPERATOR_RULE_MARKER") || !strings.Contains(string(requestJSON), "OPERATOR_SKILL_BODY") { + t.Fatal("remote request dropped operator-global rules or skills") + } + failed := map[string]bool{} + for _, message := range captured.Messages { + if result := message.ToolResult; result != nil { + failed[string(result.CallID)] = result.IsError + } + } + for _, id := range []string{"Subagent", "Parallel", "Team", "SkillDraft", "Schedule", "project-skill"} { + if !failed[id] { + t.Errorf("unsupported invocation %s did not return a model-visible error", id) + } + } + names := map[string]bool{} + for _, spec := range captured.Tools { + names[spec.Name] = true + } + for _, name := range []string{"Read", "Write", "WebFetch"} { + if !names[name] { + t.Errorf("remote catalog omitted compatible tool %q", name) + } + } + for _, name := range []string{"Subagent", "SubagentStatus", "InspectSubagent", "Parallel", "Team", "InspectMember", "SkillDraft", "Schedule"} { + if names[name] { + t.Errorf("remote catalog advertised unsupported tool %q", name) + } + } +} diff --git a/internal/creatediag/create.go b/internal/creatediag/create.go new file mode 100644 index 0000000000..8eca29be54 --- /dev/null +++ b/internal/creatediag/create.go @@ -0,0 +1,59 @@ +// Package creatediag records content-free, request-scoped remote-create progress. +package creatediag + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "time" + + "github.com/stacklok/mecatl/engine/port" +) + +type key struct{} +type trace struct { + diag port.Diagnostics + start time.Time + session string +} + +// Start is used only by the remote HTTP create handler, after authentication. +func Start(ctx context.Context, diag port.Diagnostics) context.Context { + return context.WithValue(ctx, key{}, &trace{diag: diag, start: time.Now()}) +} + +// Session hashes even caller-supplied IDs; no owner or placement identity is logged. +// The create request is sequential; the trace never escapes its request context. +func Session(ctx context.Context, id string) { + if t, ok := ctx.Value(key{}).(*trace); ok { + sum := sha256.Sum256([]byte(id)) + t.session = hex.EncodeToString(sum[:16]) + } +} + +// Note accepts only harness-owned stage/reason literals, never producer text. +func Note(ctx context.Context, stage, reason string, calls int) { + if t, ok := ctx.Value(key{}).(*trace); ok { + t.diag.Log(ctx, port.LevelDebug, "remote create stage", "stage", stage, + "reason", reason, "elapsed_ms", time.Since(t.start).Milliseconds(), + "calls", calls, "session", t.session) + } +} + +// Begin emits both sides of a blocking boundary without exposing its error text. +func Begin(ctx context.Context, stage string) func(error) { + Note(ctx, stage, "begin", 0) + return func(err error) { + reason := "ok" + switch { + case errors.Is(err, context.Canceled): + reason = "cancelled" + case errors.Is(err, context.DeadlineExceeded): + reason = "deadline" + case err != nil: + reason = "error" + } + Note(ctx, stage, reason, 1) + } +} diff --git a/internal/executionenv/errors.go b/internal/executionenv/errors.go new file mode 100644 index 0000000000..6a65285a82 --- /dev/null +++ b/internal/executionenv/errors.go @@ -0,0 +1,126 @@ +package executionenv + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "strings" +) + +// ErrorCode is a stable private-protocol error classification. +type ErrorCode string + +// Private-protocol error codes. +const ( + CodeInvalidArgument ErrorCode = "invalid_argument" + CodeUnauthenticated ErrorCode = "unauthenticated" + CodePermissionDenied ErrorCode = "permission_denied" + CodeNotFound ErrorCode = "not_found" + CodeAlreadyExists ErrorCode = "already_exists" + CodeConflict ErrorCode = "conflict" + CodeVersionMismatch ErrorCode = "version_mismatch" + CodeDirectoryNotEmpty ErrorCode = "directory_not_empty" + CodeNotReady ErrorCode = "not_ready" + CodeFenceUnknown ErrorCode = "fence_unknown" + CodeResourceExhausted ErrorCode = "resource_exhausted" + CodeInternal ErrorCode = "internal" +) + +// Valid reports whether c is part of the closed private-protocol vocabulary. +func (c ErrorCode) Valid() bool { + switch c { + case CodeInvalidArgument, CodeUnauthenticated, CodePermissionDenied, CodeNotFound, + CodeAlreadyExists, CodeConflict, CodeVersionMismatch, CodeDirectoryNotEmpty, CodeNotReady, CodeFenceUnknown, + CodeResourceExhausted, CodeInternal: + return true + default: + return false + } +} + +// Error is a bounded structured private-protocol failure. +type Error struct { + Code ErrorCode `json:"code"` + Message string `json:"message"` + Retryable bool `json:"retryable,omitempty"` +} + +func (e *Error) Error() string { return string(e.Code) + ": " + e.Message } + +// DecodeStrict decodes exactly one bounded JSON value for the credential-free +// provider-to-workload stdin protocol, without unknown or duplicate fields. +func DecodeStrict(data []byte, dst any) error { + if len(data) > MaxJSONBody { + return &Error{Code: CodeResourceExhausted, Message: "request body exceeds limit"} + } + if err := rejectDuplicateKeys(data); err != nil { + return &Error{Code: CodeInvalidArgument, Message: "invalid JSON: " + err.Error()} + } + d := json.NewDecoder(strings.NewReader(string(data))) + d.DisallowUnknownFields() + if err := d.Decode(dst); err != nil { + return &Error{Code: CodeInvalidArgument, Message: "invalid JSON: " + err.Error()} + } + if err := d.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return &Error{Code: CodeInvalidArgument, Message: "request must contain exactly one JSON value"} + } + return nil +} + +func rejectDuplicateKeys(data []byte) error { + d := json.NewDecoder(strings.NewReader(string(data))) + if err := walkJSON(d); err != nil { + return err + } + if _, err := d.Token(); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("trailing JSON value") + } + return err + } + return nil +} +func walkJSON(d *json.Decoder) error { + tok, err := d.Token() + if err != nil { + return err + } + delim, ok := tok.(json.Delim) + if !ok { + return nil + } + switch delim { + case '{': + seen := map[string]struct{}{} + for d.More() { + t, err := d.Token() + if err != nil { + return err + } + k, ok := t.(string) + if !ok { + return errors.New("object key is not a string") + } + if _, ok := seen[k]; ok { + return fmt.Errorf("duplicate field %q", k) + } + seen[k] = struct{}{} + if err := walkJSON(d); err != nil { + return err + } + } + _, err = d.Token() + return err + case '[': + for d.More() { + if err := walkJSON(d); err != nil { + return err + } + } + _, err = d.Token() + return err + default: + return errors.New("unexpected delimiter") + } +} diff --git a/internal/executionenv/grant.go b/internal/executionenv/grant.go new file mode 100644 index 0000000000..7da228d2e2 --- /dev/null +++ b/internal/executionenv/grant.go @@ -0,0 +1,181 @@ +package executionenv + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +const grantAlgorithm = "Ed25519" + +var ( + // ErrGrantExpired identifies the only grant failure eligible for a read-only refresh. + ErrGrantExpired = errors.New("grant expired") + // ErrGrantNotYetValid identifies a grant whose validity window has not started. + ErrGrantNotYetValid = errors.New("grant not yet valid") +) + +type grantHeader struct { + Algorithm string `json:"alg"` + KeyID string `json:"kid"` + Type string `json:"typ"` +} + +// GrantClaims binds a short-lived capability to one client, owner, environment, and epoch. +type GrantClaims struct { + KeyID string `json:"-"` + Issuer string `json:"iss"` + Audience string `json:"aud"` + Client string `json:"client"` + OwnerHash string `json:"owner"` + BindingID string `json:"binding_id"` + RunID string `json:"run_id"` + ClaimID string `json:"claim_id"` + Environment EnvironmentRef `json:"environment"` + Epoch uint64 `json:"epoch"` + GrantGeneration uint64 `json:"grant_generation"` + Operations []Operation `json:"operations"` + NotBefore time.Time `json:"not_before"` + ExpiresAt time.Time `json:"expires_at"` + Nonce string `json:"nonce"` +} + +// GrantExpectation defines the exact binding and operation required by a request. +type GrantExpectation struct { + Client string + OwnerHash string + BindingID string + RunID string + ClaimID string + Environment EnvironmentRef + Epoch uint64 + GrantGeneration uint64 + Operation Operation +} + +// GrantVerifier verifies signed grants against configured trust and revocation state. +type GrantVerifier struct { + Keys map[string]ed25519.PublicKey + RevokedKeys map[string]struct{} + RevokedNonces map[string]struct{} + Issuer string + Audience string + MaxLifetime time.Duration + Now func() time.Time +} + +// SignGrant validates and signs claims with Ed25519. +func SignGrant(key ed25519.PrivateKey, c GrantClaims) (string, error) { + if len(key) != ed25519.PrivateKeySize { + return "", errors.New("invalid Ed25519 private key") + } + if err := validateClaims(c); err != nil { + return "", err + } + h, _ := json.Marshal(grantHeader{Algorithm: grantAlgorithm, KeyID: c.KeyID, Type: "MECATL-GRANT"}) + p, _ := json.Marshal(c) + unsigned := rawURL(h) + "." + rawURL(p) + sig := ed25519.Sign(key, []byte(unsigned)) + return unsigned + "." + rawURL(sig), nil +} + +// Verify validates a grant and its exact request binding. +func (v GrantVerifier) Verify(token string, e GrantExpectation) (GrantClaims, error) { //nolint:gocyclo // Fail-closed claim validation is intentionally linear. + var c GrantClaims + parts := strings.Split(token, ".") + if len(parts) != 3 { + return c, errors.New("malformed grant") + } + hb, err := decodeURL(parts[0]) + if err != nil { + return c, errors.New("malformed grant header") + } + var h grantHeader + if err := DecodeStrict(hb, &h); err != nil { + return c, errors.New("malformed grant header") + } + if h.Algorithm != grantAlgorithm || h.Type != "MECATL-GRANT" || h.KeyID == "" { + return c, errors.New("unsupported grant header") + } + if _, ok := v.RevokedKeys[h.KeyID]; ok { + return c, errors.New("grant key revoked") + } + key, ok := v.Keys[h.KeyID] + if !ok || len(key) != ed25519.PublicKeySize { + return c, errors.New("unknown grant key") + } + sig, err := decodeURL(parts[2]) + if err != nil || !ed25519.Verify(key, []byte(parts[0]+"."+parts[1]), sig) { + return c, errors.New("invalid grant signature") + } + pb, err := decodeURL(parts[1]) + if err != nil { + return c, errors.New("malformed grant claims") + } + if err := DecodeStrict(pb, &c); err != nil { + return c, errors.New("malformed grant claims") + } + c.KeyID = h.KeyID + if err := validateClaims(c); err != nil { + return c, err + } + now := time.Now().UTC() + if v.Now != nil { + now = v.Now().UTC() + } + if c.Issuer != v.Issuer || c.Audience != v.Audience { + return c, errors.New("grant issuer or audience mismatch") + } + if now.Before(c.NotBefore) { + return c, ErrGrantNotYetValid + } + if !now.Before(c.ExpiresAt) { + return c, ErrGrantExpired + } + maxLifetime := v.MaxLifetime + if maxLifetime <= 0 { + maxLifetime = 5 * time.Minute + } + if c.ExpiresAt.Sub(c.NotBefore) > maxLifetime { + return c, errors.New("grant lifetime exceeds limit") + } + if _, ok := v.RevokedNonces[c.Nonce]; ok { + return c, errors.New("grant revoked") + } + if c.Client != e.Client || c.OwnerHash != e.OwnerHash || c.BindingID != e.BindingID || c.RunID != e.RunID || c.ClaimID != e.ClaimID || c.Environment != e.Environment || c.Epoch != e.Epoch || c.GrantGeneration != e.GrantGeneration { + return c, errors.New("grant binding mismatch") + } + for _, op := range c.Operations { + if op == e.Operation { + return c, nil + } + } + return c, errors.New("grant does not authorize operation") +} + +//nolint:gocyclo // Complete fail-closed claim validation is deliberately linear. +func validateClaims(c GrantClaims) error { + if c.KeyID == "" || c.Issuer == "" || c.Audience == "" || c.Client == "" || c.OwnerHash == "" || c.BindingID == "" || c.RunID == "" || c.ClaimID == "" || c.Environment.ID == "" || c.Environment.Revision == "" || c.Epoch == 0 || c.GrantGeneration == 0 || c.Nonce == "" || c.NotBefore.IsZero() || c.ExpiresAt.IsZero() || !c.ExpiresAt.After(c.NotBefore) { + return errors.New("incomplete grant claims") + } + if len(c.Operations) == 0 || len(c.Operations) > 32 { + return errors.New("invalid grant operations") + } + seen := map[Operation]struct{}{} + for _, op := range c.Operations { + if !op.Valid() { + return fmt.Errorf("invalid grant operation %q", op) + } + if _, ok := seen[op]; ok { + return errors.New("duplicate grant operation") + } + seen[op] = struct{}{} + } + return nil +} +func rawURL(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) } +func decodeURL(s string) ([]byte, error) { return base64.RawURLEncoding.Strict().DecodeString(s) } diff --git a/internal/executionenv/protocol_test.go b/internal/executionenv/protocol_test.go new file mode 100644 index 0000000000..862780f625 --- /dev/null +++ b/internal/executionenv/protocol_test.go @@ -0,0 +1,67 @@ +package executionenv + +import ( + "crypto/ed25519" + "crypto/rand" + "testing" + "time" +) + +func TestGrantRoundTripAndExactBindings(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + now := time.Unix(1_800_000_000, 0).UTC() + claims := GrantClaims{KeyID: "k1", Issuer: "execution-provider", Audience: "mecatl-execution", Client: "spiffe://cluster/ns/mecak8s", OwnerHash: "owner", BindingID: "binding", RunID: "run", ClaimID: "claim", Environment: EnvironmentRef{ID: "env", Revision: "rev"}, Epoch: 7, GrantGeneration: 1, Operations: []Operation{OpFileRead}, NotBefore: now.Add(-time.Second), ExpiresAt: now.Add(time.Minute), Nonce: "n1"} + token, err := SignGrant(priv, claims) + if err != nil { + t.Fatal(err) + } + v := GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: claims.Issuer, Audience: claims.Audience, MaxLifetime: 5 * time.Minute, Now: func() time.Time { return now }} + if _, err := v.Verify(token, GrantExpectation{Client: claims.Client, OwnerHash: "owner", BindingID: claims.BindingID, RunID: claims.RunID, ClaimID: claims.ClaimID, Environment: claims.Environment, Epoch: 7, GrantGeneration: claims.GrantGeneration, Operation: OpFileRead}); err != nil { + t.Fatalf("verify: %v", err) + } + for name, mutate := range map[string]func(*GrantExpectation){ + "client": func(e *GrantExpectation) { e.Client = "spiffe://other" }, + "owner": func(e *GrantExpectation) { e.OwnerHash = "other" }, + "binding": func(e *GrantExpectation) { e.BindingID = "other" }, + "revision": func(e *GrantExpectation) { e.Environment.Revision = "other" }, + "epoch": func(e *GrantExpectation) { e.Epoch++ }, + "operation": func(e *GrantExpectation) { e.Operation = OpFileReplace }, + } { + t.Run(name, func(t *testing.T) { + e := GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: claims.BindingID, RunID: claims.RunID, ClaimID: claims.ClaimID, Environment: claims.Environment, Epoch: claims.Epoch, GrantGeneration: claims.GrantGeneration, Operation: OpFileRead} + mutate(&e) + if _, err := v.Verify(token, e); err == nil { + t.Fatal("expected rejection") + } + }) + } +} + +func TestGrantRejectsExpiryRevocationAndUnknownFields(t *testing.T) { + pub, priv, _ := ed25519.GenerateKey(rand.Reader) + now := time.Unix(1_800_000_000, 0).UTC() + c := GrantClaims{KeyID: "k1", Issuer: "i", Audience: "a", Client: "spiffe://c", OwnerHash: "o", BindingID: "b", RunID: "run", ClaimID: "claim", Environment: EnvironmentRef{ID: "e", Revision: "r"}, Epoch: 1, GrantGeneration: 1, Operations: []Operation{OpAttach}, NotBefore: now.Add(-time.Minute), ExpiresAt: now.Add(-time.Second), Nonce: "n"} + tok, _ := SignGrant(priv, c) + v := GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: "i", Audience: "a", Now: func() time.Time { return now }} + if _, err := v.Verify(tok, GrantExpectation{Client: c.Client, OwnerHash: c.OwnerHash, BindingID: c.BindingID, RunID: c.RunID, ClaimID: c.ClaimID, Environment: c.Environment, Epoch: 1, GrantGeneration: c.GrantGeneration, Operation: OpAttach}); err == nil { + t.Fatal("expired grant accepted") + } + c.ExpiresAt = now.Add(time.Minute) + tok, _ = SignGrant(priv, c) + v.RevokedNonces = map[string]struct{}{"n": {}} + if _, err := v.Verify(tok, GrantExpectation{Client: c.Client, OwnerHash: c.OwnerHash, BindingID: c.BindingID, RunID: c.RunID, ClaimID: c.ClaimID, Environment: c.Environment, Epoch: 1, GrantGeneration: c.GrantGeneration, Operation: OpAttach}); err == nil { + t.Fatal("revoked nonce accepted") + } +} + +func TestExecutorStdinDecodeStrictRejectsUnknownTrailingAndOversize(t *testing.T) { + var req ExecutorRequest + for _, body := range []string{`{"operation":"file.read","extra":true}`, `{"operation":"file.read"}{}`, `{"path":"` + string(make([]byte, MaxJSONBody)) + `"}`} { + if err := DecodeStrict([]byte(body), &req); err == nil { + t.Fatalf("accepted invalid body") + } + } +} diff --git a/internal/executionenv/types.go b/internal/executionenv/types.go new file mode 100644 index 0000000000..b451344cf1 --- /dev/null +++ b/internal/executionenv/types.go @@ -0,0 +1,325 @@ +// Package executionenv defines the private, versioned host-side execution-provider protocol. +// It is intentionally not part of the public Harness API. +// +//nolint:revive // Private protocol types are exported only across internal adapter packages. +package executionenv + +import "time" + +// Private protocol bounds. +const ( + ProtocolVersion = "execution-grpc/1" + MaxMessageBytes = 8 << 20 + MaxJSONBody = 8 << 20 // credential-free provider-to-workload stdin framing only + MaxFileBytes = 5 << 20 + MaxCommandBytes = 1 << 20 + MaxPathBytes = 4096 + MaxIdentityBytes = 1024 + MaxBindingBytes = 253 + MaxGrantBytes = 16 << 10 + MaxListEntries = 10_000 +) + +// Operation identifies one provider or workload-helper operation. +type Operation string + +// Supported private-protocol operations. +const ( + OpAttach Operation = "attach" + OpAcquireRun Operation = "run.acquire" + OpRenewRun Operation = "run.renew" + OpReleaseRun Operation = "run.release" + OpReferenceCommit Operation = "reference.commit" + OpReferenceAbort Operation = "reference.abort" + OpReferenceReserve Operation = "reference.reserve" + OpReferenceDeletePrepare Operation = "reference.delete.prepare" + OpReferenceDeleteConfirm Operation = "reference.delete.confirm" + OpReferenceDeleteCancel Operation = "reference.delete.cancel" + OpReferenceRelease Operation = "reference.release" + OpRetire Operation = "retire" + OpFileRead Operation = "file.read" + OpFileResolveAuthority Operation = "file.resolve_authority" + OpFileStat Operation = "file.stat" + OpFileCreate Operation = "file.create" + OpFileReplace Operation = "file.replace" + OpFileList Operation = "file.list" + OpFileRemove Operation = "file.remove" + OpFileRename Operation = "file.rename" + OpFileCopy Operation = "file.copy" + OpFileGlob Operation = "file.glob" + OpFileGrep Operation = "file.grep" + OpCommandStart Operation = "command.start" + OpCommandStatus Operation = "command.status" + OpCommandCancel Operation = "command.cancel" + OpCommandStream Operation = "command.stream" +) + +// Valid reports whether the operation belongs to the closed protocol vocabulary. +func (o Operation) Valid() bool { + switch o { + case OpAttach, OpAcquireRun, OpRenewRun, OpReleaseRun, OpReferenceCommit, OpReferenceAbort, OpReferenceReserve, OpReferenceDeletePrepare, OpReferenceDeleteConfirm, OpReferenceDeleteCancel, OpReferenceRelease, OpRetire, OpFileRead, OpFileResolveAuthority, OpFileStat, OpFileCreate, OpFileReplace, OpFileList, OpFileRemove, OpFileRename, OpFileCopy, OpFileGlob, OpFileGrep, OpCommandStart, OpCommandStatus, OpCommandCancel, OpCommandStream: + return true + } + return false +} + +// EnvironmentRef is the provider-private durable environment identity. +type EnvironmentRef struct { + ID string `json:"id"` + Revision string `json:"revision"` +} + +// Owner is an authenticated principal attestation. +type Owner struct { + Issuer string `json:"issuer"` + Subject string `json:"subject"` +} + +// RequestContext carries the exact authorization binding for an operation. +type RequestContext struct { + Environment EnvironmentRef `json:"environment"` + Owner Owner `json:"owner"` + BindingID string `json:"binding_id"` + RunID string `json:"run_id"` + ClaimID string `json:"claim_id"` + Epoch uint64 `json:"epoch"` + GrantGeneration uint64 `json:"grant_generation"` + Grant string `json:"grant"` +} + +// ValidateProfileRequest selects an operator-defined profile for validation. +type ValidateProfileRequest struct { + Profile string `json:"profile"` +} + +// ValidateProfileResponse reports immutable profile capabilities and bounds. +type ValidateProfileResponse struct { + Profile string `json:"profile"` + Digest string `json:"digest"` + Capabilities []string `json:"capabilities"` + MaxFileBytes int64 `json:"max_file_bytes"` + MaxCommandBytes int64 `json:"max_command_bytes"` + MaxCommandDurationMillis int64 `json:"max_command_duration_ms"` +} + +// EnsureEnvironmentRequest requests an idempotent environment allocation. +type EnsureEnvironmentRequest struct { + BindingID string `json:"binding_id"` + Profile string `json:"profile"` + Owner Owner `json:"owner"` +} + +// EnsureEnvironmentResponse returns allocation identity, readiness, and a short-lived grant. +type EnsureEnvironmentResponse struct { + Environment EnvironmentRef `json:"environment"` + Epoch uint64 `json:"epoch"` + Ready bool `json:"ready"` + GrantGeneration uint64 `json:"grant_generation"` + Grant string `json:"grant"` + GrantExpiresAt time.Time `json:"grant_expires_at"` +} + +// AttachEnvironmentRequest requests exact reattachment to an existing environment. +type AttachEnvironmentRequest struct { + Context RequestContext `json:"context"` + Purpose string `json:"purpose"` +} + +// PurposeSession is the supported attachment purpose. +const PurposeSession = "session" + +// AttachEnvironmentResponse returns exact attachment state and a refreshed grant. +type AttachEnvironmentResponse struct { + Environment EnvironmentRef `json:"environment"` + Epoch uint64 `json:"epoch"` + Ready bool `json:"ready"` + GrantGeneration uint64 `json:"grant_generation"` + Grant string `json:"grant"` + GrantExpiresAt time.Time `json:"grant_expires_at"` +} + +const ( + MinRunTTL = 30 * time.Second + MaxRunTTL = 5 * time.Minute + DefaultRunTTL = time.Minute + DefaultRenewInterval = 20 * time.Second +) + +type RunClaimRequest struct { + Environment EnvironmentRef + Owner Owner + BindingID string + RunID string + ClaimID string + Epoch uint64 + GrantGeneration uint64 + OperationID string + TTL time.Duration +} + +type RunClaim struct { + Environment EnvironmentRef + BindingID string + RunID string + ClaimID string + Epoch uint64 + GrantGeneration uint64 + Grant string + ExpiresAt time.Time +} + +type ReferenceState string + +const ( + ReferencePendingCreate ReferenceState = "PendingCreate" + ReferencePublished ReferenceState = "Published" + ReferencePendingDelete ReferenceState = "PendingDelete" +) + +type ReferenceRequest struct { + Environment EnvironmentRef + Owner Owner + BindingID string + SourceBindingID string + OperationID string +} + +type ReferenceIntent struct { + Environment EnvironmentRef + Owner Owner + BindingID string + State ReferenceState + OperationID string + SourceBindingID string + CreatedAt time.Time +} + +// ReferenceReleaseRequest releases one durable binding reference. +type ReferenceReleaseRequest struct { + Context RequestContext `json:"context"` +} + +// RetireEnvironmentRequest requests controlled retirement without PVC deletion. +type RetireEnvironmentRequest struct { + Environment EnvironmentRef `json:"environment"` + Owner Owner `json:"owner"` + ExpectedEpoch uint64 `json:"expected_execution_epoch"` + ExpectedPodUID string `json:"expected_pod_uid"` + ExpectedPVCUID string `json:"expected_pvc_uid"` + OperationID string `json:"operation_id"` +} + +// EmptyResponse is the successful response for operations without a payload. +type EmptyResponse struct{} + +// FileRequest carries one bounded, authorized filesystem operation. +type FileRequest struct { + Context RequestContext `json:"context"` + Operation Operation `json:"operation"` + Path string `json:"path"` + Destination string `json:"destination,omitempty"` + Pattern string `json:"pattern,omitempty"` + Data []byte `json:"data,omitempty"` + Version string `json:"version,omitempty"` + Limit int `json:"limit,omitempty"` +} + +// FileInfo is bounded filesystem metadata returned by the helper. +type FileInfo struct { + Name string `json:"name"` + Size int64 `json:"size"` + Mode uint32 `json:"mode"` + ModTime time.Time `json:"mod_time"` + IsDir bool `json:"is_dir"` +} + +// GrepMatch is one bounded textual grep result. +type GrepMatch struct { + Path string `json:"path"` + Line int `json:"line"` + Text string `json:"text"` +} + +// FileResponse carries the result of one filesystem operation. +type FileResponse struct { + Data []byte `json:"data,omitempty"` + Version string `json:"version,omitempty"` + Info *FileInfo `json:"info,omitempty"` + Entries []FileInfo `json:"entries,omitempty"` + Paths []string `json:"paths,omitempty"` + Matches []GrepMatch `json:"matches,omitempty"` + AuthorityTarget string `json:"authority_target,omitempty"` + AuthorityWorkspace string `json:"authority_workspace,omitempty"` +} + +// CommandStartRequest starts one bounded foreground shell command. +type CommandStartRequest struct { + Context RequestContext `json:"context"` + Command string `json:"command"` + TimeoutMillis int64 `json:"timeout_ms,omitempty"` +} + +// CommandStartResponse returns the command identity and terminal result. +type CommandStartResponse struct { + CommandID string `json:"command_id"` + State CommandState `json:"state"` + Result CommandStatusResponse `json:"result"` +} + +// CommandState classifies command lifecycle and fencing outcomes. +type CommandState string + +// Command states. +const ( + CommandRunning CommandState = "running" + CommandSucceeded CommandState = "succeeded" + CommandFailed CommandState = "failed" + CommandCancelled CommandState = "cancelled" + CommandFenceUnknown CommandState = "fence_unknown" +) + +// CommandQueryRequest identifies one command for status or cancellation. +type CommandQueryRequest struct { + Context RequestContext `json:"context"` + CommandID string `json:"command_id"` + Offset int64 `json:"offset,omitempty"` +} + +// CommandStatusResponse carries bounded output and terminal proof. +type CommandStatusResponse struct { + CommandID string `json:"command_id"` + State CommandState `json:"state"` + ExitCode int `json:"exit_code,omitempty"` + Stdout []byte `json:"stdout,omitempty"` + Stderr []byte `json:"stderr,omitempty"` + NextOffset int64 `json:"next_offset,omitempty"` + Truncated bool `json:"truncated,omitempty"` + TerminalReceipt string `json:"terminal_receipt,omitempty"` +} + +// ExecutorRequest is the credential-free provider-to-workload stdin protocol. +// It deliberately cannot select an environment, root, Pod, image, argv, or process environment. +type ExecutorRequest struct { + Operation Operation `json:"operation"` + Path string `json:"path,omitempty"` + Destination string `json:"destination,omitempty"` + Pattern string `json:"pattern,omitempty"` + Data []byte `json:"data,omitempty"` + Version string `json:"version,omitempty"` + Limit int `json:"limit,omitempty"` + Command string `json:"command,omitempty"` + CommandID string `json:"command_id,omitempty"` + TimeoutMillis int64 `json:"timeout_ms,omitempty"` +} + +// ExecutorResponse carries one credential-free helper result. +type ExecutorResponse struct { + FileResponse + Command *CommandStatusResponse `json:"command,omitempty"` +} + +// ExecutorEnvelope provides a terminal, credential-free result over pods/exec. +type ExecutorEnvelope struct { + Response *ExecutorResponse `json:"response,omitempty"` + Error *Error `json:"error,omitempty"` +} diff --git a/internal/executionexecutor/command_linux.go b/internal/executionexecutor/command_linux.go new file mode 100644 index 0000000000..8d431da744 --- /dev/null +++ b/internal/executionexecutor/command_linux.go @@ -0,0 +1,169 @@ +//go:build linux + +package executionexecutor + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +const descendantCleanupTimeout = 2 * time.Second + +var subreaperEnabled atomic.Bool + +type commandResult struct { + stdout []byte + stderr []byte + exitCode int + truncated bool +} + +// EnableCommandExecution makes this process a Linux child subreaper. It must be +// called by the dedicated executor helper before it starts any other process. +func EnableCommandExecution() error { + if err := unix.Prctl(unix.PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0); err != nil { + return fmt.Errorf("enable executor child subreaper: %w", err) + } + subreaperEnabled.Store(true) + return nil +} + +func runIsolatedCommand(ctx context.Context, dir, command string, outputLimit int) (commandResult, bool, error) { + if !subreaperEnabled.Load() { + return commandResult{}, false, errors.New("command execution requires the dedicated subreaper helper") + } + var stdout, stderr boundedWriter + stdout.limit = outputLimit + stderr.limit = outputLimit + cmd := exec.Command("/bin/sh", "-c", command) // #nosec G204 -- the command is the authenticated tool payload. + cmd.Dir = dir + cmd.Env = []string{"HOME=/workspace", "PATH=/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin", "TMPDIR=/tmp", "GOTMPDIR=/tmp"} + cmd.Stdout = &stdout + cmd.Stderr = &stderr + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.WaitDelay = 250 * time.Millisecond + if err := cmd.Start(); err != nil { + return commandResult{}, true, err + } + + waited := make(chan error, 1) + go func() { waited <- cmd.Wait() }() + var waitErr error + select { + case waitErr = <-waited: + case <-ctx.Done(): + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + _ = cmd.Process.Kill() + waitErr = <-waited + } + + // The shell may have exited successfully while background descendants remain. + // Kill its original process group, then repeatedly kill every process reparented + // to this dedicated subreaper. ECHILD is the only clean-completion proof. + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + cleanupCtx, cancel := context.WithTimeout(context.Background(), descendantCleanupTimeout) + defer cancel() + proven := reapAllChildren(cleanupCtx) + + result := commandResult{stdout: stdout.Bytes(), stderr: stderr.Bytes(), exitCode: exitCode(waitErr), truncated: stdout.truncated || stderr.truncated} + if ctx.Err() != nil { + return result, proven, ctx.Err() + } + return result, proven, waitErr +} + +func reapAllChildren(ctx context.Context) bool { + for { + for _, pid := range directChildren() { + _ = syscall.Kill(pid, syscall.SIGKILL) + } + for { + var status syscall.WaitStatus + pid, err := syscall.Wait4(-1, &status, syscall.WNOHANG, nil) + switch { + case errors.Is(err, syscall.ECHILD): + return true + case err != nil: + return false + case pid <= 0: + goto wait + } + } + wait: + select { + case <-ctx.Done(): + return false + case <-time.After(5 * time.Millisecond): + } + } +} + +func directChildren() []int { + tasks, err := os.ReadDir("/proc/self/task") + if err != nil { + return nil + } + seen := make(map[int]struct{}) + for _, task := range tasks { + data, err := os.ReadFile(filepath.Join("/proc/self/task", task.Name(), "children")) + if err != nil { + continue + } + for _, field := range strings.Fields(string(data)) { + pid, err := strconv.Atoi(field) + if err == nil && pid > 0 { + seen[pid] = struct{}{} + } + } + } + out := make([]int, 0, len(seen)) + for pid := range seen { + out = append(out, pid) + } + return out +} + +func exitCode(err error) int { + if err == nil { + return 0 + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode() + } + return -1 +} + +type boundedWriter struct { + buf bytes.Buffer + limit int + truncated bool +} + +func (w *boundedWriter) Write(p []byte) (int, error) { + original := len(p) + if remaining := w.limit - w.buf.Len(); remaining > 0 { + if len(p) > remaining { + p = p[:remaining] + w.truncated = true + } + _, _ = w.buf.Write(p) + } else if original > 0 { + w.truncated = true + } + return original, nil +} + +func (w *boundedWriter) Bytes() []byte { return append([]byte(nil), w.buf.Bytes()...) } diff --git a/internal/executionexecutor/command_unsupported.go b/internal/executionexecutor/command_unsupported.go new file mode 100644 index 0000000000..177168a4fb --- /dev/null +++ b/internal/executionexecutor/command_unsupported.go @@ -0,0 +1,24 @@ +//go:build !linux + +package executionexecutor + +import ( + "context" + "errors" +) + +type commandResult struct { + stdout []byte + stderr []byte + exitCode int + truncated bool +} + +// EnableCommandExecution reports that the workload helper is Linux-only. +func EnableCommandExecution() error { + return errors.New("command execution requires Linux child-subreaper support") +} + +func runIsolatedCommand(context.Context, string, string, int) (commandResult, bool, error) { + return commandResult{}, false, errors.New("command execution requires Linux child-subreaper support") +} diff --git a/internal/executionexecutor/confinement_test.go b/internal/executionexecutor/confinement_test.go new file mode 100644 index 0000000000..e7db9a73b5 --- /dev/null +++ b/internal/executionexecutor/confinement_test.go @@ -0,0 +1,76 @@ +package executionexecutor + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestFileOperationsNeverTraverseExternalSymlink(t *testing.T) { + root, outside := t.TempDir(), t.TempDir() + sentinel := filepath.Join(outside, "file") + if err := os.WriteFile(sentinel, []byte("outside-secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil { + t.Fatal(err) + } + x, err := New(root, Limits{}) + if err != nil { + t.Fatal(err) + } + defer x.Close() + for _, op := range []executionenv.Operation{executionenv.OpFileRead, executionenv.OpFileResolveAuthority, executionenv.OpFileStat, executionenv.OpFileCreate, executionenv.OpFileReplace, executionenv.OpFileList, executionenv.OpFileRemove, executionenv.OpFileRename, executionenv.OpFileCopy} { + t.Run(string(op), func(t *testing.T) { + req := executionenv.ExecutorRequest{Operation: op, Path: "escape/file", Destination: "destination", Version: "version"} + if op == executionenv.OpFileList { + req.Path = "escape" + } + if op == executionenv.OpFileCreate { + req.Path = "escape/new" + } + if _, err := x.Execute(t.Context(), req); err == nil { + t.Fatal("external symlink traversal accepted") + } + if op == executionenv.OpFileCopy || op == executionenv.OpFileRename { + if _, err := x.Execute(t.Context(), executionenv.ExecutorRequest{Operation: executionenv.OpFileCreate, Path: string(op), Data: []byte("local")}); err != nil { + t.Fatal(err) + } + req.Path = string(op) + req.Destination = "escape/new" + if _, err := x.Execute(t.Context(), req); err == nil { + t.Fatal("external destination accepted") + } + } + }) + } + for _, op := range []executionenv.Operation{executionenv.OpFileGlob, executionenv.OpFileGrep} { + req := executionenv.ExecutorRequest{Operation: op, Pattern: "**/*", Limit: 100} + if op == executionenv.OpFileGrep { + req.Path = "**/*" + req.Pattern = "outside-secret" + } + result, err := x.Execute(t.Context(), req) + if err != nil { + continue + } // Honest confinement errors are also safe. + for _, path := range result.Paths { + if strings.HasPrefix(path, "escape/") { + t.Fatalf("glob escaped: %q", path) + } + } + if len(result.Matches) != 0 { + t.Fatalf("grep leaked external content: %+v", result.Matches) + } + } + data, err := os.ReadFile(sentinel) + if err != nil || string(data) != "outside-secret" { + t.Fatalf("external file mutated: %q %v", data, err) + } + if _, err := os.Stat(filepath.Join(outside, "new")); !os.IsNotExist(err) { + t.Fatalf("external file created: %v", err) + } +} diff --git a/internal/executionexecutor/executor.go b/internal/executionexecutor/executor.go new file mode 100644 index 0000000000..1c85956a53 --- /dev/null +++ b/internal/executionexecutor/executor.go @@ -0,0 +1,246 @@ +// Package executionexecutor is the fixed, credential-free workload helper. +package executionexecutor + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io/fs" + "strings" + "time" + "unicode/utf8" + + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/osfs" + "github.com/stacklok/mecatl/internal/executionenv" +) + +// Limits bounds one executor operation. +type Limits struct { + MaxFileBytes int + MaxEntries int + CommandTimeout time.Duration +} + +// Executor serves one credential-free workload-helper process. +type Executor struct { + workspace *osfs.Workspace + limits Limits +} + +// New constructs an executor confined to root. +func New(root string, limits Limits) (*Executor, error) { + if limits.MaxFileBytes <= 0 || limits.MaxFileBytes > executionenv.MaxFileBytes { + limits.MaxFileBytes = executionenv.MaxFileBytes + } + if limits.MaxEntries <= 0 || limits.MaxEntries > executionenv.MaxListEntries { + limits.MaxEntries = executionenv.MaxListEntries + } + if limits.CommandTimeout <= 0 { + limits.CommandTimeout = 30 * time.Second + } + ws, err := osfs.NewWorkspace(root) + if err != nil { + return nil, err + } + return &Executor{workspace: ws, limits: limits}, nil +} + +// Close releases executor resources. +func (*Executor) Close() error { return nil } + +// Execute performs one bounded executor operation. +func (e *Executor) Execute(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { //nolint:gocyclo // The operation switch keeps validation and dispatch in one boundary. + if !q.Operation.Valid() { + return executionenv.ExecutorResponse{}, bad("unsupported executor operation") + } + if len(q.Data) > e.limits.MaxFileBytes || len(q.Path) > executionenv.MaxPathBytes || len(q.Destination) > executionenv.MaxPathBytes || len(q.Pattern) > executionenv.MaxPathBytes || q.Limit < 0 || q.Limit > executionenv.MaxListEntries { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "executor request exceeds configured bounds") + } + switch q.Operation { + case executionenv.OpFileRead: + return e.read(ctx, q) + case executionenv.OpFileResolveAuthority: + target, workspace, err := e.workspace.AuthorityResourcePath(q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{AuthorityTarget: target, AuthorityWorkspace: workspace}}, nil + case executionenv.OpFileStat: + return e.stat(ctx, q) + case executionenv.OpFileCreate: + return e.create(ctx, q) + case executionenv.OpFileReplace: + return e.replace(ctx, q) + case executionenv.OpFileList: + return e.list(ctx, q) + case executionenv.OpFileRemove: + return executionenv.ExecutorResponse{}, mapErr(e.workspace.Remove(ctx, q.Path)) + case executionenv.OpFileRename: + return executionenv.ExecutorResponse{}, mapErr(e.workspace.Rename(ctx, q.Path, q.Destination)) + case executionenv.OpFileCopy: + v, err := e.workspace.CopyFile(ctx, q.Path, q.Destination) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + s, _ := tool.EncodeFileVersion(v) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Version: s}}, nil + case executionenv.OpFileGlob: + paths, err := e.workspace.Glob(ctx, q.Pattern) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + if len(paths) > e.resultLimit(q.Limit) { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "glob result exceeds configured limit") + } + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Paths: paths}}, nil + case executionenv.OpFileGrep: + matches, err := e.workspace.Grep(ctx, q.Pattern, q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + if len(matches) > e.resultLimit(q.Limit) { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "grep result exceeds configured limit") + } + out := make([]executionenv.GrepMatch, len(matches)) + for i, m := range matches { + out[i] = executionenv.GrepMatch{Path: m.Path, Line: m.Line, Text: strings.ToValidUTF8(m.Text, "�")} + } + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Matches: out}}, nil + case executionenv.OpCommandStart: + return e.command(ctx, q) + default: + return executionenv.ExecutorResponse{}, bad("operation is handled by provider, not workload") + } +} +func (e *Executor) read(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + b, v, err := e.workspace.ReadVersion(ctx, q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + if len(b) > e.limits.MaxFileBytes { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "file exceeds configured limit") + } + s, _ := tool.EncodeFileVersion(v) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Data: b, Version: s}}, nil +} +func (e *Executor) stat(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + i, err := e.workspace.Stat(ctx, q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + x := convertInfo(i) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Info: &x}}, nil +} +func (e *Executor) create(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + v, err := e.workspace.CreateFile(ctx, q.Path, q.Data) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + s, _ := tool.EncodeFileVersion(v) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Version: s}}, nil +} +func (e *Executor) replace(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + v, err := e.workspace.ReplaceFile(ctx, q.Path, tool.DecodeFileVersion(q.Version), q.Data) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + s, _ := tool.EncodeFileVersion(v) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Version: s}}, nil +} +func (e *Executor) list(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + items, err := e.workspace.ReadDir(ctx, q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + if len(items) > e.resultLimit(q.Limit) { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "directory exceeds configured limit") + } + out := make([]executionenv.FileInfo, len(items)) + for i, v := range items { + out[i] = convertInfo(v) + } + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Entries: out}}, nil +} +func (e *Executor) command(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + if q.CommandID == "" || len(q.Command) == 0 || len(q.Command) > executionenv.MaxCommandBytes { + return executionenv.ExecutorResponse{}, bad("command id and bounded command are required") + } + d := e.limits.CommandTimeout + if q.TimeoutMillis > 0 { + requested := time.Duration(q.TimeoutMillis) * time.Millisecond + if requested < d { + d = requested + } + } + runCtx, cancel := context.WithTimeout(ctx, d) + defer cancel() + result, cleanupProven, err := runIsolatedCommand(runCtx, e.workspace.Root(), q.Command, e.limits.MaxFileBytes) + stdout, stderr, truncated := capStreams(result.stdout, result.stderr, e.limits.MaxFileBytes) + truncated = truncated || result.truncated + if !cleanupProven { + return executionenv.ExecutorResponse{Command: &executionenv.CommandStatusResponse{CommandID: q.CommandID, State: executionenv.CommandFenceUnknown, Stdout: stdout, Stderr: stderr, Truncated: truncated}}, nil + } + state := executionenv.CommandSucceeded + if errors.Is(runCtx.Err(), context.Canceled) || errors.Is(runCtx.Err(), context.DeadlineExceeded) { + state = executionenv.CommandCancelled + } else if err != nil || result.exitCode != 0 { + state = executionenv.CommandFailed + } + receiptInput := append(append([]byte(q.CommandID+"\x00"), stdout...), 0) + receiptHash := sha256.Sum256(append(receiptInput, stderr...)) + return executionenv.ExecutorResponse{Command: &executionenv.CommandStatusResponse{CommandID: q.CommandID, State: state, ExitCode: result.exitCode, Stdout: stdout, Stderr: stderr, Truncated: truncated, TerminalReceipt: hex.EncodeToString(receiptHash[:])}}, nil +} +func convertInfo(i tool.FileInfo) executionenv.FileInfo { + return executionenv.FileInfo{Name: i.Name, Size: i.Size, Mode: uint32(i.Mode), ModTime: i.ModTime, IsDir: i.IsDir} +} +func (e *Executor) resultLimit(requested int) int { + if requested > 0 && requested < e.limits.MaxEntries { + return requested + } + return e.limits.MaxEntries +} +func capStreams(stdout, stderr []byte, n int) ([]byte, []byte, bool) { + var stdoutClipped, stderrClipped bool + stdout, stdoutClipped = capOutput(stdout, n) + remaining := n - len(stdout) + stderr, stderrClipped = capOutput(stderr, remaining) + return stdout, stderr, stdoutClipped || stderrClipped +} + +func capOutput(b []byte, n int) ([]byte, bool) { + b = []byte(strings.ToValidUTF8(string(b), "�")) + clipped := len(b) > n + if clipped { + b = b[:n] + for !utf8.Valid(b) { + b = b[:len(b)-1] + } + } + return b, clipped +} +func bad(msg string) error { return coded(executionenv.CodeInvalidArgument, msg) } +func coded(code executionenv.ErrorCode, msg string) error { + return &executionenv.Error{Code: code, Message: msg} +} +func mapErr(err error) error { + if err == nil { + return nil + } + var mismatch *tool.VersionMismatchError + switch { + case errors.As(err, &mismatch): + return coded(executionenv.CodeVersionMismatch, "file version mismatch") + case errors.Is(err, fs.ErrNotExist): + return coded(executionenv.CodeNotFound, "path not found") + case errors.Is(err, fs.ErrExist): + return coded(executionenv.CodeAlreadyExists, "destination already exists") + case errors.Is(err, tool.ErrDirectoryNotEmpty): + return coded(executionenv.CodeDirectoryNotEmpty, "directory is not empty") + case errors.Is(err, osfs.ErrPathEscape): + return coded(executionenv.CodePermissionDenied, "path is outside workspace") + default: + return err + } +} diff --git a/internal/executionexecutor/executor_test.go b/internal/executionexecutor/executor_test.go new file mode 100644 index 0000000000..b4739a61af --- /dev/null +++ b/internal/executionexecutor/executor_test.go @@ -0,0 +1,239 @@ +package executionexecutor + +import ( + "context" + "encoding/json" + "errors" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/stacklok/mecatl/engine/adapter/fstools" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestFileOperationsAreConfinedAndVersioned(t *testing.T) { + root := t.TempDir() + x, err := New(root, Limits{MaxFileBytes: 1024, MaxEntries: 100, CommandTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer x.Close() + ctx := context.Background() + created, err := x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileCreate, Path: "a/file.txt", Data: []byte("one")}) + if err != nil { + t.Fatal(err) + } + read, err := x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileRead, Path: "a/file.txt"}) + if err != nil || string(read.Data) != "one" || read.Version != created.Version { + t.Fatalf("read=%+v err=%v", read, err) + } + resolved, err := x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileResolveAuthority, Path: "a/file.txt"}) + if err != nil || resolved.AuthorityWorkspace != root || resolved.AuthorityTarget != filepath.Join(root, "a/file.txt") { + t.Fatalf("resolved authority=%+v err=%v", resolved.FileResponse, err) + } + _, err = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileReplace, Path: "a/file.txt", Data: []byte("two"), Version: "wrong"}) + var pe *executionenv.Error + if !errors.As(err, &pe) || pe.Code != executionenv.CodeVersionMismatch { + t.Fatalf("expected version mismatch: %v", err) + } + _, err = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileRead, Path: "../../etc/passwd"}) + if err == nil { + t.Fatal("escape accepted") + } + outside := filepath.Join(root, "..", "outside") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil { + t.Fatal(err) + } + _, err = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileRead, Path: "escape"}) + if err == nil || errors.Is(err, fs.ErrNotExist) { + t.Fatalf("symlink escape not distinctly rejected: %v", err) + } + _, err = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileResolveAuthority, Path: "escape"}) + var denied *executionenv.Error + if !errors.As(err, &denied) || denied.Code != executionenv.CodePermissionDenied { + t.Fatalf("authority resolver accepted symlink escape: %v", err) + } +} + +func TestRenameAndCopyNeverClobber(t *testing.T) { + x, _ := New(t.TempDir(), Limits{MaxFileBytes: 1024, MaxEntries: 100}) + defer x.Close() + ctx := context.Background() + _, _ = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileCreate, Path: "a", Data: []byte("a")}) + _, _ = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileCreate, Path: "b", Data: []byte("b")}) + for _, op := range []executionenv.Operation{executionenv.OpFileCopy, executionenv.OpFileRename} { + if _, err := x.Execute(ctx, executionenv.ExecutorRequest{Operation: op, Path: "a", Destination: "b"}); err == nil { + t.Fatalf("%s clobbered", op) + } else { + var pe *executionenv.Error + if !errors.As(err, &pe) || pe.Code != executionenv.CodeAlreadyExists { + t.Fatalf("%s wrong error: %v", op, err) + } + } + } +} + +func TestForegroundCommandReturnsReceiptAndTimeoutTerminatesDescendants(t *testing.T) { + if testing.Short() { + t.Skip("subprocess regression test") + } + root := t.TempDir() + ok := runExecutorHelper(t, root, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "c1", Command: "printf hello", TimeoutMillis: 500}) + if ok.Command == nil || ok.Command.State != executionenv.CommandSucceeded || ok.Command.TerminalReceipt == "" || string(ok.Command.Stdout) != "hello" { + t.Fatalf("response=%+v", ok) + } + timed := runExecutorHelper(t, root, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "c2", Command: "sleep 10 & wait", TimeoutMillis: 20}) + if timed.Command == nil || timed.Command.State != executionenv.CommandCancelled || timed.Command.TerminalReceipt == "" { + t.Fatalf("response=%+v", timed) + } +} + +func TestSuccessfulDetachedChildIsKilledBeforeTerminalReceipt(t *testing.T) { + root := t.TempDir() + result := filepath.Join(root, "result") + response := runExecutorHelper(t, root, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "detached", Command: "nohup sh -c 'sleep 0.2; echo late > result' >/dev/null 2>&1 &", TimeoutMillis: 1000}) + if response.Command == nil || response.Command.State != executionenv.CommandSucceeded || response.Command.TerminalReceipt == "" { + t.Fatalf("response=%+v", response) + } + timer := time.NewTimer(400 * time.Millisecond) + defer timer.Stop() + <-timer.C + if _, err := os.Stat(result); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("detached writer survived terminal receipt: %v", err) + } +} + +func TestNativeShellCapsCombinedOutputAndRepairsUTF8(t *testing.T) { + for _, tc := range []struct { + name, command string + wantStdout, wantStderr int + }{ + {"stdout-exact-cap-repair", `i=0; while [ "$i" -lt 1020 ]; do printf x; i=$((i+1)); done; printf '\377abc'`, 1024, 0}, + {"stderr-exact-cap-repair", `i=0; while [ "$i" -lt 1020 ]; do printf x >&2; i=$((i+1)); done; printf '\377abc' >&2`, 0, 1024}, + {"combined-exact-cap-repair", `printf xxxx; i=0; while [ "$i" -lt 1016 ]; do printf x >&2; i=$((i+1)); done; printf '\377abc' >&2`, 4, 1020}, + {"stdout", `i=0; while [ "$i" -lt 2048 ]; do printf x; i=$((i+1)); done`, 1024, 0}, + {"stderr", `i=0; while [ "$i" -lt 2048 ]; do printf x >&2; i=$((i+1)); done`, 0, 1024}, + {"split-malformed", `printf '\377'; printf '\376' >&2; i=0; while [ "$i" -lt 700 ]; do printf x; printf y >&2; i=$((i+1)); done`, 703, 321}, + } { + t.Run(tc.name, func(t *testing.T) { + response := runExecutorHelper(t, t.TempDir(), executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "bounded", Command: tc.command}) + result := response.Command + if result == nil || result.State != executionenv.CommandSucceeded || result.TerminalReceipt == "" { + t.Fatalf("native command failed: %+v", response) + } + if !result.Truncated || len(result.Stdout) != tc.wantStdout || len(result.Stderr) != tc.wantStderr || len(result.Stdout)+len(result.Stderr) > 1024 { + t.Fatalf("stdout=%d stderr=%d truncated=%t", len(result.Stdout), len(result.Stderr), result.Truncated) + } + if !utf8.Valid(result.Stdout) || !utf8.Valid(result.Stderr) { + t.Fatal("native command result contains malformed UTF-8") + } + if strings.HasSuffix(tc.name, "exact-cap-repair") { + stream := result.Stdout + if tc.wantStderr != 0 { + stream = result.Stderr + } + if !strings.HasSuffix(string(stream), "�a") { + t.Fatal("repair-expansion clip did not retain the expected prefix") + } + } + if tc.name == "split-malformed" && (!strings.HasPrefix(string(result.Stdout), "�") || !strings.HasPrefix(string(result.Stderr), "�")) { + t.Fatal("malformed stream bytes were not repaired") + } + }) + } +} + +func TestNativeShellWriteInvalidatesRecordedRead(t *testing.T) { + root := t.TempDir() + x, err := New(root, Limits{}) + if err != nil { + t.Fatal(err) + } + defer x.Close() + env := tool.MustEnvironment(session.EnvironmentRef{Kind: "kubernetes", ID: "shell-cas", Revision: "v1"}, x.workspace, memledger.New(), nil) + if _, err := x.workspace.CreateFile(t.Context(), "file.txt", []byte("before")); err != nil { + t.Fatal(err) + } + read, err := (fstools.ReadTool{}).Execute(t.Context(), session.NewToolCall("read", "Read", json.RawMessage(`{"path":"file.txt"}`)), env) + if err != nil || read.IsError { + t.Fatalf("record read: %+v, %v", read, err) + } + response := runExecutorHelper(t, root, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "write", Command: "printf shell-change > file.txt"}) + if response.Command == nil || response.Command.State != executionenv.CommandSucceeded { + t.Fatalf("Shell write failed: %+v", response) + } + edit, err := (fstools.EditTool{}).Execute(t.Context(), session.NewToolCall("edit", "Edit", json.RawMessage(`{"path":"file.txt","old_string":"shell-change","new_string":"lost"}`)), env) + if err != nil || !edit.IsError || !strings.Contains(edit.Content, "changed since") { + t.Fatalf("Edit must reject the stale recorded version, not an exact-match failure: %+v, %v", edit, err) + } + data, err := x.workspace.Read(t.Context(), "file.txt") + if err != nil || string(data) != "shell-change" { + t.Fatalf("Shell change overwritten: %q, %v", data, err) + } +} + +func runExecutorHelper(t *testing.T, root string, request executionenv.ExecutorRequest) executionenv.ExecutorResponse { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^TestExecutorCommandHelper$") + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + root, "MECATL_EXECUTOR_TEST_HELPER=1", "MECATL_EXECUTOR_TEST_ROOT=" + root} + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + if err := json.NewEncoder(stdin).Encode(request); err != nil { + t.Fatal(err) + } + _ = stdin.Close() + var response executionenv.ExecutorResponse + if err := json.NewDecoder(stdout).Decode(&response); err != nil { + t.Fatal(err) + } + if err := cmd.Wait(); err != nil { + t.Fatalf("executor helper: %v", err) + } + return response +} + +func TestExecutorCommandHelper(_ *testing.T) { + if os.Getenv("MECATL_EXECUTOR_TEST_HELPER") != "1" { + return + } + if err := EnableCommandExecution(); err != nil { + os.Exit(2) + } + x, err := New(os.Getenv("MECATL_EXECUTOR_TEST_ROOT"), Limits{MaxFileBytes: 1024, MaxEntries: 100, CommandTimeout: time.Second}) + if err != nil { + os.Exit(2) + } + var request executionenv.ExecutorRequest + if err := json.NewDecoder(os.Stdin).Decode(&request); err != nil { + os.Exit(2) + } + response, err := x.Execute(context.Background(), request) + if err != nil { + os.Exit(2) + } + if err := json.NewEncoder(os.Stdout).Encode(response); err != nil { + os.Exit(2) + } + os.Exit(0) +} diff --git a/user-docs/building/deployment/mecak8s.md b/user-docs/building/deployment/mecak8s.md index e20bbd3180..6ccef1b03c 100644 --- a/user-docs/building/deployment/mecak8s.md +++ b/user-docs/building/deployment/mecak8s.md @@ -151,6 +151,458 @@ delegation cannot add filesystem access that the parent lacks. See [Execution environments](/features/execution-environments.md) for the shared placement, no-FS, child-environment, and reattachment model. +### Kubernetes execution provider + +**DRAFT candidate, not released or approved for production.** These instructions +apply to the optional native provider, not to the maturity of `mecak8s` itself. +Use an operator-controlled cluster. This slice supports foreground commands, +has no force-takeover recovery, and does not promise hostile multi-tenant +isolation. + +The optional execution provider runs as a separate service and controller with its +own `mecatl-execution` chart. `mecak8s` remains a client and receives no Pod, +PVC, custom-resource, or controller management permissions. + +The provider supplies run-wide ownership, transactional references, controlled +replacement and retirement, durable grant revocation, and reloadable TLS and +signing material. Production isolation requires a CNI that enforces NetworkPolicy +and a RuntimeClass that supplies the isolation promised by your platform. + +Before installation, provide these values through your trusted image and Secret +delivery system: + +- Digest-pinned provider and workload images. The workload image must contain + `/mecatl-executor`, `/bin/sh`, and the toolchain used by foreground commands. +- A projected security Secret with the keys `grant-k1.pem`, `tls.crt`, `tls.key`, + and `clients.pem`. The Secret bytes remain externally managed. Mount rotation + uses the projected volume's `..data` link, without `subPath`. +- A `mecak8s` client mTLS Secret containing `ca.crt`, `tls.crt`, and `tls.key`. +- Explicit provider-client pod and namespace selectors, API-server CIDRs, and DNS + resolver CIDRs. + +The provider validates every configured RuntimeClass and StorageClass with +cluster-scoped `get` requests before it becomes ready. The chart grants those +requests only for the names present in `profiles`; it grants no cluster-wide list +or watch access. + +The following profile shows every required chart key. Save it as +`execution-values.yaml` and replace each placeholder: + +```yaml +fullnameOverride: mecatl-execution +provider: + image: @sha256: + imagePullPolicy: IfNotPresent + replicas: 2 + securitySecretName: + securityManifest: | + { + "version": 1, + "generation": 42, + "issuer": "https://execution.example.com", + "audience": "mecatl-execution", + "activeKeyID": "k1", + "grantTTL": "1m", + "clockSkew": "5s", + "keys": [{ + "id": "k1", + "version": 1, + "file": "grant-k1.pem", + "publicKeySHA256": "0000000000000000000000000000000000000000000000000000000000000000", + "activateAt": "2027-01-01T00:00:00Z", + "verifyUntil": "2027-01-02T00:00:00Z", + "state": "active" + }], + "tls": { + "certificateFile": "tls.crt", + "privateKeyFile": "tls.key", + "clientCAFile": "clients.pem" + }, + "clients": [{ + "uri": "spiffe://cluster.example.com/ns/mecatl/sa/mecak8s", + "mayAttestOwner": true, + "administrator": false + }, { + "uri": "spiffe://cluster.example.com/ns/mecatl/sa/execution-admin", + "mayAttestOwner": false, + "administrator": true, + "administratorFor": ["spiffe://cluster.example.com/ns/mecatl/sa/mecak8s"] + }] + } + clientIngressSelectors: + - namespaceLabels: {kubernetes.io/metadata.name: } + podLabels: {app.kubernetes.io/name: mecak8s} + apiServerCIDRs: [/32] + dnsCIDRs: [/32] + +service: + port: 8443 + +profiles: + coding: + image: @sha256: + storageClass: + storageSize: 2Gi + cpuRequest: 100m + memoryRequest: 128Mi + cpuLimit: "1" + memoryLimit: 1Gi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: + maxFileBytes: 5242880 + maxCommandBytes: 1048576 + maxCommandDuration: 5m + maxEnvironments: 100 + +networkPolicy: + enabled: true + dnsPorts: [53] + apiServerPorts: [443] + workloadProfiles: {} # default deny; add explicit coding egress only if required + +resourceGovernance: + enabled: true + pods: "200" + persistentVolumeClaims: "100" + executionEnvironments: "100" + requestsCPU: "20" + requestsMemory: 40Gi + requestsStorage: 500Gi + requestsEphemeralStorage: 40Gi + limitsCPU: "40" + limitsMemory: 80Gi + limitsEphemeralStorage: 80Gi +``` + +Replace the all-zero `publicKeySHA256` with the lowercase hexadecimal SHA-256 +hash of the **raw 32-byte Ed25519 public key** corresponding to `grant-k1.pem`, +not the PEM text or DER encoding. Replace the sample activation +and verification dates with a current, reviewed rotation window. Increase +`generation` for every authority change, including a CA, client policy, +issuer/audience, key state, key window, or TLS identity change. + +Install the provider chart separately from `mecak8s`, before allocating any +execution environments. Use a namespace dedicated to this provider release: + +```sh +helm install mecatl-execution ./deploy/helm/mecatl-execution \ + --namespace \ + --values execution-values.yaml +kubectl rollout status deployment/mecatl-execution --namespace +``` + +Add the following block to the existing `mecak8s` values. The endpoint is a +`host:port` gRPC target (no URL scheme) and must use the provider certificate's +DNS identity. The private `mecatl.execution.v1.ExecutionProviderService` uses +protocol `execution-grpc/1` with mandatory mTLS. Keep OIDC caller authentication enabled; +the remote binding is scoped to the verified issuer and subject. + +```yaml +execution: + enabled: true + endpoint: mecatl-execution..svc:8443 + profile: coding + tlsSecret: + caKey: ca.crt + certKey: tls.crt + keyKey: tls.key +``` + +An enabled client conflicts with `workspace`, `redis.filesystem.enabled`, +Parallel, and Team. Remote sessions receive the filesystem tools and foreground +Shell, but do not ingest project instructions, rules, skills, or source from the +remote PVC. Schedules, SkillDraft, background Shell, and delegated filesystem +execution are outside this draft. + +If allocation reports `Ready=false` with `PVCUnavailable` or +`ExecutorUnavailable`, inspect the namespace's quota and Kubernetes admission +failures. Restore the failed prerequisite and wait for `Ready=True`. The +controller keeps retrying through its rate-limited queue while the allocation +exists; no reference edit, restart, or manual reconciliation is needed. Missing +authoritative PVCs or Pods and ownership mismatches remain fail-closed and are +never repaired by creating a replacement. + +Run claims and signed grants renew before the issued grant expiry, including when +an operator configures a short grant TTL. Renewal operation receipts retain the +newest 32 identities. Retrying an exact retained identity preserves the claim and +original expiry (the replacement signature may carry a new nonce); conflicting or +expired receipt replay is denied and never extends or resurrects the claim. + +The provider retains one PVC per logical environment. Session deletion and chart +uninstall do not delete committed workspace data. Retirement requires no live +references and a terminal executor; even then, this candidate retains the PVC. +A missing executor or lost terminal receipt moves the environment to +`FenceUnknown` and requires external operator fencing. The built-in recovery RPC +accepts only a still-observable Pod in `Succeeded` or `Failed` phase with every +container terminated and the exact PVC still present. If that proof is missing, +use your platform's external fencing runbook; there is no acknowledgement flag. + +The built-in recovery RPC requires the same exact identity and an independently +stable operation ID. It does not read a Secret and has no force or +acknowledgement field: + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedExecutionEpoch":"","expectedPodUid":"","expectedPvcUid":"","operationId":"recover-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/RecoverEnvironment +``` + +#### Rotate execution-provider authority + +Give every changed material file a new, generation-specific basename. This applies +to signing keys, server certificates, server private keys, and client-CA bundles. +For example, a second bundle can use `grant-k2.pem`, `server-g2.crt`, +`server-g2.key`, and `clients-g2.pem`. Keep each name's bytes immutable. + +1. Add the new files to the operator-managed security Secret, retaining the files + referenced by the current manifest. Stage the material before changing + `provider.securityManifest`. +2. Publish a higher-generation manifest whose existing `file`, `certificateFile`, + `privateKeyFile`, and `clientCAFile` fields reference those names. For CA + rotation, first publish a separately named overlap bundle, move clients and + server trust, then publish another higher generation that removes old trust. +3. Verify that the authority ConfigMap has reached the intended generation and + use a current claim to read known workspace content. Pod readiness alone can + still reflect the previous generation. A replica whose snapshot lags the + ledger rejects requests before dispatch with structured `not_ready` and + `retryable=true`. Bound any read-only verification poll and stop on wrong + content, a nonretryable error, or another error code. +4. Retain overlap files until no live or in-flight manifest references them. + Retain verification keys through their grant windows before revoking them. + Never reuse a retired name for different bytes. + +Kubernetes projects Secret and ConfigMap updates independently. A manifest that +arrives before its new files fails closed until they arrive; staged files leave +an older manifest unchanged. Overwriting referenced TLS or CA files can instead +publish a mixed bundle's digest permanently at the new generation. If that has +happened, publish a complete, immutable bundle at a **higher** generation. +Repeated requests or a restart cannot repair same-generation digest drift; +preserve the authority ledger rather than resetting it. The provider confines +file access to the mounted security directory, but immutable publication remains +your secret-management procedure's responsibility. + +#### Upgrade, uninstall, and reinstall the execution provider + +The supported lifecycle keeps the **same Helm release name, namespace, resource +names, profiles, network policy configuration, and security Secret name**. Keep +`execution-values.yaml` and the current nonsecret authority manifest in your +operator configuration store. Retain the operator-owned Secret and its key +history independently; the chart neither owns nor reads Secret contents. + +**Before quiescing:** verify that the retained profiles ConfigMap has a nonempty +`data["lifetime.json"]` and that the authority/capacity ledgers and original +release ownership are intact. Pre-retention installations without this history +cannot be automatically adopted. Stop this procedure and recover trusted retained +history; do not synthesize it from proposed values or reset authority. The chart +reports missing history separately from an incompatible lifetime configuration. + +For a compatible provider upgrade: + +1. Stop new client traffic and finish or explicitly fence active work. Stop all + provider replicas before Helm reads the ledgers: + + ```sh + kubectl --namespace scale deployment/mecatl-execution --replicas=0 + kubectl --namespace wait --for=delete pod \ + --selector app.kubernetes.io/name=mecatl-execution --timeout=2m + ``` + +2. Apply the reviewed CRD schema **before** starting the upgraded provider. Helm + does not upgrade existing CRDs: + + ```sh + kubectl apply -f deploy/helm/mecatl-execution/crds/executionenvironment.yaml + kubectl wait --for=condition=Established \ + crd/executionenvironments.execution.mecatl.dev --timeout=60s + ``` + +3. Upgrade with the preserved lifetime configuration and current authority + manifest. Helm reads the existing ConfigMaps and includes their actual ledger + data, rather than empty bootstrap data, in the new release: + + ```sh + helm upgrade mecatl-execution ./deploy/helm/mecatl-execution \ + --namespace --values execution-values.yaml --wait --timeout=4m + ``` + +4. Complete any supported, explicit environment-schema migration while client + traffic remains quiesced. Verify readiness, exact environment/PVC UIDs, data, + and network confinement before resuming traffic. Unknown schema versions and + mixed-version provider operation are unsupported. + +Default uninstall removes the provider but keeps runtime CRs, PVCs, surviving +executors, workload default-deny and profile NetworkPolicies, both authority and +capacity ledgers, and the profile/security-manifest ConfigMaps. Uninstall is not +executor termination proof or storage disposal: + +```sh +helm uninstall mecatl-execution --namespace +``` + +Reinstall with the same identity and preserved values: + +```sh +helm install mecatl-execution ./deploy/helm/mecatl-execution \ + --namespace --values execution-values.yaml --wait --timeout=4m +``` + +Helm adopts retained resources only when their managed-by label and release-name +and release-namespace annotations match. The chart rejects missing or empty +ledgers while allocations or capacity reservations survive, and rejects changes +to its retained lifetime configuration. Profile removal or egress edits are deliberately outside this +upgrade path: retained allow policies are additive, so leaving an obsolete policy +could widen access. The provider rejects an older authority manifest against the +retained high-water generation and key history; never reset that ledger to make +readiness pass. + +Use live Helm install/upgrade for this lifecycle. Offline `helm template` cannot +perform ownership or history lookups and is not an adoption mechanism. Keep +provider writers stopped for upgrades; lookup plus apply is not a cross-resource +transaction. Rendering rejects a nonzero existing provider Deployment or any +remaining provider Pod, including a terminating Pod. Changed release/namespace +adoption, chart rollback, `--take-ownership`, CRD or namespace deletion with +retained resources, and force-finalizer cleanup are unsupported. + +Final infrastructure decommission is not automated by this candidate. Supported +`RetireEnvironment` and `DeleteRetiredEnvironment` operations can terminate and +dispose of eligible individual environments; they do not remove the retained +NetworkPolicies, configuration, ledgers, or CRD. There is no supported final +infrastructure-cleanup procedure here. Manual destructive cleanup is outside the +supported lifecycle. Keep retention defaults and authority history, and never +reset ledgers or remove finalizers to bypass a failed safety check. + +#### Run an administrative lifecycle operation + +Administrative RPCs require `administrator: true`. For a distinct operations +identity, set its `administratorFor` list to the canonical URI of each client +that created the environments it may administer, as in the values example above. +An absent or empty list permits only self-administration. Keep +`mayAttestOwner: false` for an operations-only identity; administrative scope +confers no filesystem, Shell, attach, run, or reference access. See the +[scope constraints](../../features/execution-environments.md#production-security-material). + +Before first publishing `administratorFor`, quiesce client traffic and upgrade +all provider replicas to a version that understands the field. Older strict +manifest decoders reject it; mixed-version rolling operation is unsupported. +Preserve the authority high-water ConfigMap throughout the upgrade. Publish the +reviewed manifest with a higher `generation`, verify provider readiness, then +resume client traffic. + +Every administrative request still requires the original exact owner and +revision plus the operation's epoch, UID, schema, or generation preconditions. +The scope names creators, not owners; it can include a creator whose login has +been removed. After maintenance, remove its scope entry and increase +`generation` again. Subsequent requests, including receipt retries on established +connections, are denied; already admitted lifecycle operations may finish safe +reconciliation. + +An administrative `not_found` response deliberately does not distinguish a +missing environment from a wrong owner, revision, or creator scope. Check the +exact environment ID/revision and original owner against your authorized +operations record, then check that `administratorFor` names the original +creator's exact canonical client URI. If the scope is wrong, publish a reviewed +manifest at a higher `generation` and verify readiness before retrying. The RPC +will not disclose another creator's data to diagnose a scope mismatch. + +First capture the private identity while the environment still has a reference: + +```sh +kubectl --namespace get executionenvironment \ + -o jsonpath='{.spec.revision}{"\n"}{.spec.ownerIssuer}{"\n"}{.spec.ownerSubject}{"\n"}{.status.epoch}{"\n"}{.status.pod.uid}{"\n"}{.status.pvc.uid}{"\n"}{.status.grantGeneration}{"\n"}' +``` + +Record those seven lines as ``, ``, ``, +``, ``, ``, and ``. The owner hash is +not reversible, so retain the bounded `spec.ownerIssuer` and `spec.ownerSubject` +attestation in your authorized operations record before removing the final +reference. + +Set file references to trusted, mounted mTLS material. Keep private-key bytes out +of shell arguments and manifests: + +```sh +EXECUTION_ENDPOINT=mecatl-execution..svc:8443 +CA_FILE=/var/run/secrets/mecatl-admin/ca.crt +CERT_FILE=/var/run/secrets/mecatl-admin/tls.crt +KEY_FILE=/var/run/secrets/mecatl-admin/tls.key +PROTO=contracts/proto/mecatl/execution/v1/execution.proto +``` + +Replace one executor by reusing the same operation ID for every retry: + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedExecutionEpoch":"","expectedPodUid":"","expectedPvcUid":"","operationId":"replace-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/ReplaceExecutor +``` + +To retire and then delete retained storage, call `RetireEnvironment` with the same +identity fields and a new stable operation ID. Poll until +`.status.conditions[?(@.type=="Retired")].status` is `True`, then call +`DeleteRetiredEnvironment` with the retained PVC UID and another stable operation +ID. The provider refuses either request while references, claims, identity proof, +or UID checks are incomplete. + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedExecutionEpoch":"","expectedPodUid":"","expectedPvcUid":"","operationId":"retire-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/RetireEnvironment +kubectl --namespace wait executionenvironment/ \ + --for='jsonpath={.status.conditions[?(@.type=="Retired")].status}=True' \ + --timeout=10m +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedPvcUid":"","operationId":"delete-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/DeleteRetiredEnvironment +``` + +Revoke grants with the current generation and a stable operation ID. The response +returns the new generation; retrying the identical request returns the same +receipt. An old grant is denied after this CAS succeeds. + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedGrantGeneration":"","operationId":"revoke-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/RevokeEnvironment +``` + +Migration requires proto presence for `expectedSchemaVersion`. Supply exactly `0` +or `1`, plus the exact observable Pod and PVC UIDs. Omission, an unknown version, +or missing proof is refused without changing the resource. + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedSchemaVersion":1,"expectedPodUid":"","expectedPvcUid":"","operationId":"migrate-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/MigrateEnvironment +``` + +Foreground command cancellation is cooperative and bounded. The helper attempts +to terminate the command process group and reports a terminal receipt. If it +cannot prove termination, the environment is fenced instead of admitting more +work. There is no detached command status or cancellation API in this draft. + +The focused qualification task is `task e2e:k8s:execution`. It requires Go, +`ko`, Docker or rootless Podman, Kind, Helm, and `kubectl`. Optional toolbox use +requires explicit `MECATL_EXECUTION_DEV_TOOLBOX` and +`MECATL_EXECUTION_K8S_TOOLBOX` values. The task uses a synthetic OIDC issuer and +the mock model provider, creates a unique state directory and Kind cluster, and +retains both for inspection. CI invokes +`task e2e:k8s:execution:production` and automatically removes only its uniquely +owned cluster. The production target is the home for the enforcing-CNI network +and lifecycle fixture; until that fixture passes, Kind qualification is not +NetworkPolicy isolation evidence. + +When `execution.enabled` is `false`, the `mecak8s` chart mounts no execution mTLS +Secret and passes no execution-provider flags. The separate provider chart and +any environments it owns continue independently. + The no-FS default is intentional. A standard mecak8s pod is storage-free and has no authoritative filesystem root, so the server binds omitted/default profile to its configured no-FS placement. Clients never send a workspace path; explicit diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index f63297997d..7562ed6c6a 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -78,6 +78,284 @@ reattached only when the deployment supplies an `EnvironmentResolver`; a missing resolver, mismatched identity, or nil workspace returns an error instead of using a local workspace. +## Native Kubernetes lifecycle and retention + +**DRAFT candidate, not released or approved for production.** The native provider +is intended for an operator-controlled cluster and foreground commands only. +It has no force-takeover recovery and is not a hostile multi-tenancy boundary. + +Remote deployments retain operator-global prompt rules and skills. Permission +rules from enabled user-global settings and explicit permission files also apply, +including configured Deny and Ask rules under `auto` posture. Project permission +files are excluded. Local project trust +cannot admit host-local AGENTS.md, rules, skills, commands, or Git context into a +remote session. Command discovery and slash-command expansion are unavailable +for remote sessions. Explicit `no-fs` sessions keep their file-less catalog and +allocate no execution environment. + +The remote filesystem preserves the existing Read/Edit/Write version checks and +non-clobbering Copy/Move behavior. Copy, Move, and Remove do not require prior +content reads; Remove remains non-recursive. + +The optional Kubernetes execution provider stores environment ownership in a +namespaced `ExecutionEnvironment`. Provider replicas coordinate through +Kubernetes resource-version compare-and-swap; a replica restart does not clear +another replica's operation. Operation lease expiry fences the environment and +retains the unresolved operation identity for administrator recovery. + +Workspace PVCs are retained by default. Removing the last session reference, +deleting a session, uninstalling the chart, or deleting the provider does not +delete a workspace PVC. Executor replacement and retirement are private, +mutually authenticated administrator operations. They require exact environment, +execution-epoch, Pod-UID, and PVC-UID values. The provider waits for a kubelet +terminal Pod phase and terminated state for every container before removing its +Pod finalizer. A missing Pod or unreachable kubelet is not termination proof and +leaves the environment fenced. + +Retirement stops the executor but retains the PVC. PVC deletion requires the +separate `DeleteRetiredEnvironment` administrator RPC with the exact retained +PVC UID and no references or claims. Deleting the custom resource outside this +provider workflow is unsupported. Do not remove the retention or executor +finalizers manually; follow the operator's external-fencing runbook when the +provider cannot independently observe terminal compute. + +Persisted prototype resources use an explicit administrator migration. Normal +operations reject schema versions other than 2. `MigrateEnvironment` accepts +only recognized versions 0 and 1, verifies the exact live Pod and PVC identities, +and requires a healthy, idle environment. Unknown or malformed state is retained +unchanged rather than reset. Completed migration receipts expire when executor +replacement publishes a new Pod identity. After replacement, retrying the old +migration operation returns a conflict with either the original or replacement UID. + +### Production security material + +The production chart requires one projected Secret containing the TLS identity, +client CA bundle, and Ed25519 grant keys, plus `provider.securityManifest`. The +chart does not generate keys or certificates. The manifest is strict JSON with +this shape: + +```json +{ + "version": 1, + "generation": 42, + "issuer": "https://execution.example.com", + "audience": "mecatl-execution", + "activeKeyID": "k1", + "grantTTL": "1m", + "clockSkew": "5s", + "keys": [ + { + "id": "k1", + "version": 1, + "file": "grant-k1.pem", + "publicKeySHA256": "0000000000000000000000000000000000000000000000000000000000000000", + "activateAt": "2027-01-01T00:00:00Z", + "verifyUntil": "2027-01-02T00:00:00Z", + "state": "active" + } + ], + "tls": { + "certificateFile": "tls.crt", + "privateKeyFile": "tls.key", + "clientCAFile": "clients.pem" + }, + "clients": [ + { + "uri": "spiffe://cluster.example.com/ns/mecatl/sa/mecak8s", + "mayAttestOwner": true, + "administrator": false + }, + { + "uri": "spiffe://cluster.example.com/ns/mecatl/sa/execution-admin", + "mayAttestOwner": false, + "administrator": true, + "administratorFor": ["spiffe://cluster.example.com/ns/mecatl/sa/mecak8s"] + } + ] +} +``` + +The all-zero fingerprint and 2027 dates are non-secret example values. Replace +the fingerprint with the lowercase hexadecimal SHA-256 hash of the raw 32-byte +Ed25519 public key, not PEM text or DER encoding, and use a reviewed active window. + +`administratorFor` permits administration of environments created by the listed +client URIs, alongside the administrator's own environments. A nonempty list +requires `administrator: true` and accepts at most 256 unique canonical URIs: +nonempty scheme and host, lowercase scheme and host, and no userinfo, query, +fragment, or wildcard. Matching is exact, with no namespace or prefix grant. +An absent or empty list preserves self-administration only. The creator need +not remain in the client allowlist. + +The scope applies to `RetireEnvironment`, `ReplaceExecutor`, `RecoverEnvironment`, +`DeleteRetiredEnvironment`, `MigrateEnvironment`, and `RevokeEnvironment`. It +preserves each operation's exact owner and identity checks and grants no +`mayAttestOwner`, attach, file, command, run, or reference authority. Scope order +does not change the authority digest; adding or removing a creator does. Follow +the [administrative runbook](../building/deployment/mecak8s.md#run-an-administrative-lifecycle-operation) +for quiesced upgrades and scope removal. + +Use only basename file names. The projected Secret keys in this example are +`grant-k1.pem`, `tls.crt`, `tls.key`, and `clients.pem`; an external secret manager +owns their bytes. Kubernetes projected-volume `..data` symlinks are supported, +but paths escaping the mounted directory are rejected. Keep every referenced +filename immutable and use a new name for changed signing, TLS, or CA material. +Stage those files before publishing a higher-generation manifest and retain the +overlap files. Secret and ConfigMap projections are independent; see the +[authority rotation procedure](../building/deployment/mecak8s.md#rotate-execution-provider-authority) +for publication, verification, and recovery from mixed-material digest drift. +Increase `generation` for every authority change. Key IDs and `(id, version)` fingerprints cannot be +reused; the provider persists a bounded high-water ledger in its authority +ConfigMap. Keep retired keys as `verify-only` until all grants expire, then mark +them `revoked`. Invalid, incomplete, rolled-back, or newly expired material makes +readiness fail and denies new RPC authorization until corrected. The provider +re-verifies the peer certificate and URI policy against the current client CA on +every RPC, including RPCs on an existing HTTP/2 connection. Profile-resource or +controller-cache startup failures terminate the provider with that bounded reason +class before it accepts traffic. A failed `/ready` response reports +`security-authority-or-expiry`. Inspect provider logs and the named RuntimeClass, +StorageClass, and authority ConfigMap metadata; the endpoint never returns key or +certificate contents. + +`RevokeEnvironment` is an administrator-only, exact-reference CAS. Supply the +current positive grant generation; success increments it without changing the +execution epoch. Old grants then cannot authorize another operation or renewal. +An operation already accepted may still finish and clean up, so revocation is not +termination proof and a new run needs a fresh claim after the old claim is +released or fenced. + +The gRPC port remains the only execution data plane. `/live` and `/ready` are +separate, unauthenticated operational HTTP endpoints intended only for Kubernetes +probes; do not expose that port outside the cluster. Readiness requires current +security material, the authoritative generation, synchronized controller caches, +and Kubernetes access. Liveness reports process health only. + +Production values require two or more provider replicas, explicit provider-client +selectors, API-server and DNS CIDRs, and finite ResourceQuota/LimitRange values. +Executor pods are default-deny for ingress and egress. Add only profile-specific +CIDR and port egress under `networkPolicy.workloadProfiles`; there is no implicit +DNS, metadata-service, API-server, provider, or peer access. Each profile requires +an explicit RuntimeClass supplied by the cluster, but do not treat a RuntimeClass +name alone as a hostile-workload isolation guarantee. CPU, memory, ephemeral +storage, and `/tmp` are explicitly bounded per profile. Each profile also requires +`maxEnvironments`; the namespace quota is the final global bound. + +Preserve the authority ConfigMap and current manifest across upgrades and +same-release reinstalls. A missing ledger with retained allocations is not a new +installation: do not bootstrap it at generation 1. See the +[provider lifecycle procedure](/building/deployment/mecak8s.md#upgrade-uninstall-and-reinstall-the-execution-provider) +for ownership checks, retained network protection, and quiesced CRD/provider +upgrade order. Schema-2 environment migration remains an explicit operation; +complete it before enabling new sessions. + +## Live qualification (experimental) + +The repository keeps real-provider qualification separate from the default mock +suite. It is explicit opt-in, uses an already-owned retained Kind cluster, and is +not part of `task test`: + +```sh +MECATL_EXECUTION_CREDENTIAL_FILE=/absolute/path/to/provider-key \ +MECATL_EXECUTION_QUAL_STATE=/absolute/path/to/owned-state \ +task e2e:k8s:execution:live +``` + +The credential file must be a private regular file (no group or other access). +A trusted helper loads it only at runtime and creates a run-scoped Kubernetes +Secret through the API; it never renders the value into a manifest or reads the +Secret back. Only the `mecak8s` harness container receives the OpenRouter key. +The execution provider, controller, OIDC fixture, and executor workloads do not. +The task first runs the deterministic mock qualification, then runs one bounded +real-model coding smoke against `https://openrouter.ai/api/v1`. It verifies +positive token usage, required file and shell tool calls, file contents, and a +successful `go test` independently through the typed gRPC execution service. +Finally it restores the mock deployment and removes only its run-scoped Secret; +the owned cluster and execution workspaces remain for inspection. + +### Run the native qualification in GitHub Actions + +After independent review of the exact candidate, a maintainer can dispatch the +existing `e2e-live.yml` workflow in `stacklok/mecatl`: + +```sh +gh workflow run e2e-live.yml --repo stacklok/mecatl --ref \ + -f native_execution=true -f expected_sha= +``` + +`native_execution` defaults to `false`. Setting it to `true` runs only the native +job, avoiding duplicate paid inference in the ordinary live jobs. Schedules and +labelled PR runs retain their ordinary behavior and never select this job. +`expected_sha` is optional, but supply it for reviewed qualification: checkout +uses the dispatch event SHA, verifies exact equality before credential staging, +and records that SHA in the job summary. A moved branch fails the check. + +The trust gates are maintainer dispatch, the `stacklok/mecatl` repository check, +and access to the existing `OPENROUTER_API_KEY` repository secret. There is no +GitHub environment-approval gate. Review the workflow and all executed source +before dispatch; the SHA check establishes identity, not code safety. Checkout +retains no Git credentials, the token has read-only contents access, and the +native job uses no build cache. + +The same runner first completes the production Kind+Calico qualification without +a provider key or automatic cluster deletion. Only a successful production step +allows credential staging; a production ownership record alone is insufficient. +The workflow stages the key in a new 0700 CI directory and 0600 file, then removes +it from the environment before `live.sh` validates the retained owned state, +reruns mock qualification, and invokes the trusted credential loader. A missing +secret fails qualification rather than skipping successfully. The 100-minute job +has explicit step ceilings: setup 9 minutes, production 50, credential staging 1, +live 25, cleanup 10, and status/artifact reporting 3, leaving 2 minutes of overhead. +Production and live subprocess groups receive TERM after 49 and 24 minutes, +respectively, then KILL after a 15-second grace. These stage ceilings are +intentionally smaller than the sum of all subordinate build, rollout, and test +bounds. Hitting one fails qualification; it does not prove completion. Per-agent +token limits remain unchanged and are not a hard dollar cap. + +The live phase preserves the qualified cluster's local-path storage helper and +security configuration; rebuilding the executor image does not replace the +storage provisioner's helper image. + +The live script attempts to restore mock configuration and delete its Secret by +recorded UID using an independent cleanup context. A signal or stage deadline can +interrupt that attempt. An always-run workflow step has a separate 10-minute +ceiling to remove the CI-created credential file and delete the owned cluster. +Production publishes its state path and cluster identity after collision checks +and before cluster creation; live and cleanup use those fixed step outputs, not +the mutable local `current` pointer. Cleanup validates the recorded owner, name, +namespace, profile, container label, private kubeconfig, and context against that +identity. Ownership drift fails cleanup without selecting another cluster. +Failure before kubeconfig creation also fails cleanup without deleting a cluster. +Cleanup is bounded best effort: hard cancellation or runner loss can prevent it. +Hosted-runner disposal is the fallback, not evidence that cleanup succeeded. + +Artifacts are retained for seven days: a size-bounded `live-summary.json` when +available and a sanitized `qualification-status.txt`. On live failure, +`live-diagnostics.jsonl` captures the real process before mock restoration and +Secret cleanup. If this capture is missing or incomplete, or production fails, +workflow cleanup collects separate `production-diagnostics.jsonl` evidence before +cluster deletion. `diagnostics-status.txt` distinguishes pre-restoration and +fallback collection outcomes. Collection failure warns and still attempts cleanup. + +Each collector has a 45-second bound, 5-second API deadlines, and a 1 MiB output +limit. Resource evidence contains known condition/reason classes, deletion and +UID-match booleans, known finalizers, Pod phases, container exit reasons, +lease-expiry status, quota key names, and event reason counts. Log evidence passes +through a strict JSON allowlist in memory; only create stages, fixed reason +classes, elapsed milliseconds, call counts, and hashed session correlation survive. +Raw logs, stacks, PKI, kubeconfigs, Secret receipts, Helm values, resource manifests, +transcripts, and surrounding scratch directories are excluded from artifacts. + +The fixture enables `--log-level=debug`; native-execution debug diagnostics use +JSON. Create stages distinguish handler entry after authentication, session-ID +storage probing, remote Ensure, Attach polling, engine construction, persistence, +reference publication, and HTTP response construction. Polling logs its first +state, state changes, and final count rather than every retry. The live test first +checks authenticated HTTP reachability separately, then reports when create +headers were sent and a response started. A last `begin` stage without a matching +completion identifies the boundary to investigate; it does not establish the root +cause. Check live, diagnostics, and cleanup outcomes before treating the recorded +commit as qualified. + ## Limitations - No-FS sessions cannot use local file tools, shell commands, workspace forks,